158 lines
6.2 KiB
Python
158 lines
6.2 KiB
Python
"""客户端运行日志专用落盘 writer(独立于服务端 app-server.log)。
|
|
|
|
- 独占 logger "shagua.client_log" + 自己的 RotatingFileHandler,propagate=False → 不污染 app-server.log。
|
|
- 每条按「白名单键(client_ts/level/trace_id/tag/msg)提顶层 + 其余并入 data」封装,再
|
|
json.dumps 成一行写出(钉死 SLS 索引列;见 spec §5)。formatter 用 %(message)s——行本身
|
|
已是 JSON,不能再过 JsonFormatter 二次编码。
|
|
- 滚动 20MB×10(env 可调),与服务日志同机制。
|
|
⚠️ 依赖 --workers 1:RotatingFileHandler 多进程并发 doRollover 会损坏/丢日志;扩 worker
|
|
前换 QueueHandler→单写入者 / 外部 logrotate(copytruncate) / 写 stdout 交 journald。
|
|
|
|
服务端补的字段(time/source/service/client_ip/device_id/...)是「事实」,与客户端自述分开。
|
|
`time` 用服务端接收时间作 SLS 主时间(客户端时钟不可信),client_ts 另存为可查字段。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import datetime
|
|
from logging.handlers import RotatingFileHandler
|
|
from pathlib import Path
|
|
|
|
# 仅这些客户端键提到输出行顶层;其余(含客户端自带 data)一律并入 data,防 SLS 索引列爆炸
|
|
_TOP_LEVEL_KEYS = ("client_ts", "level", "trace_id", "tag", "msg")
|
|
|
|
# trace_id/tag/level 是 SLS 索引字段(spec §8):给长度上限,防客户端塞超大值撑爆索引/抬升成本。
|
|
# (msg 另有字节截断;data 内的值不限,留待后续「服务端脱敏」knob。)
|
|
_MAX_LEVEL_LEN = 16
|
|
_MAX_TRACE_ID_LEN = 256
|
|
_MAX_TAG_LEN = 128
|
|
|
|
_logger: logging.Logger | None = None
|
|
|
|
|
|
def _max_msg_bytes() -> int:
|
|
# 每次调用现读 env(不设模块级常量):便于运行期调整 / 测试 monkeypatch,开销可忽略。
|
|
return int(os.getenv("APPLOG_MAX_MSG_BYTES", "8192"))
|
|
|
|
|
|
def _build_logger() -> logging.Logger:
|
|
lg = logging.getLogger("shagua.client_log") # 与仓库 shagua.* 业务 logger 命名一致
|
|
lg.setLevel(logging.INFO)
|
|
lg.propagate = False # 不冒泡到 root → 不写进 app-server.log
|
|
log_file = os.getenv("CLIENT_LOG_FILE") or str(
|
|
Path(os.getenv("LOG_DIR", "logs")) / "app-client.log"
|
|
)
|
|
Path(log_file).parent.mkdir(parents=True, exist_ok=True)
|
|
handler = RotatingFileHandler(
|
|
log_file,
|
|
maxBytes=int(os.getenv("CLIENT_LOG_MAX_BYTES", str(20 * 1024 * 1024))),
|
|
backupCount=int(os.getenv("CLIENT_LOG_BACKUP_COUNT", "10")),
|
|
encoding="utf-8",
|
|
)
|
|
handler.setFormatter(logging.Formatter("%(message)s")) # 行已是 JSON,不再包装
|
|
lg.handlers = [handler]
|
|
return lg
|
|
|
|
|
|
def get_logger() -> logging.Logger:
|
|
global _logger
|
|
if _logger is None:
|
|
_logger = _build_logger()
|
|
return _logger
|
|
|
|
|
|
def reset_client_logger() -> None:
|
|
"""测试用:关闭并丢弃当前 logger,使下次 get_logger 按当时 env 重建(切临时文件)。"""
|
|
global _logger
|
|
if _logger is not None:
|
|
for h in list(_logger.handlers):
|
|
h.close()
|
|
_logger.handlers = []
|
|
_logger = None
|
|
|
|
|
|
def _truncate_msg(msg: str) -> tuple[str, bool]:
|
|
raw = msg.encode("utf-8")
|
|
limit = _max_msg_bytes()
|
|
if len(raw) <= limit:
|
|
return msg, False
|
|
# 按字节截断后解码,忽略截断处半个多字节字符
|
|
return raw[:limit].decode("utf-8", "ignore") + "…[truncated]", True
|
|
|
|
|
|
def _build_line(
|
|
record: dict, *, meta: dict, client_ip: str, service: str, now_iso: str
|
|
) -> str:
|
|
out: dict = {
|
|
"time": now_iso,
|
|
"source": "client",
|
|
"service": service,
|
|
"client_ip": client_ip,
|
|
}
|
|
# 批级公共字段(非空才带)
|
|
for k in ("device_id", "user_id", "app_ver", "platform", "sent_at"):
|
|
v = meta.get(k)
|
|
if v is not None:
|
|
out[k] = v
|
|
# 白名单键提顶层(索引字段做长度上限 + 统一转 str,保证 SLS 里类型/大小可控)
|
|
if record.get("level") is not None:
|
|
out["level"] = str(record["level"])[:_MAX_LEVEL_LEN].upper()
|
|
if record.get("trace_id"):
|
|
out["trace_id"] = str(record["trace_id"])[:_MAX_TRACE_ID_LEN]
|
|
if record.get("tag"):
|
|
out["tag"] = str(record["tag"])[:_MAX_TAG_LEN]
|
|
if record.get("client_ts") is not None:
|
|
out["client_ts"] = record["client_ts"]
|
|
if record.get("msg") is not None:
|
|
msg, truncated = _truncate_msg(str(record["msg"]))
|
|
out["msg"] = msg
|
|
if truncated:
|
|
out["msg_truncated"] = True
|
|
# 其余键并入 data:先收白名单外的散键(兜底),再让客户端显式的 data 覆盖同名散键
|
|
# —— 显式 data 为准,不静默丢客户端明确给的值(保真)。
|
|
data: dict = {}
|
|
for k, v in record.items():
|
|
if k in _TOP_LEVEL_KEYS or k == "data":
|
|
continue
|
|
data[k] = v
|
|
client_data = record.get("data")
|
|
if isinstance(client_data, dict):
|
|
data.update(client_data)
|
|
if data:
|
|
out["data"] = data
|
|
return json.dumps(out, ensure_ascii=False, default=str)
|
|
|
|
|
|
def write_records(
|
|
records: list[dict], *, meta: dict, client_ip: str
|
|
) -> tuple[int, int]:
|
|
"""把一批客户端日志逐条写入专用文件。返回 (received, dropped)。
|
|
|
|
尽力而为(fire-and-forget):logger 初始化或单条写入失败只跳过并计 dropped,
|
|
不抛给上层——端点因此永不因写日志而 500。
|
|
"""
|
|
try:
|
|
lg = get_logger()
|
|
except Exception: # noqa: BLE001 — 初始化失败也不能让端点 500
|
|
logging.getLogger("shagua.applog").exception("client log writer init failed")
|
|
return 0, len(records)
|
|
service = os.getenv("CLIENT_LOG_SERVICE_NAME", "app-client")
|
|
# 一批共用同一「服务端接收时间」:降开销,且语义上是服务端「收到」而非逐条「处理」时间。
|
|
now_iso = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
|
|
received = dropped = 0
|
|
for rec in records:
|
|
try:
|
|
line = _build_line(
|
|
rec, meta=meta, client_ip=client_ip, service=service, now_iso=now_iso
|
|
)
|
|
lg.info(line)
|
|
received += 1
|
|
except Exception: # noqa: BLE001 — 坏条跳过,不影响其余
|
|
logging.getLogger("shagua.applog").exception(
|
|
"client log record dropped client_ip=%s", client_ip
|
|
)
|
|
dropped += 1
|
|
return received, dropped
|