31f61f6aad
## 背景 PR #162 已合并。本 PR 为其后续日志增强,方便人工按一次任务完整回查美团、京东每日自动对账。 ## 日志内容 - 每次运行生成唯一 `run_id`,记录 `scheduled` / `startup_catchup` 触发来源 - 记录北京时间计划日期、近 3 天回拉窗口、环境、数据库方言、主机名和 PID - 分平台记录开始、成功、跳过、失败及执行耗时 - 成功结果记录 fetched / inserted / updated / pages / api_requests,京东额外记录小时窗口数 - 京东上游失败记录具体小时窗口、页码和请求序号;美团记录失败页码 - 最终汇总记录 success / partial_success / failed、失败平台、是否需要人工补跑和下次执行时间 - 日志使用现有 `extra` 结构化字段写入 JSON 日志,便于 SLS/人工检索 ## 兼容性 - 手动对账接口、事务和返回模型不变 - 不记录密钥、Token、完整上游响应或订单明细 - 仓储层仅增加可选审计上下文和请求计数;不改变拉取及 upsert 逻辑 ## 验证 - `pytest tests/test_cps_reconcile_worker.py tests/test_cps_admin.py tests/test_admin_read.py tests/test_observe.py -q`:44 passed - 相关文件 Ruff 检查通过(忽略文件原有 UP017 提示) - `git diff --check` 通过 --------- Co-authored-by: unknown <798648091@qq.com> Reviewed-on: #163 Co-authored-by: linkeyu <linkeyu@wonderable.ai> Co-committed-by: linkeyu <linkeyu@wonderable.ai>
198 lines
7.2 KiB
Python
198 lines
7.2 KiB
Python
"""美团、京东 CPS 每日自动对账 worker。"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import date, datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from app.core import cps_reconcile_worker as worker
|
|
from app.core.config import settings
|
|
from app.core.rewards import CN_TZ
|
|
from app.db.session import SessionLocal
|
|
from app.integrations.meituan import MeituanCpsError
|
|
|
|
|
|
def _configure_platforms(monkeypatch) -> None:
|
|
monkeypatch.setattr(settings, "MT_CPS_APP_KEY", "mt-key")
|
|
monkeypatch.setattr(settings, "MT_CPS_APP_SECRET", "mt-secret")
|
|
monkeypatch.setattr(settings, "JD_UNION_APP_KEY", "jd-key")
|
|
monkeypatch.setattr(settings, "JD_UNION_APP_SECRET", "jd-secret")
|
|
monkeypatch.setattr(settings, "CPS_AUTO_RECONCILE_LOOKBACK_DAYS", 3)
|
|
|
|
|
|
def test_reconcile_once_pulls_meituan_and_jd_by_update_time(monkeypatch) -> None:
|
|
_configure_platforms(monkeypatch)
|
|
calls: dict[str, dict] = {}
|
|
|
|
def fake_meituan(db, **kwargs):
|
|
calls["meituan"] = kwargs
|
|
return {
|
|
"fetched": 2,
|
|
"inserted": 1,
|
|
"updated": 1,
|
|
"pages": 1,
|
|
"api_requests": 1,
|
|
}
|
|
|
|
def fake_jd(db, **kwargs):
|
|
calls["jd"] = kwargs
|
|
return {
|
|
"fetched": 3,
|
|
"inserted": 2,
|
|
"updated": 1,
|
|
"pages": 2,
|
|
"api_requests": 72,
|
|
"windows": 72,
|
|
}
|
|
|
|
monkeypatch.setattr(worker.cps_repo, "reconcile_orders", fake_meituan)
|
|
monkeypatch.setattr(worker.cps_repo, "reconcile_jd_orders", fake_jd)
|
|
monkeypatch.setattr(worker, "_touch_lock", lambda: None)
|
|
now = datetime(2026, 7, 22, 5, 0, tzinfo=CN_TZ)
|
|
|
|
result = worker._reconcile_once(now)
|
|
|
|
assert result["errors"] == {}
|
|
assert result["meituan"]["fetched"] == 2
|
|
assert result["jd"]["fetched"] == 3
|
|
assert calls["meituan"]["query_time_type"] == 2
|
|
assert calls["jd"]["query_time_type"] == 3
|
|
assert calls["meituan"]["end_time"] == int(now.timestamp())
|
|
assert calls["meituan"]["start_time"] == int((now - timedelta(days=3)).timestamp())
|
|
assert calls["jd"]["start_time"] == now - timedelta(days=3)
|
|
assert calls["jd"]["end_time"] == now
|
|
assert calls["meituan"]["audit_context"]["run_id"] == result["run_id"]
|
|
assert calls["jd"]["audit_context"]["run_id"] == result["run_id"]
|
|
assert result["status"] == "success"
|
|
assert result["manual_retry_required"] is False
|
|
|
|
|
|
def test_meituan_failure_does_not_block_jd(monkeypatch) -> None:
|
|
_configure_platforms(monkeypatch)
|
|
jd_called = False
|
|
|
|
def fail_meituan(db, **kwargs):
|
|
raise MeituanCpsError("temporary failure")
|
|
|
|
def fake_jd(db, **kwargs):
|
|
nonlocal jd_called
|
|
jd_called = True
|
|
return {"fetched": 1, "inserted": 1, "updated": 0, "pages": 1}
|
|
|
|
monkeypatch.setattr(worker.cps_repo, "reconcile_orders", fail_meituan)
|
|
monkeypatch.setattr(worker.cps_repo, "reconcile_jd_orders", fake_jd)
|
|
monkeypatch.setattr(worker, "_touch_lock", lambda: None)
|
|
|
|
result = worker._reconcile_once(datetime(2026, 7, 22, 5, 0, tzinfo=CN_TZ))
|
|
|
|
assert result["errors"]["meituan"] == "temporary failure"
|
|
assert jd_called is True
|
|
assert result["jd"]["fetched"] == 1
|
|
assert result["meituan"]["status"] == "failed"
|
|
assert result["status"] == "partial_success"
|
|
assert result["manual_retry_required"] is True
|
|
|
|
|
|
def test_should_run_once_after_five_am() -> None:
|
|
before = datetime(2026, 7, 22, 4, 59, tzinfo=CN_TZ)
|
|
at_five = datetime(2026, 7, 22, 5, 0, tzinfo=CN_TZ)
|
|
|
|
assert worker._should_run(None, before, 5) is False
|
|
assert worker._should_run(None, at_five, 5) is True
|
|
assert worker._should_run(date(2026, 7, 22), at_five, 5) is False
|
|
|
|
|
|
def test_trigger_and_next_run_are_beijing_time() -> None:
|
|
before_five = datetime(2026, 7, 22, 4, 30, tzinfo=CN_TZ)
|
|
after_five = datetime(2026, 7, 22, 6, 0, tzinfo=CN_TZ)
|
|
|
|
assert worker._trigger_for_run(before_five, after_five, 5) == "scheduled"
|
|
assert worker._trigger_for_run(after_five, after_five, 5) == "startup_catchup"
|
|
assert worker._next_run_at(before_five, 5) == datetime(2026, 7, 22, 5, 0, tzinfo=CN_TZ)
|
|
assert worker._next_run_at(after_five, 5) == datetime(2026, 7, 23, 5, 0, tzinfo=CN_TZ)
|
|
|
|
|
|
def test_reconcile_emits_structured_audit_logs(monkeypatch, caplog) -> None:
|
|
_configure_platforms(monkeypatch)
|
|
monkeypatch.setattr(
|
|
worker.cps_repo,
|
|
"reconcile_orders",
|
|
lambda db, **kwargs: {
|
|
"fetched": 2,
|
|
"inserted": 1,
|
|
"updated": 1,
|
|
"pages": 1,
|
|
"api_requests": 1,
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
worker.cps_repo,
|
|
"reconcile_jd_orders",
|
|
lambda db, **kwargs: {
|
|
"fetched": 3,
|
|
"inserted": 2,
|
|
"updated": 1,
|
|
"pages": 1,
|
|
"api_requests": 72,
|
|
"windows": 72,
|
|
},
|
|
)
|
|
monkeypatch.setattr(worker, "_touch_lock", lambda: None)
|
|
monkeypatch.setattr(
|
|
worker,
|
|
"_cn_now",
|
|
lambda: datetime(2026, 7, 22, 5, 1, tzinfo=CN_TZ),
|
|
)
|
|
|
|
with caplog.at_level(logging.INFO, logger=worker.logger.name):
|
|
result = worker._reconcile_once(
|
|
datetime(2026, 7, 22, 5, 0, tzinfo=CN_TZ),
|
|
run_id="audit-run-1",
|
|
trigger="scheduled",
|
|
)
|
|
|
|
records = {record.event: record for record in caplog.records if hasattr(record, "event")}
|
|
assert records["cps_reconcile.started"].run_id == "audit-run-1"
|
|
assert records["cps_reconcile.started"].lookback_days == 3
|
|
assert records["cps_reconcile.started"].scheduled_date == "2026-07-22"
|
|
assert records["cps_reconcile.started"].hostname
|
|
assert records["cps_reconcile.completed"].task_status == "success"
|
|
assert records["cps_reconcile.completed"].manual_retry_required is False
|
|
assert records["cps_reconcile.completed"].meituan_result["fetched"] == 2
|
|
assert records["cps_reconcile.completed"].jd_result["api_requests"] == 72
|
|
assert result["next_run_at"] == "2026-07-23T05:00:00+08:00"
|
|
|
|
|
|
def test_jd_request_failure_logs_exact_window(monkeypatch, caplog) -> None:
|
|
start = datetime(2026, 7, 20, 5, 0, tzinfo=CN_TZ)
|
|
|
|
def fail_query(**kwargs):
|
|
raise RuntimeError("upstream timeout")
|
|
|
|
monkeypatch.setattr(worker.cps_repo.jd_union, "query_order_rows", fail_query)
|
|
with SessionLocal() as db, caplog.at_level(logging.ERROR, logger=worker.logger.name):
|
|
with pytest.raises(RuntimeError, match="upstream timeout"):
|
|
worker.cps_repo.reconcile_jd_orders(
|
|
db,
|
|
start_time=start,
|
|
end_time=start + timedelta(hours=2),
|
|
audit_context={"run_id": "audit-run-2", "trigger": "scheduled"},
|
|
)
|
|
|
|
record = next(
|
|
item
|
|
for item in caplog.records
|
|
if getattr(item, "event", "") == "cps_reconcile.request_failed"
|
|
)
|
|
assert record.run_id == "audit-run-2"
|
|
assert record.platform == "jd"
|
|
assert record.failed_window_start == "2026-07-20T05:00:00+08:00"
|
|
assert record.failed_window_end == "2026-07-20T06:00:00+08:00"
|
|
assert record.failed_page == 1
|
|
|
|
|
|
def test_start_worker_respects_auto_switch(monkeypatch) -> None:
|
|
monkeypatch.setattr(settings, "CPS_AUTO_RECONCILE_ENABLED", False)
|
|
assert worker.start_cps_reconcile_worker() is None
|