Files
shaguabijia-app-server/tests/test_compare_alert_stuck_worker.py
T
guke 63daeeaf9b fix(compare-alert): cancelled trace 可读但无卡点时回退耗时兜底(超长放弃不再漏报)
线上 trace 几乎总可读,旧逻辑「可读但没判出卡点 → 不报」使 total_ms>90s 阈值形同
虚设,超长放弃(实测 113s / 516s)一条都报不出。改为只有判出卡点才独占带卡点的 T5,
其余(可读没卡点 / 读不到 / 无 trace)一律回退耗时/步数兜底,超长照报(卡点列留空)。
同步更新 stuck-detection 设计文档 6.1 + 修订说明。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-06 13:52:48 +08:00

146 lines
6.3 KiB
Python

"""build_hits 集成测:cancelled trace 优先/保底切换、failed 附卡点、限量。"""
import json
from pathlib import Path
from app.core.compare_alert_worker import _fmt_stuck, build_hits
from app.services.trace_stuck import StuckPoint
class _Rec:
def __init__(self, **kw):
self.trace_id = kw.get("trace_id", "t")
self.status = kw.get("status", "cancelled")
self.total_ms = kw.get("total_ms")
self.step_count = kw.get("step_count")
self.fail_reason = kw.get("fail_reason")
self.information = kw.get("information")
self.app_version = kw.get("app_version")
self.created_at = kw.get("created_at")
self.trace_url = kw.get("trace_url")
self.user_id = kw.get("user_id")
def _frame(pdir: Path, idx: int, step: str, page: str) -> None:
pdir.mkdir(parents=True, exist_ok=True)
body = {"pipeline_step": step, "detected_page": page, "windows": [{"n": ["x" * 200]}]}
(pdir / f"step_{idx:03d}.json").write_text(
json.dumps(body, ensure_ascii=False), encoding="utf-8"
)
_KW = dict(
stuck_threshold=15, max_tail=40, max_trace_reads=30,
cancelled_ms_threshold=90000, cancelled_step_threshold=30,
timeout_keywords=("超时",), unrecognized_keywords=("未识别",), biz_exclude_keywords=(),
)
def test_cancelled_stuck_reports_via_trace(tmp_path):
for i in range(18):
_frame(tmp_path / "20260804_x" / "meituan", i, "add_one_dish", "meal_detail_popup")
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_x/",
total_ms=5000, step_count=3) # 保底不会中,靠 trace 判卡死
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
assert len(hits) == 1
assert hits[0].alert_type == "T5"
assert hits[0].reason == "深度放弃"
assert "美团·加菜" in hits[0].stuck_point
assert "" in hits[0].stuck_point
def test_cancelled_readable_not_stuck_long_duration_reports(tmp_path):
# trace 可读但没判出原地卡点:仍过耗时/步数阈值兜底,超长(>90s)照报 T5,卡点列留空。
# (线上 trace 几乎总可读,若不回退则 total_ms 阈值形同虚设、超长放弃永不报——见 compare-fail-alert 排查。)
p = tmp_path / "20260804_y" / "eleme"
_frame(p, 0, "set_address", "home")
for i in range(1, 6):
_frame(p, i, "enter_store", "store")
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_y/",
total_ms=95000, step_count=40)
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
assert len(hits) == 1
assert hits[0].alert_type == "T5"
assert hits[0].reason == "深度放弃"
assert hits[0].stuck_point is None # 没卡点 → 卡片卡点列显 "-"
def test_cancelled_readable_not_stuck_short_no_report(tmp_path):
# trace 可读没卡点、且耗时/步数都没超阈值 → 正常早退,不报(兜底阈值把住,不误报)。
p = tmp_path / "20260804_ys" / "eleme"
_frame(p, 0, "set_address", "home")
for i in range(1, 6):
_frame(p, i, "enter_store", "store")
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_ys/",
total_ms=5000, step_count=3)
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
assert hits == []
def test_cancelled_unreadable_falls_back(tmp_path):
rec = _Rec(status="cancelled", trace_url="https://x/traces/nope/",
total_ms=95000, step_count=3)
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
assert len(hits) == 1
assert hits[0].alert_type == "T5" and "深度放弃" in hits[0].reason
def test_no_work_log_dir_uses_fallback(tmp_path):
rec = _Rec(status="cancelled", trace_url="https://x/traces/y/",
total_ms=95000, step_count=3)
hits = build_hits([rec], work_log_dir="", **_KW)
assert len(hits) == 1 and "深度放弃" in hits[0].reason
def test_failed_gets_stuck_point_appended(tmp_path):
for i in range(20):
_frame(tmp_path / "20260804_f" / "meituan", i, "add_one_dish", "meal_detail_popup")
rec = _Rec(status="failed", fail_reason="启动超时",
trace_url="https://x/traces/20260804_f/")
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
assert len(hits) == 1
assert hits[0].alert_type == "T2"
assert "美团·加菜" in hits[0].stuck_point
assert "卡在" not in hits[0].reason
def test_max_trace_reads_zero_skips_trace(tmp_path):
for i in range(18):
_frame(tmp_path / "20260804_z" / "meituan", i, "add_one_dish", "meal_detail_popup")
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_z/",
total_ms=95000, step_count=3)
kw = {**_KW, "max_trace_reads": 0}
hits = build_hits([rec], work_log_dir=str(tmp_path), **kw)
# 没读 trace → 回退保底 → deep(95s) → 深度放弃
assert len(hits) == 1 and "深度放弃" in hits[0].reason
def test_shared_reads_budget_across_branches(tmp_path):
# cancelled 和 failed 共用 max_trace_reads 预算;预算=1 时 cancelled 先消耗,failed 拿不到卡点
for i in range(18):
_frame(tmp_path / "20260804_c" / "meituan", i, "add_one_dish", "meal_detail_popup")
for i in range(20):
_frame(tmp_path / "20260804_d" / "meituan", i, "add_one_dish", "meal_detail_popup")
cancelled = _Rec(status="cancelled", trace_url="https://x/traces/20260804_c/",
total_ms=5000, step_count=3)
failed = _Rec(status="failed", fail_reason="启动超时",
trace_url="https://x/traces/20260804_d/")
kw = {**_KW, "max_trace_reads": 1}
hits = build_hits([cancelled, failed], work_log_dir=str(tmp_path), **kw)
assert len(hits) == 2
# cancelled 消耗了唯一预算 → 报卡死,卡点在 stuck_point
assert hits[0].alert_type == "T5" and hits[0].stuck_point is not None
# failed 超预算 → 仍报 T2,但 stuck_point 为 None
assert hits[1].alert_type == "T2" and hits[1].stuck_point is None
# ---- _fmt_stuck 单测 ----
def test_fmt_stuck_with_ms():
sp = StuckPoint(platform="meituan", pipeline_step="add_one_dish", frames=110, stuck_ms=32000)
assert _fmt_stuck(sp) == "美团·加菜 110帧/32s"
def test_fmt_stuck_without_ms():
sp = StuckPoint(platform="meituan", pipeline_step="add_one_dish", frames=110, stuck_ms=None)
assert _fmt_stuck(sp) == "美团·加菜 110帧"