63daeeaf9b
线上 trace 几乎总可读,旧逻辑「可读但没判出卡点 → 不报」使 total_ms>90s 阈值形同 虚设,超长放弃(实测 113s / 516s)一条都报不出。改为只有判出卡点才独占带卡点的 T5, 其余(可读没卡点 / 读不到 / 无 trace)一律回退耗时/步数兜底,超长照报(卡点列留空)。 同步更新 stuck-detection 设计文档 6.1 + 修订说明。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
146 lines
6.3 KiB
Python
146 lines
6.3 KiB
Python
"""build_hits 集成测:cancelled trace 优先/保底切换、failed 附卡点、限量。"""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from app.core.compare_alert_worker import _fmt_stuck, build_hits
|
|
from app.services.trace_stuck import StuckPoint
|
|
|
|
|
|
class _Rec:
|
|
def __init__(self, **kw):
|
|
self.trace_id = kw.get("trace_id", "t")
|
|
self.status = kw.get("status", "cancelled")
|
|
self.total_ms = kw.get("total_ms")
|
|
self.step_count = kw.get("step_count")
|
|
self.fail_reason = kw.get("fail_reason")
|
|
self.information = kw.get("information")
|
|
self.app_version = kw.get("app_version")
|
|
self.created_at = kw.get("created_at")
|
|
self.trace_url = kw.get("trace_url")
|
|
self.user_id = kw.get("user_id")
|
|
|
|
|
|
def _frame(pdir: Path, idx: int, step: str, page: str) -> None:
|
|
pdir.mkdir(parents=True, exist_ok=True)
|
|
body = {"pipeline_step": step, "detected_page": page, "windows": [{"n": ["x" * 200]}]}
|
|
(pdir / f"step_{idx:03d}.json").write_text(
|
|
json.dumps(body, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
|
|
|
|
_KW = dict(
|
|
stuck_threshold=15, max_tail=40, max_trace_reads=30,
|
|
cancelled_ms_threshold=90000, cancelled_step_threshold=30,
|
|
timeout_keywords=("超时",), unrecognized_keywords=("未识别",), biz_exclude_keywords=(),
|
|
)
|
|
|
|
|
|
def test_cancelled_stuck_reports_via_trace(tmp_path):
|
|
for i in range(18):
|
|
_frame(tmp_path / "20260804_x" / "meituan", i, "add_one_dish", "meal_detail_popup")
|
|
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_x/",
|
|
total_ms=5000, step_count=3) # 保底不会中,靠 trace 判卡死
|
|
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
|
|
assert len(hits) == 1
|
|
assert hits[0].alert_type == "T5"
|
|
assert hits[0].reason == "深度放弃"
|
|
assert "美团·加菜" in hits[0].stuck_point
|
|
assert "帧" in hits[0].stuck_point
|
|
|
|
|
|
def test_cancelled_readable_not_stuck_long_duration_reports(tmp_path):
|
|
# trace 可读但没判出原地卡点:仍过耗时/步数阈值兜底,超长(>90s)照报 T5,卡点列留空。
|
|
# (线上 trace 几乎总可读,若不回退则 total_ms 阈值形同虚设、超长放弃永不报——见 compare-fail-alert 排查。)
|
|
p = tmp_path / "20260804_y" / "eleme"
|
|
_frame(p, 0, "set_address", "home")
|
|
for i in range(1, 6):
|
|
_frame(p, i, "enter_store", "store")
|
|
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_y/",
|
|
total_ms=95000, step_count=40)
|
|
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
|
|
assert len(hits) == 1
|
|
assert hits[0].alert_type == "T5"
|
|
assert hits[0].reason == "深度放弃"
|
|
assert hits[0].stuck_point is None # 没卡点 → 卡片卡点列显 "-"
|
|
|
|
|
|
def test_cancelled_readable_not_stuck_short_no_report(tmp_path):
|
|
# trace 可读没卡点、且耗时/步数都没超阈值 → 正常早退,不报(兜底阈值把住,不误报)。
|
|
p = tmp_path / "20260804_ys" / "eleme"
|
|
_frame(p, 0, "set_address", "home")
|
|
for i in range(1, 6):
|
|
_frame(p, i, "enter_store", "store")
|
|
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_ys/",
|
|
total_ms=5000, step_count=3)
|
|
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
|
|
assert hits == []
|
|
|
|
|
|
def test_cancelled_unreadable_falls_back(tmp_path):
|
|
rec = _Rec(status="cancelled", trace_url="https://x/traces/nope/",
|
|
total_ms=95000, step_count=3)
|
|
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
|
|
assert len(hits) == 1
|
|
assert hits[0].alert_type == "T5" and "深度放弃" in hits[0].reason
|
|
|
|
|
|
def test_no_work_log_dir_uses_fallback(tmp_path):
|
|
rec = _Rec(status="cancelled", trace_url="https://x/traces/y/",
|
|
total_ms=95000, step_count=3)
|
|
hits = build_hits([rec], work_log_dir="", **_KW)
|
|
assert len(hits) == 1 and "深度放弃" in hits[0].reason
|
|
|
|
|
|
def test_failed_gets_stuck_point_appended(tmp_path):
|
|
for i in range(20):
|
|
_frame(tmp_path / "20260804_f" / "meituan", i, "add_one_dish", "meal_detail_popup")
|
|
rec = _Rec(status="failed", fail_reason="启动超时",
|
|
trace_url="https://x/traces/20260804_f/")
|
|
hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW)
|
|
assert len(hits) == 1
|
|
assert hits[0].alert_type == "T2"
|
|
assert "美团·加菜" in hits[0].stuck_point
|
|
assert "卡在" not in hits[0].reason
|
|
|
|
|
|
def test_max_trace_reads_zero_skips_trace(tmp_path):
|
|
for i in range(18):
|
|
_frame(tmp_path / "20260804_z" / "meituan", i, "add_one_dish", "meal_detail_popup")
|
|
rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_z/",
|
|
total_ms=95000, step_count=3)
|
|
kw = {**_KW, "max_trace_reads": 0}
|
|
hits = build_hits([rec], work_log_dir=str(tmp_path), **kw)
|
|
# 没读 trace → 回退保底 → deep(95s) → 深度放弃
|
|
assert len(hits) == 1 and "深度放弃" in hits[0].reason
|
|
|
|
|
|
def test_shared_reads_budget_across_branches(tmp_path):
|
|
# cancelled 和 failed 共用 max_trace_reads 预算;预算=1 时 cancelled 先消耗,failed 拿不到卡点
|
|
for i in range(18):
|
|
_frame(tmp_path / "20260804_c" / "meituan", i, "add_one_dish", "meal_detail_popup")
|
|
for i in range(20):
|
|
_frame(tmp_path / "20260804_d" / "meituan", i, "add_one_dish", "meal_detail_popup")
|
|
cancelled = _Rec(status="cancelled", trace_url="https://x/traces/20260804_c/",
|
|
total_ms=5000, step_count=3)
|
|
failed = _Rec(status="failed", fail_reason="启动超时",
|
|
trace_url="https://x/traces/20260804_d/")
|
|
kw = {**_KW, "max_trace_reads": 1}
|
|
hits = build_hits([cancelled, failed], work_log_dir=str(tmp_path), **kw)
|
|
assert len(hits) == 2
|
|
# cancelled 消耗了唯一预算 → 报卡死,卡点在 stuck_point
|
|
assert hits[0].alert_type == "T5" and hits[0].stuck_point is not None
|
|
# failed 超预算 → 仍报 T2,但 stuck_point 为 None
|
|
assert hits[1].alert_type == "T2" and hits[1].stuck_point is None
|
|
|
|
|
|
# ---- _fmt_stuck 单测 ----
|
|
|
|
def test_fmt_stuck_with_ms():
|
|
sp = StuckPoint(platform="meituan", pipeline_step="add_one_dish", frames=110, stuck_ms=32000)
|
|
assert _fmt_stuck(sp) == "美团·加菜 110帧/32s"
|
|
|
|
|
|
def test_fmt_stuck_without_ms():
|
|
sp = StuckPoint(platform="meituan", pipeline_step="add_one_dish", frames=110, stuck_ms=None)
|
|
assert _fmt_stuck(sp) == "美团·加菜 110帧"
|