From 63daeeaf9bbe2a4fd508df746583d5ebfe3c873d Mon Sep 17 00:00:00 2001 From: guke Date: Thu, 6 Aug 2026 13:52:48 +0800 Subject: [PATCH] =?UTF-8?q?fix(compare-alert):=20cancelled=20trace=20?= =?UTF-8?q?=E5=8F=AF=E8=AF=BB=E4=BD=86=E6=97=A0=E5=8D=A1=E7=82=B9=E6=97=B6?= =?UTF-8?q?=E5=9B=9E=E9=80=80=E8=80=97=E6=97=B6=E5=85=9C=E5=BA=95(?= =?UTF-8?q?=E8=B6=85=E9=95=BF=E6=94=BE=E5=BC=83=E4=B8=8D=E5=86=8D=E6=BC=8F?= =?UTF-8?q?=E6=8A=A5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 线上 trace 几乎总可读,旧逻辑「可读但没判出卡点 → 不报」使 total_ms>90s 阈值形同 虚设,超长放弃(实测 113s / 516s)一条都报不出。改为只有判出卡点才独占带卡点的 T5, 其余(可读没卡点 / 读不到 / 无 trace)一律回退耗时/步数兜底,超长照报(卡点列留空)。 同步更新 stuck-detection 设计文档 6.1 + 修订说明。 Co-Authored-By: Claude Opus 4.8 --- app/core/compare_alert_worker.py | 20 ++++++++++--------- ...26-08-05-compare-stuck-detection-design.md | 17 ++++++++++------ tests/test_compare_alert_stuck_worker.py | 20 +++++++++++++++++-- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/app/core/compare_alert_worker.py b/app/core/compare_alert_worker.py index 7462c43..60a1dcb 100644 --- a/app/core/compare_alert_worker.py +++ b/app/core/compare_alert_worker.py @@ -102,10 +102,12 @@ def build_hits( unrecognized_keywords: tuple[str, ...], biz_exclude_keywords: tuple[str, ...], ) -> list[AlertHit]: - """编排:cancelled 走 trace 优先(读到确认没卡则不报,读不到回退保底);failed 附卡点。 + """编排:cancelled 先读 trace,判出原地卡点就带卡点报 T5,否则(可读没卡点/读不到)一律回退 + 耗时/步数兜底——超长放弃照报(卡点列留空);failed 命中后附末段卡点。 - trace 读取限量 max_trace_reads 次/轮;任何 trace 异常都在 trace_stuck 内部降级为 - 「读不到」,cancelled 因而回退保底、failed 不附卡点,绝不影响报警发送。 + trace 读取限量 max_trace_reads 次/轮;任何 trace 异常都在 trace_stuck 内部降级为「读不到」, + cancelled 因而走耗时兜底、failed 不附卡点,绝不影响报警发送。trace 只做「锦上添花」标注卡点, + 绝不因「可读但没判出卡点」把超长放弃吞掉(线上 trace 几乎总可读,否则 total_ms 阈值形同虚设)。 """ base = Path(work_log_dir) if work_log_dir else None reads = 0 @@ -120,13 +122,13 @@ def build_hits( td, threshold=stuck_threshold, max_tail=max_tail ) reads += 1 - if res is not None and res.readable: - if res.points: - stuck = "、".join(_fmt_stuck(sp) for sp in res.points) - hit = _dc_replace(make_hit(rec, "T5", "深度放弃"), stuck_point=stuck) - else: - hit = None # 读到且确认没卡 → 不报 + if res is not None and res.readable and res.points: + stuck = "、".join(_fmt_stuck(sp) for sp in res.points) + hit = _dc_replace(make_hit(rec, "T5", "深度放弃"), stuck_point=stuck) else: + # trace 判出卡点 → 上面带卡点报。其余一律回退耗时/步数兜底:可读但没判出卡点、 + # 读不到、无 trace,都过 total_ms/step 阈值——超长放弃照报(卡点列留空),不再因 + # 「trace 可读但不原地卡」把超长放弃整条吞掉(线上 trace 几乎总可读,否则耗时阈值形同虚设)。 hit = classify_cancelled_fallback( rec, cancelled_ms_threshold=cancelled_ms_threshold, diff --git a/docs/superpowers/specs/2026-08-05-compare-stuck-detection-design.md b/docs/superpowers/specs/2026-08-05-compare-stuck-detection-design.md index 04ccf26..4d17282 100644 --- a/docs/superpowers/specs/2026-08-05-compare-stuck-detection-design.md +++ b/docs/superpowers/specs/2026-08-05-compare-stuck-detection-design.md @@ -97,14 +97,19 @@ def last_step(trace_dir: Path) -> StuckPoint | None: ``` worker 对 cancelled 候选: res = read_stuck_points(dir) - if not res.readable: # 读不到 trace（目录被清/生产一时读不到）→ 回退保底 - >90s或>30步 → T5「深度放弃·等待Xs/Y步」; 否则不报 - elif res.points: # 读到且有卡死平台 → 报卡死 - 报 T5, reason = "卡在 " + "、".join(f"{平台}·{环节}" for res.points) - else: # 读到且没卡死（末段在推进 = 正常深度使用后退出）→ 不报 - 不报 + if res.readable and res.points: # 读到且有卡死平台 → 报卡死（带卡点环节） + 报 T5, reason="深度放弃", stuck_point = "、".join(f"{平台}·{环节}" for res.points) + else: # 其余一律回退耗时/步数兜底(见下方 2026-08-06 修订) + >90s或>30步 → T5「深度放弃」(卡点列留空); 否则不报 ``` +> **2026-08-06 修订(compare-fail-alert 排查)**:原设计「读到且没卡死 → 不报」在线上是死路—— +> **线上 trace 几乎总可读**(WORK_LOG_DIR 已配、同机直读),于是耗时兜底那条分支基本永不触发, +> `total_ms>90s` 阈值形同虚设,**超长放弃(实测 113s / 516s)一条都报不出来**。改为:只有「判出卡点」 +> 独占带卡点的 T5;**其余(可读没卡点 / 读不到 / 无 trace)一律回退耗时兜底**,超长照报(卡点留空)。 +> 权衡:这会重新引入第 1 节「误报」——用户正常浏览 90s+ 后退出也会报。若噪音大,调高 +> `COMPARE_ALERT_CANCELLED_MS_THRESHOLD`(如 180s/300s)收敛,不动代码。 + ### 6.2 failed（T1/T2/T6，判定不变 + 附卡点） ``` diff --git a/tests/test_compare_alert_stuck_worker.py b/tests/test_compare_alert_stuck_worker.py index fe9cb61..de6a3d8 100644 --- a/tests/test_compare_alert_stuck_worker.py +++ b/tests/test_compare_alert_stuck_worker.py @@ -48,8 +48,9 @@ def test_cancelled_stuck_reports_via_trace(tmp_path): assert "帧" in hits[0].stuck_point -def test_cancelled_readable_not_stuck_no_report(tmp_path): - # trace 确认没卡(在推进);即便 total_ms/step 超阈值也不报(信 trace,不回退保底) +def test_cancelled_readable_not_stuck_long_duration_reports(tmp_path): + # trace 可读但没判出原地卡点:仍过耗时/步数阈值兜底,超长(>90s)照报 T5,卡点列留空。 + # (线上 trace 几乎总可读,若不回退则 total_ms 阈值形同虚设、超长放弃永不报——见 compare-fail-alert 排查。) p = tmp_path / "20260804_y" / "eleme" _frame(p, 0, "set_address", "home") for i in range(1, 6): @@ -57,6 +58,21 @@ def test_cancelled_readable_not_stuck_no_report(tmp_path): rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_y/", total_ms=95000, step_count=40) hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW) + assert len(hits) == 1 + assert hits[0].alert_type == "T5" + assert hits[0].reason == "深度放弃" + assert hits[0].stuck_point is None # 没卡点 → 卡片卡点列显 "-" + + +def test_cancelled_readable_not_stuck_short_no_report(tmp_path): + # trace 可读没卡点、且耗时/步数都没超阈值 → 正常早退,不报(兜底阈值把住,不误报)。 + p = tmp_path / "20260804_ys" / "eleme" + _frame(p, 0, "set_address", "home") + for i in range(1, 6): + _frame(p, i, "enter_store", "store") + rec = _Rec(status="cancelled", trace_url="https://x/traces/20260804_ys/", + total_ms=5000, step_count=3) + hits = build_hits([rec], work_log_dir=str(tmp_path), **_KW) assert hits == []