Compare commits

..

17 Commits

Author SHA1 Message Date
guke 00404c8b45 feat(observe): 生产部署(单机)—— 硬化 compose + nginx 反代 + README(SSH隧道/保留期/专用账号)
- docker-compose.prod.yml: 只绑 127.0.0.1、命名卷、mem 1g、密码走 .env(:? 守卫)
- nginx/observe.shaguabijia.com.conf: 子域名反代 + IP白名单 + TLS + WS 透传
- .gitignore: 忽略 prod 密码 .env
- README: 生产部署步骤 + UI 访问两方案(SSH隧道推荐)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 14:38:26 +08:00
guke 964032d16f feat(observe): 现成 OpenObserve 仪表盘(QPS/P95/分位/错误率) + 导入说明
dashboard-api-metrics.json: v8 schema 4 面板,已在 v0.91.1 实例 test-import 验证可用。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 13:39:39 +08:00
guke 1f97b19d21 observe: 关停时补记残留丢弃计数(终审 minor:账不丢在关停期)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 12:53:34 +08:00
guke 8938b951fb feat(observe): OpenObserve 本地 compose + README(查询/仪表盘) + .env.example
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 12:17:28 +08:00
guke 0cf4cc706b feat(observe): main.py 挂中间件 + lifespan 启停上报 worker
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 12:14:14 +08:00
guke a5f63cb53c harden(observe): 防重复 start 泄漏 client + 关停丢余量注释 + 补 stop/失败存活测试
评审建议(Task4 code-quality):
- start 已启动则不重复建 client(避免泄漏旧连接池)
- 关停只发一批、超出丢弃,补注释说明 best-effort
- 补测: _run_loop 失败后存活 / stop drain+发最后一批+关 client

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 12:11:16 +08:00
guke 65e9b422fe fix(observe): 保留 asyncio.TimeoutError 捕获(3.10 兼容,撤销 UP041 自动改写)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 12:04:25 +08:00
guke 0350a140fe feat(observe): 加后台批量上报 worker(失败丢批不重试)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 12:03:06 +08:00
guke 95a4c5c3cc feat(observe): 加 RequestMetricsMiddleware(路由模板+状态码+耗时) 2026-07-06 11:33:13 +08:00
guke 822c2ca2a6 docs(plan): 去注解引号 + Task7 lint 只查改动文件(基线558既有ruff错)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 11:16:00 +08:00
guke 9354588ba2 style(observe): 去冗余注解引号 + 整理 import(ruff UP037/I001)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 11:14:52 +08:00
guke 6c624c3cea feat(observe): 加有界事件队列与 record_event(满则丢) 2026-07-06 11:04:00 +08:00
guke 70c5c4cf08 docs(plan): 拆分 observe.py 导入,避免 Task 2 提交时 ruff F401
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 10:58:28 +08:00
guke 804de02188 test(observe): 补 observe_configured 缺用户名分支断言
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 10:56:38 +08:00
guke 76cb981d84 feat(observe): 加 OBSERVE_* 配置与 observe_configured 门槛
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-07-06 10:48:27 +08:00
guke 20fa32e884 docs: OpenObserve 接口指标实现计划(分步 TDD)
7 个任务:配置门槛 → 事件队列/record_event → 中间件 → 上报 worker →
main 接线 → OpenObserve compose/README/.env → 端到端验证。每任务 TDD + 提交。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 10:40:24 +08:00
guke 29aa6fbff8 docs: OpenObserve 接口 QPS/耗时可观测设计 spec
方案 A:轻量自研 ASGI 中间件 + 后台 worker 批量直采到本地 Docker OpenObserve。
仅 app-server(8770);默认关、opt-in;队列满丢弃、上报失败不重试、跳过 /health。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-06 09:50:18 +08:00
66 changed files with 2424 additions and 1902 deletions
+16
View File
@@ -137,3 +137,19 @@ PANGLE_REPORT_SECURITY_KEY=
# GroMore AppId(报表 site_id 维度)→ 应用环境;默认取现网两个应用,按需覆盖。
PANGLE_REPORT_SITE_ID_PROD=5830519
PANGLE_REPORT_SITE_ID_TEST=5832303
# ===== 可观测(OpenObserve 接口指标)=====
# 采集每个接口 QPS + 耗时 + 错误率,批量直采到 OpenObserve(本地 Docker,见 deploy/openobserve/)。
# 默认关;开启需 ENABLED=true 且填 USER/PASSWORD(与 docker-compose 里 root 账号一致)。
# 未开/缺凭证 → 中间件透传、worker 不启动,整套 no-op,不影响业务。
OBSERVE_ENABLED=false
OBSERVE_ENDPOINT=http://localhost:5080
OBSERVE_ORG=default
OBSERVE_STREAM=app_requests
OBSERVE_USER=admin@shaguabijia.local
OBSERVE_PASSWORD=Complexpass#123
# 进阶(一般不用改):攒批间隔秒 / 单批最大条数 / 有界队列上限(满则丢) / 上报超时秒
OBSERVE_FLUSH_INTERVAL_SEC=5
OBSERVE_BATCH_MAX=200
OBSERVE_QUEUE_MAX=10000
OBSERVE_TIMEOUT_SEC=5
-69
View File
@@ -1,69 +0,0 @@
"""admin_role 表(RBAC 角色 → 可见页面)+ 种子内建角色
新增后台 RBAC:角色持有一组页面 key,登录后左侧只展示这些页。super_admin 内建全权(pages 存空、
effective_pages 特判为全部)、不可编辑删除;operator/finance 播默认页集,可由 super_admin 增删改。
admin_user.role 弱引用本表 name。全新环境顺序应用即得三条内建角色。
Revision ID: admin_role_table
Revises: comparison_product_names
Create Date: 2026-07-04 00:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
revision: str = "admin_role_table"
down_revision: str | Sequence[str] | None = "comparison_product_names"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_JSON = sa.JSON().with_variant(postgresql.JSONB(), "postgresql")
# 与 app/admin/permissions.py 的 BUILTIN_ROLES 同口径(迁移内联一份,不耦合 app 常量)。
# 页集对齐 Prototypes/dashboard/permissions.md 的 ROLES。key 承重(require_role),label 为展示名。
_BUILTIN = [
{"name": "super_admin", "label": "管理员", "pages": [], "is_builtin": True},
{"name": "operator", "label": "运营", "pages": [
"dashboard", "coupon-data", "ad-revenue-report", "comparison-records",
"cps", "device-liveness", "price-reports", "feedbacks",
], "is_builtin": False},
{"name": "finance", "label": "财务", "pages": [
"dashboard", "ad-revenue-report", "cps", "withdraws",
], "is_builtin": False},
{"name": "tech", "label": "技术", "pages": [
"dashboard", "device-liveness", "config", "ad-revenue", "event-logs", "audit-logs",
], "is_builtin": False},
]
def upgrade() -> None:
op.create_table(
"admin_role",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("name", sa.String(length=32), nullable=False),
sa.Column("label", sa.String(length=32), nullable=False, server_default=""),
sa.Column("pages", _JSON, nullable=False),
sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.func.now(), nullable=False,
),
)
op.create_index("ix_admin_role_name", "admin_role", ["name"], unique=True)
tbl = sa.table(
"admin_role",
sa.column("name", sa.String),
sa.column("label", sa.String),
sa.column("pages", _JSON),
sa.column("is_builtin", sa.Boolean),
)
op.bulk_insert(tbl, _BUILTIN)
def downgrade() -> None:
op.drop_index("ix_admin_role_name", table_name="admin_role")
op.drop_table("admin_role")
@@ -1,29 +0,0 @@
"""admin_user 加 plain_password 列(明文登录密码,仅后台 UI 建/重置的账号留存)
权限管理页需能复看某成员已确定的登录密码转交本人。系统只存哈希无法反推,故对「后台 UI 创建/重置」
的管理员额外留存一份明文;脚本/起后台建的超管为 None(前端不显示密码)。旧账号无此列值 → 同样不显示。
Revision ID: admin_user_plain_password
Revises: admin_role_table
Create Date: 2026-07-04 00:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "admin_user_plain_password"
down_revision: str | Sequence[str] | None = "admin_role_table"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
with op.batch_alter_table("admin_user", schema=None) as batch_op:
batch_op.add_column(sa.Column("plain_password", sa.String(length=128), nullable=True))
def downgrade() -> None:
with op.batch_alter_table("admin_user", schema=None) as batch_op:
batch_op.drop_column("plain_password")
@@ -1,70 +0,0 @@
"""comparison_record 加 product_names 列(下单商品名派生串)+ 回填历史行
admin 比价记录页要把原「店/商品」一列拆成「店」+「商品」两列、并支持按商品名搜索。
items 是 JSON(SQLite 下 json.dumps ensure_ascii 会把中文转义存成 \\uXXXX,无法直接 CAST+LIKE
命中中文),故把商品名派生成普通文本列,跨库 LIKE 一致、可索引。
写路径(upsert_record / harvest_done)已同步派生;本迁移建列并从已存 items 回填历史行。
全新环境顺序应用即得空列 + 回填(表本为空,回填 no-op)。
Revision ID: comparison_product_names
Revises: comparison_record_trace_unique
Create Date: 2026-07-04 00:00:00.000000
"""
import json
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "comparison_product_names"
down_revision: str | Sequence[str] | None = "comparison_record_trace_unique"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _product_names(items) -> str | None:
"""items([{name,...}]) → 顿号分隔的商品名串(去重保序)。与 repositories.comparison
._product_names_from_items 同口径,迁移内联一份避免耦合 app 代码。"""
if isinstance(items, str): # SQLite: items 以 JSON 文本存,取回是 str
try:
items = json.loads(items)
except (ValueError, TypeError):
return None
if not isinstance(items, list) or not items:
return None
names: list[str] = []
for it in items:
name = it.get("name") if isinstance(it, dict) else None
if not name:
continue
s = str(name).strip()
if s and s not in names:
names.append(s)
joined = "".join(names)
return joined[:500] or None
def upgrade() -> None:
with op.batch_alter_table("comparison_record", schema=None) as batch_op:
batch_op.add_column(sa.Column("product_names", sa.String(length=512), nullable=True))
# 回填:从已存 items 派生商品名。items 小(菜名列表),一次取回即可;只更新有商品名的行。
bind = op.get_bind()
rows = bind.execute(
sa.text("SELECT id, items FROM comparison_record")
).fetchall()
for rid, items in rows:
pn = _product_names(items)
if pn:
bind.execute(
sa.text("UPDATE comparison_record SET product_names = :pn WHERE id = :id"),
{"pn": pn, "id": rid},
)
def downgrade() -> None:
with op.batch_alter_table("comparison_record", schema=None) as batch_op:
batch_op.drop_column("product_names")
@@ -1,49 +0,0 @@
"""comparison_record: 唯一键 (user_id,trace_id) → trace_id 单列 + user_id 可空
Revision ID: comparison_record_trace_unique
Revises: feedback_type_reply
Create Date: 2026-07-03 12:00:00.000000
比价记录改「后端 harvest」:app-server 在帧0(pricebot 出 trace_id)即建行,随 done/finalize
逐步补全,客户端不再 POST 记录。故本表要两处调整:
1. user_id 改**可空**——harvest 建行那刻(软鉴权 / 老客户端匿名)可能还没有 user_id。
2. 唯一键 (user_id, trace_id) → **trace_id 单列**——trace_id 由 app-server 签发、全局唯一,
一次比价一行;harvest 建行时 user_id 还没有,不能再用复合键去重。
⚠️ 上线前(QA)务必确认 comparison_record 无重复 trace_id:旧复合唯一键**允许**同 trace_id
跨不同 user_id(实际上客户端 UUID 从不重复,但约束没拦),若真有重复,建 trace 单列唯一会失败。
查: SELECT trace_id, COUNT(*) c FROM comparison_record GROUP BY trace_id HAVING c > 1;
SQLite 不支持直接 drop/alter 约束/列,用 batch_alter_table(建临时表+拷数据+换名),
与 coupon_engage_per_package / store_mapping_* 同款;Postgres 直接执行原生 ALTER。
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'comparison_record_trace_unique'
down_revision: Union[str, Sequence[str], None] = 'feedback_type_reply'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
with op.batch_alter_table('comparison_record', schema=None) as batch_op:
# harvest 帧0 建行时 user_id 可能暂缺 → 放开非空。
batch_op.alter_column('user_id', existing_type=sa.Integer(), nullable=True)
# 复合唯一 → trace_id 单列唯一。
batch_op.drop_constraint('uq_comparison_user_trace', type_='unique')
batch_op.create_unique_constraint('uq_comparison_trace', ['trace_id'])
def downgrade() -> None:
with op.batch_alter_table('comparison_record', schema=None) as batch_op:
batch_op.drop_constraint('uq_comparison_trace', type_='unique')
batch_op.create_unique_constraint(
'uq_comparison_user_trace', ['user_id', 'trace_id']
)
# 回退非空前提是当时无 null user_id 行(harvest 期可能有孤儿行,回滚需先清理)。
batch_op.alter_column('user_id', existing_type=sa.Integer(), nullable=False)
-2
View File
@@ -32,7 +32,6 @@ from app.admin.routers.feedback_qr import router as feedback_qr_router
from app.admin.routers.onboarding import router as onboarding_router
from app.admin.routers.ops_marquee_seed import router as ops_marquee_seed_router
from app.admin.routers.price_report import router as price_report_router
from app.admin.routers.roles import router as roles_router
from app.admin.routers.users import router as users_router
from app.admin.routers.wallet import router as wallet_router
from app.admin.routers.withdraw import router as withdraw_router
@@ -99,7 +98,6 @@ admin_app.include_router(feedback_router)
admin_app.include_router(event_logs_router)
admin_app.include_router(feedback_qr_router)
admin_app.include_router(admins_router)
admin_app.include_router(roles_router)
admin_app.include_router(audit_router)
admin_app.include_router(config_router)
admin_app.include_router(comparison_router)
-80
View File
@@ -1,80 +0,0 @@
"""admin 后台「页面权限目录」—— RBAC 的权限侧单一真源。
一个权限 = 一个页面(= 左侧导航项)。角色持有一组 page key,登录后左侧只展示这些页(见前端
layout.tsx 按 pages 过滤导航)。key 必须与前端路由一级对齐(/dashboard → "dashboard")。
super_admin 为内建全权角色,恒可见全部页(effective_pages 特判)。新增页面 = 这里加一条 +
前端导航加对应项(key 一致即可被各角色勾选可见)。
"""
from __future__ import annotations
SUPER_ADMIN_ROLE = "super_admin"
# 分组镜像前端导航(app/(main)/layout.tsx 的 NAV_GROUPS);key = 路由一级
PERMISSION_CATALOG: list[dict] = [
{"group": "看板", "pages": [
{"key": "dashboard", "label": "数据大盘"},
{"key": "coupon-data", "label": "领券数据"},
{"key": "ad-revenue-report", "label": "广告收益"},
{"key": "comparison-records", "label": "比价记录"},
{"key": "cps", "label": "CPS收益"},
{"key": "device-liveness", "label": "设备存活"},
]},
{"group": "奖励审核", "pages": [
{"key": "withdraws", "label": "提现审核"},
{"key": "price-reports", "label": "低价审核"},
{"key": "feedbacks", "label": "用户反馈"},
]},
{"group": "数据配置", "pages": [
{"key": "config", "label": "系统配置"},
{"key": "ad-revenue", "label": "广告配置"},
{"key": "users", "label": "用户管理"},
]},
{"group": "其他", "pages": [
{"key": "admins", "label": "权限管理"},
{"key": "event-logs", "label": "埋点日志"},
{"key": "audit-logs", "label": "审计日志"},
]},
]
# 全部页面 key(super_admin 有效可见 = 此全集;也用于校验角色 pages 合法性)
ALL_PAGE_KEYS: list[str] = [p["key"] for g in PERMISSION_CATALOG for p in g["pages"]]
_ALL_SET = set(ALL_PAGE_KEYS)
# 内建角色定义(种子 / ensure 兜底的单一真源):(key, 中文展示名, 默认可见页)。
# 页集对齐 Prototypes/dashboard/permissions.md 的 ROLES;super_admin 恒全权(pages 空、bypass)。
# key 承重(require_role / admin_user.role),勿改;label 为 UI 展示名。
BUILTIN_ROLES: list[dict] = [
{"name": SUPER_ADMIN_ROLE, "label": "管理员", "pages": []},
{"name": "operator", "label": "运营", "pages": [
"dashboard", "coupon-data", "ad-revenue-report", "comparison-records",
"cps", "device-liveness", "price-reports", "feedbacks",
]},
{"name": "finance", "label": "财务", "pages": [
"dashboard", "ad-revenue-report", "cps", "withdraws",
]},
{"name": "tech", "label": "技术", "pages": [
"dashboard", "device-liveness", "config", "ad-revenue", "event-logs", "audit-logs",
]},
]
# 内建 key → 中文展示名(展示 / 自愈用)
BUILTIN_LABELS: dict[str, str] = {r["name"]: r["label"] for r in BUILTIN_ROLES}
def sanitize_pages(pages: list[str] | None) -> list[str]:
"""过滤掉不在目录里的 key(去重、保序),防脏数据/目录收缩后的悬空 key。"""
seen: set[str] = set()
out: list[str] = []
for k in pages or []:
if k in _ALL_SET and k not in seen:
seen.add(k)
out.append(k)
return out
def effective_pages(role_name: str, role_pages: list[str] | None) -> list[str]:
"""某角色的有效可见页:super_admin → 全部;其余 → 其 pages 与目录取交(自愈悬空 key)。"""
if role_name == SUPER_ADMIN_ROLE:
return list(ALL_PAGE_KEYS)
return sanitize_pages(role_pages)
-81
View File
@@ -1,81 +0,0 @@
"""admin_role 表 CRUD + 「角色 → 有效可见页」解析。"""
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.admin.permissions import (
ALL_PAGE_KEYS,
BUILTIN_ROLES,
SUPER_ADMIN_ROLE,
sanitize_pages,
)
from app.models.admin_role import AdminRole
def ensure_builtin_roles(db: Session) -> None:
"""空表时播种内建角色(管理员/运营/财务/技术)。幂等:表非空即 no-op。
给 create_all 环境(测试 / 未跑迁移的全新库)兜底,迁移已播种则不重复。
仅 super_admin 为 is_builtin(锁定不可改删);运营/财务/技术可编辑页/删除。"""
if db.execute(select(AdminRole.id).limit(1)).first() is not None:
return
db.add_all([
AdminRole(
name=r["name"], label=r["label"], pages=list(r["pages"]),
is_builtin=(r["name"] == SUPER_ADMIN_ROLE),
)
for r in BUILTIN_ROLES
])
db.commit()
def list_roles(db: Session) -> list[AdminRole]:
ensure_builtin_roles(db)
# super_admin(内建)恒排最前,其余按创建序
return list(
db.execute(
select(AdminRole).order_by(AdminRole.is_builtin.desc(), AdminRole.id)
).scalars().all()
)
def get_role(db: Session, name: str) -> AdminRole | None:
return db.execute(
select(AdminRole).where(AdminRole.name == name)
).scalar_one_or_none()
def create_role(db: Session, *, name: str, label: str, pages: list[str]) -> AdminRole:
# 自定义角色:name(key)= label = 用户输入的名称
role = AdminRole(name=name, label=label, pages=sanitize_pages(pages), is_builtin=False)
db.add(role)
db.commit()
db.refresh(role)
return role
def update_role(
db: Session, role: AdminRole, *, label: str | None = None, pages: list[str] | None = None
) -> AdminRole:
# 只改展示名 label + 可见页;name(key)不可变(admin_user.role / require_role 承重)
if label is not None:
role.label = label
if pages is not None:
role.pages = sanitize_pages(pages)
db.commit()
db.refresh(role)
return role
def delete_role(db: Session, role: AdminRole) -> None:
db.delete(role)
db.commit()
def effective_pages_of(db: Session, role_name: str) -> list[str]:
"""某角色名的有效可见页:super_admin → 全部;其余 → 查表 pages 与目录取交。"""
if role_name == SUPER_ADMIN_ROLE:
return list(ALL_PAGE_KEYS)
ensure_builtin_roles(db)
role = get_role(db, role_name)
return sanitize_pages(role.pages if role else None)
+1 -9
View File
@@ -20,20 +20,12 @@ def get_by_username(db: Session, username: str) -> AdminUser | None:
def create_admin(
db: Session,
*,
username: str,
password: str,
role: str = "operator",
plain_password: str | None = None,
db: Session, *, username: str, password: str, role: str = "operator"
) -> AdminUser:
"""建管理员。plain_password 非空则额外留存明文(后台 UI 建的账号传,供权限管理页复看);
脚本/起后台建账号不传(留 None → 前端不显示密码)。"""
admin = AdminUser(
username=username,
password_hash=hash_password(password),
role=role,
plain_password=plain_password,
)
db.add(admin)
db.commit()
+1 -9
View File
@@ -148,14 +148,11 @@ def list_comparison_records(
phone: str | None = None,
status: str | None = None,
business_type: str | None = None,
store: str | None = None,
product: str | None = None,
limit: int = 20,
cursor: int | None = None,
) -> tuple[list[ComparisonRecord], int | None, int]:
"""admin 比价记录列表(debug)。按 user_id 精确 或 phone 前缀定位用户 + status/业务类型筛,
store(店名)/product(商品名)子串模糊匹配,offset 分页(创建时间倒序、id 兜底)。
join User 取 phone/nickname 瞬态挂记录上。"""
offset 分页(创建时间倒序、id 兜底)。join User 取 phone/nickname 瞬态挂记录上。"""
stmt = select(ComparisonRecord)
if user_id is not None:
stmt = stmt.where(ComparisonRecord.user_id == user_id)
@@ -169,11 +166,6 @@ def list_comparison_records(
stmt = stmt.where(ComparisonRecord.status == status)
if business_type:
stmt = stmt.where(ComparisonRecord.business_type == business_type)
if store:
stmt = stmt.where(ComparisonRecord.store_name.like(f"%{store}%"))
if product:
# 商品名搜 product_names 派生文本列(非 items JSON:SQLite 下 JSON 中文被转义无法直接 LIKE)。
stmt = stmt.where(ComparisonRecord.product_names.like(f"%{product}%"))
items, next_cursor, total = offset_paginate(
db, stmt,
(desc(ComparisonRecord.created_at), desc(ComparisonRecord.id)),
+3 -49
View File
@@ -5,8 +5,6 @@ from fastapi import APIRouter, Depends, HTTPException, Request
from app.admin.audit import write_audit
from app.admin.deps import AdminDb, CurrentAdmin, get_client_ip, require_role
from app.admin.permissions import SUPER_ADMIN_ROLE
from app.admin.repositories import admin_role as role_repo
from app.admin.repositories import admin_user as admin_repo
from app.admin.schemas.admin import AdminCreateRequest, AdminUpdateRequest
from app.admin.schemas.auth import AdminOut
@@ -25,23 +23,9 @@ def _active_super_count(db: AdminDb) -> int:
)
def _validate_role(db: AdminDb, role: str) -> None:
"""角色必须是 super_admin 或 admin_role 表里已存在的角色,否则 400。"""
if role == SUPER_ADMIN_ROLE:
return
role_repo.ensure_builtin_roles(db) # 空表(测试/全新库)兜底播种,再校验
if role_repo.get_role(db, role) is None:
raise HTTPException(status_code=400, detail=f"角色不存在: {role}")
@router.get("", response_model=list[AdminOut], summary="管理员列表(含明文密码,super_admin 专属)")
@router.get("", response_model=list[AdminOut], summary="管理员列表")
def list_admins(db: AdminDb) -> list[AdminOut]:
out: list[AdminOut] = []
for a in admin_repo.list_admins(db):
item = AdminOut.model_validate(a)
item.password = a.plain_password # 明文:仅本 super_admin 专属路由下发,供权限管理页复看
out.append(item)
return out
return [AdminOut.model_validate(a) for a in admin_repo.list_admins(db)]
@router.post("", response_model=AdminOut, summary="创建管理员")
@@ -50,10 +34,8 @@ def create_admin(
) -> AdminOut:
if admin_repo.get_by_username(db, body.username) is not None:
raise HTTPException(status_code=409, detail="用户名已存在")
_validate_role(db, body.role)
new = admin_repo.create_admin(
db, username=body.username, password=body.password, role=body.role,
plain_password=body.password, # UI 建的账号留存明文,供权限管理页复看
db, username=body.username, password=body.password, role=body.role
)
write_audit(
db, admin, action="admin.create", target_type="admin", target_id=new.id,
@@ -85,9 +67,6 @@ def update_admin(
if demotes_super and _active_super_count(db) <= 1:
raise HTTPException(status_code=400, detail="不能降级/禁用最后一个超级管理员")
if body.role is not None:
_validate_role(db, body.role)
changes: dict = {}
if body.role is not None and body.role != target.role:
changes["role"] = {"before": target.role, "after": body.role}
@@ -98,7 +77,6 @@ def update_admin(
if body.password is not None:
changes["password"] = "reset"
target.password_hash = hash_password(body.password)
target.plain_password = body.password # 同步留存明文,权限管理页复看保持一致
if not changes:
raise HTTPException(status_code=400, detail="无任何变更字段")
db.commit()
@@ -108,27 +86,3 @@ def update_admin(
detail=changes, ip=get_client_ip(request), commit=True,
)
return AdminOut.model_validate(target)
@router.delete("/{admin_id}", summary="删除管理员(带审计)")
def delete_admin(admin_id: int, request: Request, admin: CurrentAdmin, db: AdminDb) -> dict:
target = admin_repo.get_by_id(db, admin_id)
if target is None:
raise HTTPException(status_code=404, detail="管理员不存在")
if admin_id == admin.id:
raise HTTPException(status_code=400, detail="不能删除自己")
# 防自锁:删掉某个 active super_admin 前,确认后仍至少剩 1 个,否则进「零可用超管」死局。
if (
target.role == "super_admin"
and target.status == "active"
and _active_super_count(db) <= 1
):
raise HTTPException(status_code=400, detail="不能删除最后一个超级管理员")
username = target.username
db.delete(target)
db.commit()
write_audit(
db, admin, action="admin.delete", target_type="admin", target_id=admin_id,
detail={"username": username, "role": target.role}, ip=get_client_ip(request), commit=True,
)
return {"deleted": True}
+4 -12
View File
@@ -6,7 +6,6 @@ import logging
from fastapi import APIRouter, Depends, HTTPException
from app.admin.deps import AdminDb, CurrentAdmin
from app.admin.repositories import admin_role as role_repo
from app.admin.repositories import admin_user as admin_repo
from app.admin.schemas.auth import AdminLoginRequest, AdminLoginResponse, AdminOut
from app.admin.security import create_admin_token
@@ -18,13 +17,6 @@ logger = logging.getLogger("shagua.admin.auth")
router = APIRouter(prefix="/admin/api/auth", tags=["admin-auth"])
def _admin_out_with_pages(admin, db: AdminDb) -> AdminOut: # noqa: ANN001
"""AdminOut + 当前角色有效可见页(前端左侧导航按此过滤)。"""
out = AdminOut.model_validate(admin)
out.pages = role_repo.effective_pages_of(db, admin.role)
return out
@router.post(
"/login",
response_model=AdminLoginResponse,
@@ -47,10 +39,10 @@ def login(req: AdminLoginRequest, db: AdminDb) -> AdminLoginResponse:
return AdminLoginResponse(
access_token=token,
expires_in=expires_in,
admin=_admin_out_with_pages(admin, db),
admin=AdminOut.model_validate(admin),
)
@router.get("/me", response_model=AdminOut, summary="当前管理员(含有效可见页)")
def me(admin: CurrentAdmin, db: AdminDb) -> AdminOut:
return _admin_out_with_pages(admin, db)
@router.get("/me", response_model=AdminOut, summary="当前管理员")
def me(admin: CurrentAdmin) -> AdminOut:
return AdminOut.model_validate(admin)
+1 -4
View File
@@ -32,15 +32,12 @@ def list_comparison_records(
phone: Annotated[str | None, Query(description="手机号前缀")] = None,
status: Annotated[str | None, Query(pattern="^(success|failed|cancelled)$")] = None,
business_type: Annotated[str | None, Query()] = None,
store: Annotated[str | None, Query(description="店名子串模糊匹配")] = None,
product: Annotated[str | None, Query(description="商品名子串模糊匹配")] = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
cursor: Annotated[int | None, Query()] = None,
) -> CursorPage[AdminComparisonListItem]:
items, next_cursor, total = queries.list_comparison_records(
db, user_id=user_id, phone=phone, status=status,
business_type=business_type, store=store, product=product,
limit=limit, cursor=cursor,
business_type=business_type, limit=limit, cursor=cursor,
)
return CursorPage(
items=[AdminComparisonListItem.model_validate(r) for r in items],
+2 -7
View File
@@ -55,14 +55,9 @@ def _item(db, key: str) -> ConfigItemOut:
raise HTTPException(status_code=404, detail="未知配置项")
@router.get("", response_model=list[ConfigItemOut], summary="所有可配项 + 当前值(不含 hidden)")
@router.get("", response_model=list[ConfigItemOut], summary="所有可配项 + 当前值")
def list_config(db: AdminDb) -> list[ConfigItemOut]:
# hidden 项(已下线/由专用页管理,如福利页任务·里程碑·看广告调参、首页轮播数据源)不在本页渲染。
return [
ConfigItemOut(**item)
for item in app_config.list_all(db)
if not CONFIG_DEFS[item["key"]].get("hidden")
]
return [ConfigItemOut(**item) for item in app_config.list_all(db)]
@router.patch("/{key}", response_model=ConfigItemOut, summary="改某项配置(带审计)")
+1 -33
View File
@@ -8,7 +8,6 @@ from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel
from app.admin.audit import write_audit
from app.admin.deps import AdminDb, get_client_ip, get_current_admin, require_role
@@ -23,11 +22,7 @@ from app.admin.schemas.ops_marquee_seed import (
)
from app.models.admin import AdminUser
from app.models.ops_marquee_seed import OpsMarqueeSeed
from app.repositories import app_config, ops_marquee
class MarqueeModeUpdate(BaseModel):
mode: str # mixed / real / seed
from app.repositories import ops_marquee
router = APIRouter(
prefix="/admin/api/marquee-seeds",
@@ -62,33 +57,6 @@ def preview_feed(
return OpsSavingsFeedPreviewOut(items=ops_marquee.get_feed(db, limit=limit))
# 注:/mode 两个端点须在 /{seed_id} 之前注册,否则 PATCH /mode 会被 /{seed_id} 抢先按 id 解析。
@router.get("/mode", summary="首页轮播数据源模式(mixed/real/seed)")
def get_feed_mode(db: AdminDb) -> dict:
"""当前轮播取数模式:mixed=真实优先+种子补位(默认)/ real=只真实 / seed=只种子·合成。"""
return {"mode": ops_marquee.get_feed_mode(db)}
@router.patch("/mode", summary="改首页轮播数据源模式(带审计)")
def set_feed_mode(
body: MarqueeModeUpdate,
request: Request,
admin: Annotated[AdminUser, Depends(require_role("operator"))],
db: AdminDb,
) -> dict:
if body.mode not in ops_marquee.FEED_MODES:
raise HTTPException(status_code=400, detail="mode 需为 mixed / real / seed")
before = ops_marquee.get_feed_mode(db)
app_config.set_value(db, "marquee_feed_mode", body.mode, admin_id=admin.id, commit=False)
write_audit(
db, admin, action="ops_marquee_seed.set_mode", target_type="config",
target_id="marquee_feed_mode", detail={"before": before, "after": body.mode},
ip=get_client_ip(request), commit=False,
)
db.commit()
return {"mode": body.mode}
@router.post("", response_model=OpsMarqueeSeedOut, summary="新增轮播种子(带审计)")
def create_seed(
body: OpsMarqueeSeedCreate,
-129
View File
@@ -1,129 +0,0 @@
"""admin RBAC 角色管理(仅 super_admin):列角色 / 权限目录 / 增删改角色。均写审计。
角色 = 一组「可见页面」(见 app/admin/permissions.py)。角色登录后台后左侧只展示其 pages 对应导航项。
只有 super_admin(内建全权角色)能进本组端点(dependencies=require_role() 无参 = 仅 super_admin)。
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, Request
from sqlalchemy import func, select
from app.admin.audit import write_audit
from app.admin.deps import AdminDb, CurrentAdmin, get_client_ip, require_role
from app.admin.permissions import (
PERMISSION_CATALOG,
SUPER_ADMIN_ROLE,
effective_pages,
sanitize_pages,
)
from app.admin.repositories import admin_role as role_repo
from app.admin.schemas.role import (
PermissionGroup,
RoleCreateRequest,
RoleOut,
RoleUpdateRequest,
)
from app.models.admin import AdminUser
from app.models.admin_role import AdminRole
router = APIRouter(
prefix="/admin/api/roles",
tags=["admin-roles"],
dependencies=[Depends(require_role())], # 无参 = 仅 super_admin
)
def _usage_counts(db: AdminDb) -> dict[str, int]:
rows = db.execute(
select(AdminUser.role, func.count(AdminUser.id)).group_by(AdminUser.role)
).all()
return {r: c for r, c in rows}
def _to_out(role: AdminRole, usage: dict[str, int]) -> RoleOut:
return RoleOut(
id=role.id,
name=role.name,
label=role.label or role.name,
pages=effective_pages(role.name, role.pages),
is_builtin=role.is_builtin,
in_use=usage.get(role.name, 0),
)
@router.get("", response_model=list[RoleOut], summary="角色列表(含可见页 + 使用数)")
def list_roles(db: AdminDb) -> list[RoleOut]:
usage = _usage_counts(db)
return [_to_out(r, usage) for r in role_repo.list_roles(db)]
@router.get("/catalog", response_model=list[PermissionGroup], summary="页面权限目录(分组)")
def get_catalog() -> list[PermissionGroup]:
return [PermissionGroup(**g) for g in PERMISSION_CATALOG]
@router.post("", response_model=RoleOut, summary="新增角色(带审计)")
def create_role(
body: RoleCreateRequest, request: Request, admin: CurrentAdmin, db: AdminDb
) -> RoleOut:
name = body.name.strip()
if not name:
raise HTTPException(status_code=400, detail="角色名称不能为空")
if name == SUPER_ADMIN_ROLE:
raise HTTPException(status_code=400, detail="super_admin 为内建角色,不能新建")
if role_repo.get_role(db, name) is not None:
raise HTTPException(status_code=409, detail="角色名称已存在")
# 自定义角色:name(key)= label = 输入名称
role = role_repo.create_role(db, name=name, label=name, pages=body.pages)
write_audit(
db, admin, action="role.create", target_type="role", target_id=str(role.id),
detail={"name": name, "pages": role.pages}, ip=get_client_ip(request), commit=True,
)
return _to_out(role, _usage_counts(db))
@router.patch("/{role_id}", response_model=RoleOut, summary="改角色(展示名/可见页,带审计)")
def update_role(
role_id: int, body: RoleUpdateRequest, request: Request, admin: CurrentAdmin, db: AdminDb
) -> RoleOut:
role = db.get(AdminRole, role_id)
if role is None:
raise HTTPException(status_code=404, detail="角色不存在")
if role.is_builtin:
raise HTTPException(status_code=400, detail="内建角色「管理员」不可编辑")
new_label = body.label.strip() if body.label is not None else None
changes: dict = {}
if new_label and new_label != role.label:
changes["label"] = {"before": role.label, "after": new_label}
if body.pages is not None:
changes["pages"] = {"before": role.pages, "after": sanitize_pages(body.pages)}
if not changes:
raise HTTPException(status_code=400, detail="无任何变更字段")
# 只改展示名 + 可见页;key(name)不可变,故无需级联 admin_user.role
role_repo.update_role(db, role, label=new_label, pages=body.pages)
write_audit(
db, admin, action="role.update", target_type="role", target_id=str(role_id),
detail=changes, ip=get_client_ip(request), commit=True,
)
return _to_out(role, _usage_counts(db))
@router.delete("/{role_id}", summary="删角色(带审计;内建/在用不可删)")
def delete_role(role_id: int, request: Request, admin: CurrentAdmin, db: AdminDb) -> dict:
role = db.get(AdminRole, role_id)
if role is None:
raise HTTPException(status_code=404, detail="角色不存在")
if role.is_builtin:
raise HTTPException(status_code=400, detail="内建角色不可删除")
used = _usage_counts(db).get(role.name, 0)
if used > 0:
raise HTTPException(status_code=400, detail=f"该角色仍有 {used} 名管理员在用,请先改派再删")
name = role.name
role_repo.delete_role(db, role)
write_audit(
db, admin, action="role.delete", target_type="role", target_id=str(role_id),
detail={"name": name}, ip=get_client_ip(request), commit=True,
)
return {"deleted": True}
+3 -3
View File
@@ -6,19 +6,19 @@ from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
# 角色不再硬编码枚举:改为任意角色名(自定义角色由 admin_role 表管理),存在性在路由层校验。
_Role = Literal["super_admin", "finance", "operator"]
class AdminCreateRequest(BaseModel):
username: str = Field(..., min_length=3, max_length=64)
password: str = Field(..., min_length=8, max_length=72) # bcrypt ≤72 字节
role: str = Field("operator", min_length=1, max_length=32)
role: _Role = "operator"
class AdminUpdateRequest(BaseModel):
"""改角色 / 启用禁用 / 重置密码,字段都可选(只改传了的)。"""
role: str | None = Field(None, min_length=1, max_length=32)
role: _Role | None = None
status: Literal["active", "disabled"] | None = None
password: str | None = Field(None, min_length=8, max_length=72)
-6
View File
@@ -20,12 +20,6 @@ class AdminOut(BaseModel):
status: str
created_at: datetime
last_login_at: datetime | None = None
# 该管理员当前角色的有效可见页(= 左侧导航项 key);仅登录 / /me 填充,列表接口默认空。
# 前端据此过滤左侧导航(super_admin = 全部页)。见 app/admin/permissions.py。
pages: list[str] = []
# 明文登录密码:仅「管理员账号列表」(super_admin 专属路由)填充,供权限管理页编辑时复看;
# 无留存(脚本建的超管 / 旧账号)为 None → 前端不显示。登录 / /me 不下发(保持 None)。
password: str | None = None
class AdminLoginResponse(BaseModel):
-1
View File
@@ -23,7 +23,6 @@ class AdminComparisonListItem(BaseModel):
status: str
information: str | None = None
store_name: str | None = None
product_names: str | None = None # 下单商品名派生串(顿号分隔;「商品」列展示 + 商品搜索)
source_platform_name: str | None = None
best_platform_name: str | None = None
source_price_cents: int | None = None
-35
View File
@@ -1,35 +0,0 @@
"""admin RBAC 角色 + 权限目录 schemas。"""
from __future__ import annotations
from pydantic import BaseModel, Field
class PermissionPage(BaseModel):
key: str
label: str
class PermissionGroup(BaseModel):
group: str
pages: list[PermissionPage]
class RoleOut(BaseModel):
id: int
name: str # 角色 key(承重,不可变)
label: str # 展示名(UI 显示)
pages: list[str] # 有效可见页 key(super_admin = 全部页)
is_builtin: bool # 内建角色(super_admin):不可编辑/删除
in_use: int # 使用该角色的管理员数(删除前端提示 / 拦截用)
class RoleCreateRequest(BaseModel):
# 新增自定义角色:name 即用户输入的名称(同时作 key 与展示名)
name: str = Field(..., min_length=1, max_length=32)
pages: list[str] = Field(default_factory=list)
class RoleUpdateRequest(BaseModel):
# 只改展示名 + 可见页;key 不可变
label: str | None = Field(None, min_length=1, max_length=32)
pages: list[str] | None = None
-24
View File
@@ -54,29 +54,5 @@ def get_current_user(
return user
def get_current_user_optional(
credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer)],
db: Annotated[Session, Depends(get_db)],
) -> User | None:
"""软鉴权:有合法 Bearer 就返回 user,否则(无 header / 无效 token / 用户禁用)一律返回
None,**不 raise**。
比价 step / finalize 灰度期用:新客户端带 JWT → 拿到 user_id 绑定 harvest 行;
老客户端(不带 JWT)→ None,harvest 行 user_id 暂空,由其后续 /compare/record 上报补齐。
等新版覆盖率够高,再把这几条端点从软鉴权收紧成硬 get_current_user。
"""
if credentials is None or credentials.scheme.lower() != "bearer":
return None
try:
payload = decode_token(credentials.credentials, expected_type="access")
user = db.get(User, int(payload["sub"]))
except (TokenError, KeyError, ValueError, TypeError):
return None
if user is None or user.status != "active":
return None
return user
CurrentUser = Annotated[User, Depends(get_current_user)]
OptionalUser = Annotated[User | None, Depends(get_current_user_optional)]
DbSession = Annotated[Session, Depends(get_db)]
+7 -6
View File
@@ -12,11 +12,11 @@ from __future__ import annotations
import logging
from fastapi import APIRouter, HTTPException, Request, status
from fastapi import APIRouter, Depends, HTTPException, Request, status
from app.api.deps import CurrentUser, DbSession
from app.core import test_account
from app.core.ratelimit import enforce_rate_limit
from app.core.ratelimit import enforce_rate_limit, rate_limit
from app.core.security import TokenError, decode_token, issue_token_pair
from app.integrations.jiguang import JiguangError, mask_phone, verify_and_get_phone
from app.integrations.sms import SmsError, send_code, verify_code
@@ -41,7 +41,7 @@ router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
# 手机号登录防刷:同一设备(device_id) + 同一 IP 每小时最多的登录尝试次数(成功/失败都计)。
SMS_LOGIN_MAX_PER_HOUR = 5
# 发码防刷:同一设备(device_id) + 同一 IP 每小时最多的发码次数。
# 堵「换手机号绕开单号 60s 冷却」的洞 —— 冷却是单号维度,一机换号能绕开。
# 堵「换手机号绕开单号 60s 冷却 / 单号每日上限」的洞 —— 那两道是单号维度,一机换号能绕开。
SMS_SEND_MAX_PER_HOUR_PER_DEVICE = 5
@@ -100,8 +100,8 @@ def sms_send(req: SmsSendRequest, request: Request) -> SmsSendResponse:
return SmsSendResponse(sent=True, mock=True, cooldown_sec=0)
# 防刷:同一设备(device_id) + 同一 IP 每小时最多 SMS_SEND_MAX_PER_HOUR_PER_DEVICE 次发码。
# 补「换手机号绕开单号 60s 冷却」的洞(冷却是单号维度,一机换号能绕);设备维度按机器封顶,
# 挡短信轰炸/烧钱。放在真发(send_code)之前 → 超限直接拦下、不真发短信。
# 补「换手机号绕开单号 60s 冷却 / 单号每日上限」的洞(那两道是单号维度,一机换号能绕);设备维度按机器封顶,
# 挡短信轰炸/烧钱。放在真发(send_code)之前 → 超限直接拦下、不真发短信。与路由上 IP 维度(10次/分钟)互补。
enforce_rate_limit(
request,
scope="sms-send-device",
@@ -125,6 +125,7 @@ def sms_send(req: SmsSendRequest, request: Request) -> SmsSendResponse:
"/sms/login",
response_model=TokenWithUser,
summary="手机号+验证码登录",
dependencies=[Depends(rate_limit(20, 60, "sms-login"))], # 防撞库爆破(另有单码失败次数上限)
)
def sms_login(req: SmsLoginRequest, request: Request, db: DbSession) -> TokenWithUser:
# 测试账号:免验证码登录 + 每日上限 + 每次都走新手引导(详见 app/core/test_account.py)。
@@ -142,7 +143,7 @@ def sms_login(req: SmsLoginRequest, request: Request, db: DbSession) -> TokenWit
return _login_response(user, onboarding_completed=False, force_onboarding=True)
# 防刷:同一设备(device_id) + 同一 IP 每小时最多 SMS_LOGIN_MAX_PER_HOUR 次登录尝试。放在验证码校验
# **之前** → 输错验证码的失败尝试也计数,才挡得住撞库/爆破(另有单码失败 SMS_MAX_VERIFY_ATTEMPTS 次即作废兜底)
# **之前** → 输错验证码的失败尝试也计数,才挡得住撞库/爆破。与路由上 IP 维度的 sms-login 限流(同 IP)互补
# ⚠️ 按设备而非手机号 → 一台机器换不同手机号刷登录也受限(防一机狂登多号);device_id 空(老客户端)时
# 退化为该 IP 下所有空设备聚一桶,仍受限。
enforce_rate_limit(
+60 -203
View File
@@ -1,114 +1,49 @@
"""外卖比价业务透传端点 + 后端 harvest 落库
"""外卖比价业务透传端点。
把客户端 POST /api/v1/intent/* 和 /api/v1/price/step 转发给 pricebot,同时**由 app-server
在透传里直接落库比价记录**(不再靠客户端 POST /compare/record):
- 帧0(客户端首帧不带 trace_id)→ app-server 用 uuid 签发 trace_id、注入转发 body、回给
客户端;并按 trace_id 建 running 行(harvest_running)。
- 最终 done 帧 → 更新成 success/failed + 结果(harvest_done)。发奖不在这里:#113 已把
邀请奖口径从「比价」移到「实际下单」(order.py)。
- /trace/finalize(用户终止/Phase1 未识别,无 done)→ 更新成 cancelled/failed
(harvest_abort,**不降级 success**)。
trace_url 从 pricebot 响应**顶层 trace_url** 取(pricebot 每帧都带,见其 goal_engine.process_step)。
把客户端 POST /api/v1/intent/recognize 和 /api/v1/price/step 请求原样转发给
pricebot-backend 的 /api/intent/recognize 和 /api/price/step。MVP 阶段**不鉴权**
(同 coupon/step,见 docs/待办与技术债.md P1:device_id 透传区分设备,待补 JWT +
device_id↔user_id 绑定后才能做用户级画像)。
鉴权:软鉴权(OptionalUser)——新客户端带 JWT → 绑 user_id;老客户端不带 → user_id 暂空,
由其后续 /compare/record 上报补(灰度期两条写路径按 trace_id reconcile,success 不被降级)
- Phase 1 /intent/recognize:从源平台(淘宝闪购/美团/京东外卖)购物车页识别店名+菜品+
价格,一次性
- Phase 2 /price/step:多轮循环,在目标平台复现订单读到手价,直到 done。
真正的目标驱动比价逻辑在 pricebot-backend(GoalEngine,另一个 repo),本接口是透传壳 + 落库
pricebot 协议文档: pricebot-backend/docs/main/02_api_protocol.md
真正的目标驱动比价逻辑在 pricebot-backend(GoalEngine,另一个 repo),本接口是透传壳。
电商(ecom)那两个端点 food MVP 暂不需要,以后放开 scene 时再加 /ecom/intent/recognize
和 /ecom/step 两行即可。
pricebot 协议文档:
pricebot-backend/docs/main/02_api_protocol.md
"""
from __future__ import annotations
import json
import logging
import time
import uuid
from typing import Any
import httpx
from fastapi import APIRouter, HTTPException, Request, status
from fastapi.concurrency import run_in_threadpool
from app.api.deps import OptionalUser
from app.core.config import settings
from app.core.logging import trace_id_ctx
from app.core.pricebot_client import get_pricebot_client
from app.core.pricebot_router import pick_pricebot
from app.db.session import SessionLocal
from app.repositories import comparison as crud_compare
logger = logging.getLogger("shagua.compare")
router = APIRouter(prefix="/api/v1", tags=["compare"])
# ============================================================
# harvest 落库(阻塞 SQLAlchemy → run_in_threadpool,独立 SessionLocal,不阻塞事件循环;
# 任何写库异常都吞掉、绝不连累比价透传返回 —— 同 coupon.py 现有 best-effort 写法)。
# ============================================================
async def _passthrough(request: Request, upstream_path: str) -> dict[str, Any]:
"""把请求体原样转发给 pricebot-backend 的 {upstream_path}
def _harvest_running_blocking(
trace_id: str, user_id: int | None, business_type: str,
device_id: str | None, device_info: dict | None, trace_url: str | None,
) -> None:
with SessionLocal() as db:
crud_compare.harvest_running(
db, trace_id=trace_id, user_id=user_id, business_type=business_type,
device_id=device_id, device_info=device_info, trace_url=trace_url,
)
logger.info(
"harvest running row (user=%s)", user_id,
extra={"phase": "harvest_running", "status": "running", "user_id": user_id},
)
def _harvest_done_blocking(
trace_id: str, user_id: int | None, done_params: dict, business_type: str,
device_id: str | None, device_info: dict | None, trace_url: str | None,
) -> None:
with SessionLocal() as db:
rec, newly_success = crud_compare.harvest_done(
db, trace_id=trace_id, user_id=user_id, done_params=done_params,
business_type=business_type, device_id=device_id,
device_info=device_info, trace_url=trace_url,
)
logger.info(
"harvest done → %s saved=%s newly=%s", rec.status,
rec.saved_amount_cents, newly_success,
extra={"phase": "harvest_done", "status": rec.status,
"saved_cents": rec.saved_amount_cents,
"best_platform": rec.best_platform_id, "newly_success": newly_success,
"user_id": user_id},
)
# 不在此处发邀请奖:#113 已把发奖口径从「比价」移到「实际下单」(order.py),harvest
# 只记录比价、不发奖。否则比价先于下单 + try_reward 幂等闸会让奖落在「比价」这步,
# 架空 #113 的「下单才发奖」防刷意图(newly_success 仅留作日志观测)。
def _harvest_abort_blocking(
trace_id: str, status_hint: str, reason: str | None, trace_url: str | None,
) -> None:
with SessionLocal() as db:
rec = crud_compare.harvest_abort(
db, trace_id=trace_id, status=status_hint, reason=reason, trace_url=trace_url,
)
logger.info(
"harvest abort → %s", (rec.status if rec else "no-row"),
extra={"phase": "harvest_abort",
"status": (rec.status if rec else None), "reason": reason},
)
async def _forward(
request: Request, upstream_path: str, user, *, harvest_first_frame: bool = True,
) -> tuple[dict[str, Any], str, dict[str, Any]]:
"""透传给 pricebot 的 {upstream_path},并 mint trace_id + 首帧建 running 行。
返回 (resp_json, trace_id, meta)。
- 客户端首帧不带 trace_id → app-server 用 uuid 签发、写进转发 body、回填进响应顶层
`trace_id`(客户端据此拿到后续帧都带上)。带了(老客户端 / 后续帧)→ 原样用、走原始 bytes 快路。
- 仅**本次 mint(=首帧)**建 running 行(idempotent);后续帧不再写库。
跟 coupon_step 同款透传壳:不鉴权、不做 schema 校验,仅读 device_id/trace_id/step
打日志。比价单帧是大上下文 / 逐帧 LLM,超时用 PRICEBOT_COMPARE_TIMEOUT_SEC(60s,
比领券的 30s 长)。
"""
# 读原始字节,避免"反序列化→再序列化"的双重 JSON(省 ~一半透传 CPU,让 app-server
# 单 worker 也扛得住高并发)。只 json.loads 一次拿 trace_id 做亲和 + 打日志,转发时
# 直接发原始 bytes(content=raw),不重新 dumps。
raw = await request.body()
try:
meta = json.loads(raw)
@@ -117,41 +52,26 @@ async def _forward(
if not isinstance(meta, dict):
meta = {}
trace_id = meta.get("trace_id")
minted = False
if not trace_id:
trace_id = str(uuid.uuid4())
meta["trace_id"] = trace_id
raw = json.dumps(meta).encode() # 仅首帧重新序列化(注入 trace_id);后续帧走原始 bytes
minted = True
# 请求级 trace_id 贯穿:此后本请求(含 run_in_threadpool 里的 harvest)每行日志自动带 trace_id
trace_id_ctx.set(trace_id)
base = pick_pricebot(trace_id)
# 按 trace_id 一致性 hash 选 pricebot 实例(同一比价的所有帧落同一进程,内存维护 state)
base = pick_pricebot(meta.get("trace_id"))
url = f"{base.rstrip('/')}{upstream_path}"
timeout = settings.PRICEBOT_COMPARE_TIMEOUT_SEC
step = meta.get("step")
logger.info(
"→ pricebot %s step=%s%s", upstream_path, step, " [mint]" if minted else "",
extra={"phase": "forward", "endpoint": upstream_path, "step": step,
"device_id": meta.get("device_id"), "minted": minted,
"query": meta.get("query"), "user_id": (user.id if user else None)},
"compare %s device_id=%s trace_id=%s step=%s",
upstream_path,
meta.get("device_id"),
meta.get("trace_id"),
meta.get("step"),
)
t0 = time.monotonic()
try:
client = get_pricebot_client()
resp = await client.post(
url, content=raw, headers={"Content-Type": "application/json"}, timeout=timeout
)
except httpx.RequestError as e:
logger.error(
"← pricebot %s 不可达: %s", upstream_path, e,
extra={"phase": "resp", "endpoint": upstream_path, "step": step,
"error": "upstream_unreachable"},
)
logger.error("[pricebot] request failed: %s", e)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=f"pricebot upstream unreachable: {e}",
@@ -159,112 +79,49 @@ async def _forward(
if resp.status_code >= 500:
logger.error(
"pricebot %s 5xx=%d", upstream_path, resp.status_code,
extra={"phase": "resp", "endpoint": upstream_path, "step": step,
"http_status": resp.status_code, "error": "upstream_5xx"},
"[pricebot] 5xx status=%d body=%s",
resp.status_code,
resp.text[:500],
)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=f"pricebot upstream returned {resp.status_code}",
)
resp_json = resp.json()
cost_ms = int((time.monotonic() - t0) * 1000)
if not isinstance(resp_json, dict):
return {}, trace_id, meta
# 回传 app-server 签发的 trace_id(新客户端首帧据此拿到,后续帧都带上它)
resp_json.setdefault("trace_id", trace_id)
_action = resp_json.get("action") or {}
logger.info(
"← pricebot %s cmd=%s cont=%s %dms", upstream_path,
_action.get("command"), resp_json.get("continue"), cost_ms,
extra={"phase": "resp", "endpoint": upstream_path, "step": step,
"command": _action.get("command"), "continue": resp_json.get("continue"),
"cost_ms": cost_ms},
)
# 首帧建 running 行(仅本次 mint;老客户端自带 trace_id → 不 mint → 由 done / 其 POST 建行)
if minted and harvest_first_frame:
try:
await run_in_threadpool(
_harvest_running_blocking, trace_id,
(user.id if user else None), "food",
meta.get("device_id"), meta.get("device_info"),
resp_json.get("trace_url"),
)
except Exception as e: # noqa: BLE001
logger.warning("harvest_running failed trace=%s: %s", trace_id, e)
return resp_json, trace_id, meta
return resp.json()
@router.post("/intent/recognize", summary="外卖比价 Phase 1 意图识别 (透传 + 建 running 行)")
async def intent_recognize(request: Request, user: OptionalUser) -> dict[str, Any]:
resp, _, _ = await _forward(request, "/api/intent/recognize", user)
return resp
@router.post("/intent/recognize", summary="外卖比价 Phase 1 意图识别 (透传到 pricebot)")
async def intent_recognize(request: Request) -> dict[str, Any]:
return await _passthrough(request, "/api/intent/recognize")
@router.post("/intent/step", summary="外卖比价 Phase 1 多帧意图识别 (透传, 仅淘宝源)")
async def intent_step(request: Request, user: OptionalUser) -> dict[str, Any]:
resp, _, _ = await _forward(request, "/api/intent/step", user)
return resp
@router.post("/intent/step", summary="外卖比价 Phase 1 多帧意图识别 (透传到 pricebot, 仅淘宝源)")
async def intent_step(request: Request) -> dict[str, Any]:
# 多帧版意图识别(展开+滚动采集→提取): 循环调用直到 done(done 帧顶层带
# result+calibration)。目前仅淘宝源走这条, 其它源走上面单次 /intent/recognize。
return await _passthrough(request, "/api/intent/step")
@router.post("/intent/precoupon/step", summary="外卖比价 Phase 0 识别前先用券 (透传, 仅美团源)")
async def intent_precoupon_step(request: Request, user: OptionalUser) -> dict[str, Any]:
resp, _, _ = await _forward(request, "/api/intent/precoupon/step", user)
return resp
@router.post(
"/intent/precoupon/step",
summary="外卖比价 Phase 0 意图识别前先用券 (透传到 pricebot, 仅美团源)",
)
async def intent_precoupon_step(request: Request) -> dict[str, Any]:
# 美团源平台『意图识别前先用券』多帧循环: 客户端在调 /intent/recognize 之前先循环
# 调本端点到 done(continue=false)。订单页底部有『点击使用X红包』就自动选最大免费券
# 用上, 已用券/无券则首帧秒过。与 /intent/step 同属 intent 域, 复用同一透传壳。
return await _passthrough(request, "/api/intent/precoupon/step")
@router.post("/price/step", summary="外卖比价 Phase 2 步进 (透传 + done 落库)")
async def price_step(request: Request, user: OptionalUser) -> dict[str, Any]:
resp, trace_id, meta = await _forward(request, "/api/price/step", user)
# 最终 done 帧(command=done 且 continue=false)→ harvest 更新成终态。
# (单平台中途 done 被 pricebot 改写成 wait+continue=true,不会命中这里,同 coupon 语义。)
action = resp.get("action") or {}
if action.get("command") == "done" and not resp.get("continue", True):
done_params = action.get("params") or {}
try:
await run_in_threadpool(
_harvest_done_blocking, trace_id, (user.id if user else None),
done_params, "food",
meta.get("device_id"), meta.get("device_info"),
resp.get("trace_url") or done_params.get("trace_url"),
)
except Exception as e: # noqa: BLE001
logger.warning("harvest_done failed trace=%s: %s", trace_id, e)
return resp
@router.post("/price/step", summary="外卖比价 Phase 2 步进 (透传到 pricebot)")
async def price_step(request: Request) -> dict[str, Any]:
return await _passthrough(request, "/api/price/step")
@router.post("/trace/epilogue", summary="比价结果页尾声帧 (透传到 pricebot, 用户视角截图入 trace)")
async def trace_epilogue(request: Request, user: OptionalUser) -> dict[str, Any]:
# App 收到 done、渲染完结果页后,把自己页面的截图(base64)传给 pricebot 存进 trace
# 目录并触发重传 —— trace 里补上"用户实际看到的汇总页"(步骤帧只有目标 App 画面)。
# body ~几百 KB(截图 base64), 纯透传壳(harvest_first_frame=False:不建行/不落库,
# 该 trace 的记录已由 done/finalize 落终态)。#112 原走 _passthrough,合并到 harvest
# 分支后统一走 _forward(_passthrough 已并入它)。
resp, _, _ = await _forward(
request, "/api/trace/epilogue", user, harvest_first_frame=False,
)
return resp
@router.post("/trace/finalize", summary="比价 trace 收尾上云 (透传 + 夭折落库)")
async def trace_finalize(request: Request, user: OptionalUser) -> dict[str, Any]:
# 用户终止 / Phase1 未识别没到 done 帧: pricebot 打包半截上云返回 {trace_url};
# app-server 顺手把该 trace 的 running 行更新成夭折终态(**不降级 success**)。
# 客户端 finalize body 带 status(cancelled/failed)+ reason;老客户端只带 trace_id →
# 默认 cancelled,其后续 /compare/record 上报再补精确态。
resp, trace_id, meta = await _forward(
request, "/api/trace/finalize", user, harvest_first_frame=False,
)
try:
await run_in_threadpool(
_harvest_abort_blocking, trace_id,
(meta.get("status") or "cancelled"),
(meta.get("reason") or meta.get("information")),
(resp.get("trace_url") if isinstance(resp, dict) else None),
)
except Exception as e: # noqa: BLE001
logger.warning("harvest_abort failed trace=%s: %s", trace_id, e)
return resp
@router.post("/trace/finalize", summary="比价 trace 收尾上云 (透传到 pricebot, 终止/未识别拿 trace_url)")
async def trace_finalize(request: Request) -> dict[str, Any]:
# 用户终止 / Phase1 未识别没走到 done 帧, pricebot 没上云也没回传 trace_url。客户端收尾时
# 打这个, _passthrough 按 trace_id 一致性 hash 落到处理这条 trace 的同一 pricebot 进程
# (dir_cache 在那, 才能算对 trace 目录), 由后者打包上云返回 {trace_url}。
return await _passthrough(request, "/api/trace/finalize")
+22 -9
View File
@@ -1,13 +1,15 @@
"""比价记录 endpoint(「我的比价记录」+ admin 数据源)。
"""比价记录 endpoint(「我的比价记录」数据源)。
路由前缀 `/api/v1/compare`:
POST /record (灰度期兼容)老客户端上报,按 **trace_id** 幂等 + 不降级 success
POST /record 上报一次比价结果(幂等:同 user+trace_id 覆盖)
GET /records 比价记录列表(游标分页)
GET /records/{id} 单条详情(含 raw_payload 全量)
**均需鉴权**(CurrentUser)。⚠️ 写路径现以 `compare.py` 透传壳的**后端 harvest** 为主
(帧0 建 running 行 → done/finalize 落终态,新客户端不再 POST);本 POST /record 仅灰度期
给老客户端用,与 harvest 按 trace_id reconcile。新版覆盖够高后可下线本 POST(阶段3)。
**均需鉴权**(CurrentUser)——与同文件无关的不鉴权透传 `compare.py` 分开:那个是
转发壳(MVP 不鉴权),这里是按用户维度落库的业务接口,必须有 user_id。
注:本轮只做 server 端,客户端(android 仓)在 done 帧后调 POST /record 上报的改动
另起一轮(见 app-server docs/待办与技术债.md P1)。
"""
from __future__ import annotations
@@ -19,15 +21,16 @@ from app.api.deps import CurrentUser, DbSession
from app.db.session import SessionLocal
from app.models.comparison import ComparisonRecord
from app.repositories import comparison as crud_compare
from app.repositories import invite as crud_invite
from app.services.pricebot_llm_calls import fetch_llm_calls
from app.schemas.compare_record import (
CompareStatsOut,
ComparisonRecordCreatedOut,
ComparisonRecordDetailOut,
ComparisonRecordIn,
ComparisonRecordOut,
ComparisonRecordPage,
ComparisonRecordOut,
)
from app.services.pricebot_llm_calls import fetch_llm_calls
logger = logging.getLogger("shagua.compare_record")
@@ -50,8 +53,18 @@ def report_record(
# 任务做,不阻塞上报响应(顺带给 pricebot 落盘留足余量)。upsert 已 commit,后台用
# 独立 session 按 record id 回填 llm_calls + 派生 llm_call_count/retry_count。
background_tasks.add_task(_backfill_llm_calls, rec.id, rec.trace_id)
# 注:邀请发奖已从"比价成功"挪到"实际下单"(见 api/v1/order.py report_order)——
# 冰拍板:被邀请人完成比价并实际下单才算邀请成功,仅完成比价不再发奖
# 邀请 v2 发奖:被邀请人完成一次【成功】比价 → 给邀请人发邀请奖励金(幂等,只发一次)。
# best-effort:发奖异常不影响比价上报本身(rec 已 commit),只 log;邀请人补偿靠后续对账
if rec.status == "success":
try:
reward = crud_invite.try_reward_on_compare(db, user.id)
if reward.status == "granted":
logger.info(
"invite compare reward granted inviter=%s invitee=%s cents=%s",
reward.inviter_user_id, user.id, reward.reward_cents,
)
except Exception as e: # noqa: BLE001 best-effort,发奖失败不阻塞上报
logger.warning("invite compare reward failed invitee=%s: %s", user.id, e)
logger.info(
"compare record user=%s trace=%s biz=%s status=%s saved=%s (llm_calls backfill queued)",
user.id,
-19
View File
@@ -1,14 +1,9 @@
import logging
from fastapi import APIRouter
from app.api.deps import CurrentUser, DbSession
from app.repositories import invite as crud_invite
from app.repositories import savings as crud_savings
from app.schemas.order import OrderReportOut, OrderReportRequest
logger = logging.getLogger("shagua.order")
router = APIRouter(prefix="/api/v1/order", tags=["order"])
@@ -20,20 +15,6 @@ router = APIRouter(prefix="/api/v1/order", tags=["order"])
def report_order(req: OrderReportRequest, user: CurrentUser, db: DbSession) -> OrderReportOut:
# 记账唯一真相表是 savings_record(source='compare')。
rec, duplicated = crud_savings.create_from_report(db, user.id, req)
# 邀请 v2 发奖:被邀请人【实际下单】(而非仅完成比价)才给邀请人发邀请奖励金(幂等,只发一次)。
# 触发点从"比价成功上报"挪到这里(冰:完成比价并实际下单才算成功)。仅首次真实上报触发,
# 重复上报(duplicated)跳过。best-effort:发奖异常不影响订单上报本身(rec 已 commit),只 log;
# try_reward_on_compare 自带 compare_reward_granted 幂等闸,漏发靠后续对账补。
if not duplicated:
try:
reward = crud_invite.try_reward_on_compare(db, user.id)
if reward.status == "granted":
logger.info(
"invite order reward granted inviter=%s invitee=%s cents=%s",
reward.inviter_user_id, user.id, reward.reward_cents,
)
except Exception as e: # noqa: BLE001 best-effort,发奖失败不阻塞订单上报
logger.warning("invite order reward failed invitee=%s: %s", user.id, e)
return OrderReportOut(
id=rec.id,
platform=rec.platform or req.platform,
-12
View File
@@ -69,18 +69,6 @@ def complete_onboarding(
return OkResponse()
@router.post("/onboarding/reset", response_model=OkResponse, summary="重置新手引导(删该 设备+账号 完成标记,下次登录重走)")
def reset_onboarding(
req: OnboardingCompleteRequest, user: CurrentUser, db: DbSession
) -> OkResponse:
"""删该 (当前账号, device_id) 的引导完成标记 → 下次登录 onboarding_completed=false,客户端重走。
与 /onboarding/complete 互逆。device_id 取客户端硬件级 ANDROID_ID(与登录/complete 一致)。
幂等:无记录也返回 ok。仅删自己(当前 JWT 用户)这台设备的记录,不影响别的账号/设备。"""
deleted = onboarding_repo.delete_completion(db, user_id=user.id, device_id=req.device_id)
logger.info("onboarding reset user_id=%d device_len=%d deleted=%d", user.id, len(req.device_id), deleted)
return OkResponse()
@router.get(
"/onboarding/status",
response_model=OnboardingStatusResponse,
+26
View File
@@ -81,6 +81,7 @@ class Settings(BaseSettings):
SMS_SIGN_ID: int = 31729 # 极光短信签名 ID(非机密,可被 .env 覆盖)
SMS_TEMPLATE_ID: int = 1 # 极光短信模板 ID(变量名 code,有效期 5 分钟)
SMS_CODE_LENGTH: int = 6 # 验证码位数(本服务生成;前端 code 字段 4-8 位兼容)
SMS_DAILY_LIMIT_PER_PHONE: int = 10 # 单手机号每日发送上限(防刷 + 控费)
SMS_MAX_VERIFY_ATTEMPTS: int = 5 # 单个验证码最多校验失败次数,超过即作废(防爆破)
# ===== 测试账号(release 包全流程联调用)=====
@@ -316,6 +317,31 @@ class Settings(BaseSettings):
return []
return [o.strip() for o in self.CORS_ALLOW_ORIGINS.split(",") if o.strip()]
# ===== 可观测(OpenObserve 接口指标)=====
# 采集每个接口的 QPS + 耗时 + 错误率,批量直采到 OpenObserve(本地 Docker)。
# 默认关(prod 安全):未开启 → 中间件透传、worker 不启动,整套 no-op。
# 开启需 ENABLED=true 且 ENDPOINT/USER/PASSWORD 齐全(见 observe_configured)。
OBSERVE_ENABLED: bool = False
OBSERVE_ENDPOINT: str = "http://localhost:5080" # OpenObserve base URL
OBSERVE_ORG: str = "default" # 组织名
OBSERVE_STREAM: str = "app_requests" # stream 名(首次上报自动建)
OBSERVE_USER: str = "" # Basic auth 邮箱
OBSERVE_PASSWORD: str = "" # Basic auth 密码/token
OBSERVE_FLUSH_INTERVAL_SEC: float = 5.0 # worker 最长攒批间隔
OBSERVE_BATCH_MAX: int = 200 # 单批最大事件数
OBSERVE_QUEUE_MAX: int = 10000 # 有界队列上限,满则丢
OBSERVE_TIMEOUT_SEC: float = 5.0 # 上报 HTTP 超时
@property
def observe_configured(self) -> bool:
"""观测上报可用 = 总开关开 且 endpoint/账号/密码齐全(缺则整套 no-op)。"""
return bool(
self.OBSERVE_ENABLED
and self.OBSERVE_ENDPOINT
and self.OBSERVE_USER
and self.OBSERVE_PASSWORD
)
@property
def is_prod(self) -> bool:
return self.APP_ENV == "prod"
+6 -18
View File
@@ -11,10 +11,7 @@ from typing import Any
from app.core import rewards as r
# type 约定(给前端渲染编辑控件用):int / int_list / dict_str_int / bool / enum
# hidden=True:仍是合法可配项(业务照常 get_value / admin 可经专用端点读写),但**不在通用
# 「系统配置」页渲染**(admin/routers/config.py:list_config 按此过滤)。用于把已下线/已改由
# 专用页管理的项从福利页 Tab 收起,同时保留后端默认值与写入能力。
# type 约定(给前端渲染编辑控件用):int / int_list / dict_str_int / bool
CONFIG_DEFS: dict[str, dict[str, Any]] = {
"signin_rewards": {
"default": list(r.SIGNIN_REWARDS), "label": "签到 7 天金币档位",
@@ -33,21 +30,18 @@ CONFIG_DEFS: dict[str, dict[str, Any]] = {
"default": r.WITHDRAW_MAX_CENTS, "label": "提现最高额(分)",
"group": "钱包", "type": "int",
},
# 从福利页 Tab 收起(hidden)的项:任务 / 里程碑 整组 + 看广告组里的「单次金币(遗留展示值)/
# 每轮次数 / 比价领券信息流广告开关 comparing_ad_enabled」。key 与默认值保留、后端业务照常读取,
# 仅不在配置页渲染。看广告组保留可见的:每日上限 / 单次金币上限 / 关闭后冷却。
"task_rewards": {
"default": dict(r.TASK_REWARDS), "label": "一次性任务奖励",
"group": "任务", "type": "dict_str_int", "help": "task_key → 金币。", "hidden": True,
"group": "任务", "type": "dict_str_int", "help": "task_key → 金币。",
},
"record_milestones": {
"default": list(r.RECORD_MILESTONES), "label": "比价里程碑金币档位",
"group": "里程碑", "type": "int_list", "hidden": True,
"group": "里程碑", "type": "int_list",
"help": "累计成功比价第 1~N 档解锁发的金币。",
},
"ad_reward_coin": {
"default": r.AD_REWARD_COIN, "label": "看广告单次金币",
"group": "看广告", "type": "int", "hidden": True,
"group": "看广告", "type": "int",
"help": "历史兼容/测试展示值;正式发放按 eCPM 公式计算。",
},
"ad_daily_limit": {
@@ -60,7 +54,7 @@ CONFIG_DEFS: dict[str, dict[str, Any]] = {
},
"ad_round_count": {
"default": r.VIDEO_ROUND_REQUIRED_COUNT, "label": "每轮看广告次数",
"group": "看广告", "type": "int", "hidden": True, "help": "当前为 1,表示每次广告关闭后触发短冷却。",
"group": "看广告", "type": "int", "help": "当前为 1,表示每次广告关闭后触发短冷却。",
},
"ad_cooldown_sec": {
"default": r.VIDEO_ROUND_COOLDOWN_SECONDS, "label": "广告关闭后冷却(秒)",
@@ -73,7 +67,7 @@ CONFIG_DEFS: dict[str, dict[str, Any]] = {
},
"comparing_ad_enabled": {
"default": True, "label": "比价/领券期信息流广告",
"group": "看广告", "type": "bool", "hidden": True,
"group": "看广告", "type": "bool",
"help": (
"开启后,比价进行中 + 领券等候期会在悬浮窗展示穿山甲信息流广告(变现行为);"
"关闭则全程不出广告。客户端按 app 启动 / 每场比价开始时拉取并缓存,故为「最终一致」的"
@@ -90,10 +84,4 @@ CONFIG_DEFS: dict[str, dict[str, Any]] = {
"提现页「批量对账」手动按钮不受影响。"
),
},
# 首页轮播数据源(hidden:不进通用配置 Tab,由「首页轮播种子」页的专用端点 /marquee-seeds/mode 读写)。
"marquee_feed_mode": {
"default": "mixed", "label": "首页轮播数据源",
"group": "首页轮播", "type": "enum", "hidden": True,
"help": "mixed=真实优先+种子补位(默认);real=只用真实比价记录;seed=只用种子/合成(演示)。",
},
}
+8 -57
View File
@@ -2,15 +2,9 @@
业务代码用 `logger = logging.getLogger("shagua.xxx")` 即可, 本模块在 main.py 启动时调一次。
- 控制台(stdout): 人类可读文本, 给 systemd / 本地看; 有 trace 时行尾附 `trace=xxx`
- 文件 `logs/app-server.log`: 单行 JSON, 供**阿里云 SLS / Logtail** 采集(JSON 模式零正则);
异常栈内嵌为字段不换行 → 每条日志一行。
- **结构化字段(SLS 可直接查/聚合)**:
- `trace_id`: 请求级贯穿——在入口 `trace_id_ctx.set(...)` 后, 本请求内**每一行日志**(含
run_in_threadpool 里的 harvest, contextvars 自动拷进线程)都自动带上, 无需手写。
SLS 里 `trace_id: "xxx"` 一查即得整条比价链路, 按 time 升序即请求顺序。
- 任意 `logger.info(msg, extra={"phase": ..., "step": ..., "command": ...})` 的 extra
键都会平铺进 JSON 顶层 → SLS 可按 phase/step/command/cost_ms 等过滤聚合。
- 控制台(stdout): 人类可读文本, 给 systemd / 本地看。
- 文件 `logs/app-server.log`: 单行 JSON, 供阿里云 SLS/Logtail 采集(JSON 模式零正则);
异常栈为字段内嵌不换行 → 每条日志一行。
- 环境变量:
- LOG_JSON_CONSOLE=1 控制台也输出 JSON
- LOG_DIR / LOG_FILE 改落盘路径(默认 logs/app-server.log)
@@ -23,36 +17,13 @@ import json
import logging
import os
import sys
from contextvars import ContextVar
from datetime import datetime
from logging.handlers import RotatingFileHandler
from pathlib import Path
# 请求级 trace_id:入口(如 compare.py 透传壳)set 之后, 本请求上下文(含 run_in_threadpool
# 拷贝出去的线程)内所有日志自动带上。默认空串 = 非请求上下文(启动/后台 worker)。
trace_id_ctx: ContextVar[str] = ContextVar("trace_id", default="")
# 标准 LogRecord 属性 + 格式化期附加项:凡不在此集合的 record 属性都视为业务 extra, 平铺进 JSON。
_RESERVED = set(
logging.LogRecord("", 0, "", 0, "", (), None).__dict__
) | {"message", "asctime", "trace_id", "taskName"}
class _ContextFilter(logging.Filter):
"""把 trace_id_ctx 注入每条 record(供两个 formatter 取用)。挂在 handler 上,
命中每条(含 propagate 上来的)记录, 在 format 之前置好 record.trace_id。"""
def filter(self, record: logging.LogRecord) -> bool:
if not hasattr(record, "trace_id"):
record.trace_id = trace_id_ctx.get()
return True
class JsonFormatter(logging.Formatter):
"""LogRecord 单行 JSON(SLS/Logtail 友好)。trace_id 提到顶层、extra 键平铺, 异常栈内嵌。"""
"""LogRecord 序列化成单行 JSON(SLS/Logtail 友好)。异常栈内嵌为字段, 整条仍是一行"""
def __init__(self, service: str = "app-server"):
super().__init__()
self.service = service
@@ -64,17 +35,10 @@ class JsonFormatter(logging.Formatter):
"level": record.levelname,
"service": self.service,
"logger": record.name,
"func": record.funcName,
"line": record.lineno,
"message": record.getMessage(),
}
tid = getattr(record, "trace_id", "") or trace_id_ctx.get()
if tid:
data["trace_id"] = tid
# 业务 extra 字段(phase / step / endpoint / command / cost_ms / status ...)平铺进顶层
for k, v in record.__dict__.items():
if k not in _RESERVED and not k.startswith("_"):
data[k] = v
data["func"] = record.funcName
data["line"] = record.lineno
data["message"] = record.getMessage()
if record.exc_info:
data["exception"] = self.formatException(record.exc_info)
if record.stack_info:
@@ -82,15 +46,6 @@ class JsonFormatter(logging.Formatter):
return json.dumps(data, ensure_ascii=False, default=str)
class TextFormatter(logging.Formatter):
"""控制台文本:标准行 + 有 trace_id 时行尾附 `trace=xxx`(无 trace 的启动/后台日志不加噪)。"""
def format(self, record: logging.LogRecord) -> str:
base = super().format(record)
tid = getattr(record, "trace_id", "") or trace_id_ctx.get()
return f"{base} trace={tid}" if tid else base
_CONFIGURED = False
@@ -108,17 +63,14 @@ def setup_logging(debug: bool = False) -> None:
for h in list(root.handlers):
root.removeHandler(h)
ctx_filter = _ContextFilter()
# 控制台: 默认文本(systemd/本地看); LOG_JSON_CONSOLE=1 时输出 JSON
console = logging.StreamHandler(sys.stdout)
if os.getenv("LOG_JSON_CONSOLE") == "1":
console.setFormatter(JsonFormatter(service))
else:
console.setFormatter(
TextFormatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
)
console.addFilter(ctx_filter)
root.addHandler(console)
# 文件: 单行 JSON, 供 Logtail 采集(自动轮转, 单文件 10MB, 保留 5 个)
@@ -130,7 +82,6 @@ def setup_logging(debug: bool = False) -> None:
log_file, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8",
)
file_handler.setFormatter(JsonFormatter(service))
file_handler.addFilter(ctx_filter)
root.addHandler(file_handler)
# 第三方库降噪
+110
View File
@@ -0,0 +1,110 @@
"""接口指标埋点:有界事件队列 + 纯 ASGI 中间件。
每个 HTTP 请求测总耗时、抓路由模板 + 状态码,非阻塞塞进有界队列;由 observe_worker
后台批量上报到 OpenObserve。请求路径上无任何 I/O。未配置观测时中间件直接透传。
"""
from __future__ import annotations
import asyncio
import os
import time
from starlette.routing import Match
from app.core.config import settings
# 不采集的路径(纯噪音):健康检查。
_SKIP_PATHS = frozenset({"/health"})
# 未匹配路由(404/扫描器)归一到此,防维度爆炸。
_UNMATCHED = "__unmatched__"
# service 字段:与 logging.py 同源(LOG_SERVICE_NAME),默认 app-server。
_SERVICE = os.getenv("LOG_SERVICE_NAME", "app-server")
# 有界事件队列(懒创建,见 get_queue):首次取用时在运行中的 loop 里建,避免 import 期
# 无 loop 的边角问题;put_nowait/get_nowait 不需运行中的 loop → 可在无 loop 下测试。
_queue: asyncio.Queue[dict] | None = None
# 队列满时的丢弃计数,worker 定期取出打日志。
_dropped = 0
def get_queue() -> asyncio.Queue[dict]:
"""返回全局有界事件队列(懒创建)。测试可 monkeypatch 模块级 _queue 换成小队列。"""
global _queue
if _queue is None:
_queue = asyncio.Queue(maxsize=settings.OBSERVE_QUEUE_MAX)
return _queue
def take_dropped() -> int:
"""取出并清零累计丢弃数(供 worker 打点)。"""
global _dropped
n, _dropped = _dropped, 0
return n
def record_event(event: dict) -> None:
"""非阻塞入队;队列满则丢弃当前事件并计数。永不抛异常、永不阻塞请求。"""
global _dropped
try:
get_queue().put_nowait(event)
except asyncio.QueueFull:
_dropped += 1
def _resolve_route(scope) -> str:
"""从 scope 取路由模板(如 /things/{tid})。优先 scope['route'](现代 Starlette
路由后写入);取不到则手动匹配一次(老版本兜底);仍无 → __unmatched__(404/扫描器)。"""
route = scope.get("route")
path = getattr(route, "path", None)
if path:
return path
app_ = scope.get("app")
router = getattr(app_, "router", None)
for candidate in getattr(router, "routes", []):
try:
match, _ = candidate.matches(scope)
except Exception: # noqa: BLE001 - 匹配兜底,任一路由异常不影响整体
continue
if match == Match.FULL and getattr(candidate, "path", None):
return candidate.path
return _UNMATCHED
class RequestMetricsMiddleware:
"""纯 ASGI 中间件:测每个 http 请求耗时,记 method/route/status/duration。
放在最外层(main.py 里 CORS 之后 add),测到含 CORS 的完整耗时。未配置观测 → 透传。
"""
def __init__(self, app) -> None:
self.app = app
async def __call__(self, scope, receive, send) -> None:
if scope["type"] != "http" or not settings.observe_configured:
await self.app(scope, receive, send)
return
if scope.get("path") in _SKIP_PATHS:
await self.app(scope, receive, send)
return
start = time.perf_counter()
status_holder = {"status": 500} # 下游异常未产出 response 时兜底 500
async def send_wrapper(message) -> None:
if message["type"] == "http.response.start":
status_holder["status"] = message["status"]
await send(message)
try:
await self.app(scope, receive, send_wrapper)
finally:
duration_ms = (time.perf_counter() - start) * 1000.0
record_event({
"_timestamp": int(time.time() * 1_000_000), # µs,OpenObserve 时间列
"service": _SERVICE,
"env": settings.APP_ENV,
"method": scope.get("method", ""),
"route": _resolve_route(scope),
"status": status_holder["status"],
"duration_ms": round(duration_ms, 3),
})
+128
View File
@@ -0,0 +1,128 @@
"""接口指标后台上报 worker:批量 drain 事件队列 → POST 到 OpenObserve。
对齐 heartbeat_monitor_worker 等的 start_*/stop_* 形态。best-effort 遥测:catch 全部
异常,上报失败直接丢批不重试。未配置观测 → start 返回 None(不启动),整套 no-op。
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
import httpx
from app.core.config import settings
from app.core.observe import get_queue, take_dropped
logger = logging.getLogger("shagua.observe")
# 上报用的 httpx client,start 时建、stop 时关。
_client: httpx.AsyncClient | None = None
async def _collect_batch() -> list[dict]:
"""等到 ≥1 条(或到 flush 间隔)后,连抽到 BATCH_MAX 条或抽空。超时且空 → 返回 []。"""
queue = get_queue()
batch: list[dict] = []
try:
first = await asyncio.wait_for(
queue.get(), timeout=settings.OBSERVE_FLUSH_INTERVAL_SEC
)
except asyncio.TimeoutError: # noqa: UP041 - 3.10 兼容:该版 wait_for 抛的 asyncio.TimeoutError ≠ 内置 TimeoutError
return batch
batch.append(first)
while len(batch) < settings.OBSERVE_BATCH_MAX:
try:
batch.append(queue.get_nowait())
except asyncio.QueueEmpty:
break
return batch
async def _post_batch(client: httpx.AsyncClient, batch: list[dict]) -> None:
"""POST 一批事件到 OpenObserve 的 _json ingest 端点。非 2xx 仅告警。"""
url = f"/api/{settings.OBSERVE_ORG}/{settings.OBSERVE_STREAM}/_json"
resp = await client.post(url, json=batch)
if resp.status_code >= 300:
logger.warning(
"observe ingest failed status=%s body=%s",
resp.status_code,
resp.text[:200],
)
async def _run_loop(client: httpx.AsyncClient) -> None:
try:
while True:
batch = await _collect_batch()
dropped = take_dropped()
if dropped:
logger.warning("observe dropped %d events (queue full)", dropped)
if not batch:
continue
try:
await _post_batch(client, batch)
except Exception: # noqa: BLE001 - best-effort 遥测,失败丢批不重试、不退出
logger.warning(
"observe post batch failed, dropped %d events",
len(batch),
exc_info=True,
)
except asyncio.CancelledError:
logger.info("observe worker stopped")
raise
def start_observe_worker() -> asyncio.Task | None:
"""启动上报 worker。未配置观测 → 返回 None(no-op)。约定每进程只调一次(lifespan)。"""
global _client
if not settings.observe_configured:
return None
if _client is not None:
# 约定 start 每进程只调一次;已启动则不重复建 client(避免泄漏旧连接池)。
logger.warning("observe worker already started; ignoring duplicate start")
return None
_client = httpx.AsyncClient(
base_url=settings.OBSERVE_ENDPOINT,
auth=(settings.OBSERVE_USER, settings.OBSERVE_PASSWORD),
timeout=settings.OBSERVE_TIMEOUT_SEC,
)
logger.info(
"observe worker started endpoint=%s org=%s stream=%s",
settings.OBSERVE_ENDPOINT,
settings.OBSERVE_ORG,
settings.OBSERVE_STREAM,
)
return asyncio.create_task(_run_loop(_client), name="observe-worker")
async def stop_observe_worker(task: asyncio.Task | None) -> None:
"""收尾:cancel worker → best-effort 发最后一批 → 关 client。"""
global _client
if task is None:
return
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
dropped = take_dropped() # 收口:补记最后一个 flush 窗口累计的丢弃数,不让账丢在关停期
if dropped:
logger.warning("observe dropped %d events (queue full) before shutdown", dropped)
if _client is not None:
# worker 已停,安全 drain 剩余并 best-effort 发最后一批(短超时,不卡关停);
# 超过一批(BATCH_MAX)的剩余直接丢,不做多轮 flush(best-effort,关停从速)。
try:
queue = get_queue()
final: list[dict] = []
while len(final) < settings.OBSERVE_BATCH_MAX:
try:
final.append(queue.get_nowait())
except asyncio.QueueEmpty:
break
if final:
await asyncio.wait_for(
_post_batch(_client, final), timeout=settings.OBSERVE_TIMEOUT_SEC
)
except Exception: # noqa: BLE001 - 关停期尽力而为,失败忽略
pass
await _client.aclose()
_client = None
+25 -8
View File
@@ -8,16 +8,15 @@
校验)→ 鉴权复用极光一键登录的 `JG_APP_KEY`/`JG_MASTER_SECRET`(同一极光应用)。
验证码存储:**进程内存**(单 worker uvicorn 够用)。重启丢失(用户重发即可)。多
worker / 多机时内存不共享 → 冷却、校验都会失效,届时迁移到 DB/Redis。
worker / 多机时内存不共享 → 冷却、每日上限、校验都会失效,届时迁移到 DB/Redis。
见 docs/待办与技术债.md。
防刷层(短信花钱 + `/sms/send` 在登录前无法 JWT 鉴权):
防刷层(短信花钱 + `/sms/send` 在登录前无法 JWT 鉴权):
1. 单号 `SMS_SEND_INTERVAL_SEC` 冷却(本文件)
2. 单设备(device_id)每小时频控(api 层 auth.sms_send 内 enforce_rate_limit)+ 极光控制台 IP 白名单/防轰炸(运维侧)。
2. 单号每日 `SMS_DAILY_LIMIT_PER_PHONE` 条上限(本文件)
3. 单设备(device_id)每小时频控(api 层 auth.sms_send 内 enforce_rate_limit)+ 极光控制台 IP 白名单/防轰炸(运维侧)。
⚠️ 原「单 IP 频控(rate_limit 依赖)」2026-06-26 按产品要求删除、改设备维度;但 device_id 客户端可伪造/轮换,
脚本轮换 id 能绕过本层 → 挡脚本狂发主要靠极光控制台侧(+ 可选 nginx 限流)。
⚠️ 原「单号每日上限」2026-07-03 按精简要求删除(mentor 定:登录风控只留单号冷却 + 单设备频控);
单号维度现仅剩 60s 冷却,「换号轰炸」由单设备频控封顶。
另:单码校验失败 `SMS_MAX_VERIFY_ATTEMPTS` 次即作废(防爆破),验过即作废(一次性)。
"""
from __future__ import annotations
@@ -27,6 +26,7 @@ import logging
import secrets
import time
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
import httpx
@@ -56,17 +56,22 @@ class _CodeRecord:
# 进程内存(单 worker 有效;多 worker 不共享,见模块 docstring)
_codes: dict[str, _CodeRecord] = {} # phone -> 当前有效验证码
_last_sent: dict[str, float] = {} # phone -> 上次发送 epoch(冷却)
_daily_count: dict[str, tuple[str, int]] = {} # phone -> (date_str, 当日发送数)
_lock = Lock()
_GC_THRESHOLD = 10000 # 任一内存 dict 超此阈值,send 时顺手清过期项(防无限增长,仿 ratelimit)
def _today() -> str:
return datetime.now().strftime("%Y-%m-%d")
def _gen_code() -> str:
"""生成 N 位数字验证码(用 secrets 而非 random;允许前导 0)。"""
return "".join(secrets.choice("0123456789") for _ in range(settings.SMS_CODE_LENGTH))
def _gc(now: float) -> None:
"""顺手清理过期内存项,防个 dict 无限增长。仅在持锁时调用,且某 dict 超
"""顺手清理过期内存项,防个 dict 无限增长。仅在持锁时调用,且某 dict 超
_GC_THRESHOLD 才扫它(低频,开销可忽略)。"""
if len(_codes) > _GC_THRESHOLD:
for p in [p for p, r in _codes.items() if now > r.expires_at]:
@@ -75,6 +80,10 @@ def _gc(now: float) -> None:
cutoff = now - settings.SMS_SEND_INTERVAL_SEC
for p in [p for p, ts in _last_sent.items() if ts < cutoff]:
_last_sent.pop(p, None)
if len(_daily_count) > _GC_THRESHOLD:
today = _today()
for p in [p for p, (d, _c) in _daily_count.items() if d != today]:
_daily_count.pop(p, None)
def send_code(phone: str) -> int:
@@ -93,9 +102,17 @@ def send_code(phone: str) -> int:
remain = int(settings.SMS_SEND_INTERVAL_SEC - elapsed)
raise SmsError(f"发送过于频繁,请 {remain}s 后再试")
today = _today()
day, cnt = _daily_count.get(phone, ("", 0))
if day != today:
cnt = 0
if cnt >= settings.SMS_DAILY_LIMIT_PER_PHONE:
raise SmsError("今日验证码发送次数已达上限,请明天再试")
code = _gen_code()
# 预占:先记冷却/存码,释放锁后再发网络(发失败保留冷却,见下)
# 预占:先记冷却/计数/存码,释放锁后再发网络(发失败保留冷却+计数,见下)
_last_sent[phone] = now
_daily_count[phone] = (today, cnt + 1)
_codes[phone] = _CodeRecord(code=code, expires_at=now + settings.SMS_CODE_TTL_SEC)
# --- lock 外:真正发送(网络 IO 不持锁)---
@@ -106,7 +123,7 @@ def send_code(phone: str) -> int:
_send_via_jiguang(phone, code)
logger.info("[SMS] sent to %s****", phone[:3])
except Exception as e:
# 发送失败:**保留冷却**(失败也限速,挡住余额不足/签名失效时
# 发送失败:**保留冷却 + 每日计数**(失败也限速,挡住余额不足/签名失效时
# 前端重试狂打极光),只清掉没发出去的码(用户收不到,留着无意义且占内存)。
with _lock:
_codes.pop(phone, None)
+10
View File
@@ -50,6 +50,11 @@ from app.core.heartbeat_monitor_worker import (
stop_heartbeat_monitor,
)
from app.core.logging import setup_logging
from app.core.observe import RequestMetricsMiddleware
from app.core.observe_worker import (
start_observe_worker,
stop_observe_worker,
)
from app.core.pricebot_client import aclose_pricebot_client, get_pricebot_client
from app.core.withdraw_reconcile_worker import (
start_withdraw_reconcile_worker,
@@ -74,12 +79,14 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
reconcile_task = start_withdraw_reconcile_worker()
heartbeat_task = start_heartbeat_monitor()
daily_exchange_task = start_daily_exchange_worker()
observe_task = start_observe_worker()
try:
yield
finally:
await stop_heartbeat_monitor(heartbeat_task)
await stop_withdraw_reconcile_worker(reconcile_task)
await stop_daily_exchange_worker(daily_exchange_task)
await stop_observe_worker(observe_task)
await aclose_pricebot_client()
logger.info("shutting down")
@@ -101,6 +108,9 @@ if settings.cors_origins_list:
allow_headers=["*"],
)
# 接口指标埋点(放在 CORS 之后 = 最外层:测到含 CORS 的完整耗时)。未配置观测时中间件自 no-op。
app.add_middleware(RequestMetricsMiddleware)
@app.get("/health", tags=["meta"])
def health() -> dict[str, str]:
-1
View File
@@ -5,7 +5,6 @@ from app.models.ad_pangle_revenue import AdPangleDailyRevenue # noqa: F401
from app.models.ad_reward import AdRewardRecord # noqa: F401
from app.models.ad_watch_log import AdWatchLog # noqa: F401
from app.models.admin import AdminAuditLog, AdminUser # noqa: F401
from app.models.admin_role import AdminRole # noqa: F401
from app.models.analytics_event import AnalyticsEvent # noqa: F401
from app.models.app_config import AppConfig # noqa: F401
from app.models.comparison import ComparisonRecord # noqa: F401
-3
View File
@@ -25,9 +25,6 @@ class AdminUser(Base):
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
username: Mapped[str] = mapped_column(String(64), unique=True, index=True, nullable=False)
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
# 明文登录密码:仅「后台 UI 创建/重置」的管理员留存,供超管在权限管理页复看转交。
# 脚本/起后台时建的超管账号不写(为 None → 前端「不显示密码」)。⚠️ 内部工具便利取舍,见 create/list。
plain_password: Mapped[str | None] = mapped_column(String(128), nullable=True)
# super_admin(全权+管账号)/ finance(钱:提现+金币)/ operator(用户+反馈+大盘)
role: Mapped[str] = mapped_column(String(20), nullable=False, default="operator")
# active / disabled
-42
View File
@@ -1,42 +0,0 @@
"""admin 角色 → 可见页面(权限)映射表。
RBAC 的「角色」侧:每个角色持有一组「页面 key」(= 左侧导航项),决定该角色登录后台后左边能看到
哪些页。super_admin 是内建全权角色(is_builtin=True),恒可见全部页(effective_pages 里特判,
不依赖本表存的 pages)、不可编辑/删除。其余角色(含 operator/finance)可由 super_admin 增删改。
admin_user.role 存的是本表的 name(字符串弱引用,不建外键——与既有 require_role 字符串口径一致;
删除在用角色由业务层拦截,见 routers/roles.py)。页面 key 清单见 app/admin/permissions.py。
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import JSON, Boolean, DateTime, Integer, String, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base
_JSON = JSON().with_variant(JSONB(), "postgresql")
class AdminRole(Base):
__tablename__ = "admin_role"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
# 角色标识 key(不可变,承重):admin_user.role 引用它、require_role 按它鉴权。
# 内建 = super_admin/operator/finance/tech(英文,勿改);自定义 = 创建时的名称。
name: Mapped[str] = mapped_column(String(32), unique=True, index=True, nullable=False)
# 展示名(可改):内建 = 管理员/运营/财务/技术;自定义默认 = name。UI 一律展示 label。
label: Mapped[str] = mapped_column(String(32), nullable=False, default="")
# 该角色可见页面 key 列表(= 左侧导航项),见 app/admin/permissions.py 的 PERMISSION_CATALOG
pages: Mapped[list] = mapped_column(_JSON, nullable=False, default=list)
# 内建角色(当前仅 super_admin):不可编辑/删除,恒全权
is_builtin: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
def __repr__(self) -> str: # pragma: no cover
return f"<AdminRole name={self.name} pages={len(self.pages)} builtin={self.is_builtin}>"
+4 -11
View File
@@ -39,19 +39,16 @@ _JSON = JSON().with_variant(JSONB(), "postgresql")
class ComparisonRecord(Base):
__tablename__ = "comparison_record"
__table_args__ = (
# trace_id 由 app-server 签发、全局唯一 → 一次比价一行,后端 harvest 按它 upsert
# (原 (user_id,trace_id) 复合唯一改为 trace_id 单列:harvest 帧0 建行时 user_id 可能暂缺。)
UniqueConstraint("trace_id", name="uq_comparison_trace"),
# 同一用户同一次比价(trace_id)只存一条:客户端重试/误点重复上报时幂等覆盖
UniqueConstraint("user_id", "trace_id", name="uq_comparison_user_trace"),
# 首页轮播 / 省钱战绩聚合都按 status='success' 过滤 + created_at 近期排序;
# 复合索引避免随数据量增大退化成全表扫(单列 created_at 索引不含 status)。
Index("ix_comparison_status_created", "status", "created_at"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
# 后端 harvest 在帧0(pricebot 出 trace_id)即建行,软鉴权下 user_id 可能暂缺(老客户端/匿名)→ 可空。
# C 端「我的比价记录」按 user_id 过滤天然排除 null-user 行;admin 全看(含孤儿行)。
user_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("user.id"), index=True, nullable=True
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("user.id"), index=True, nullable=False
)
# 仍记录设备号(同一用户多设备的行为区分 / 与不鉴权期 device_id 数据对账)
device_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -86,10 +83,6 @@ class ComparisonRecord(Base):
# ===== 订单概要 =====
store_name: Mapped[str | None] = mapped_column(String(128), nullable=True)
# 下单商品名拼接串(顿号分隔,从 items[].name 去重派生),供 admin 列表「商品」列展示 + 商品名搜索。
# items 是 JSON(SQLite 下 json.dumps ensure_ascii 把中文转义,无法直接 CAST+LIKE),故另派生成普通
# 文本列——跨库 LIKE 一致、可加索引。写路径(upsert_record / harvest_done)落库时同步派生。
product_names: Mapped[str | None] = mapped_column(String(512), nullable=True)
total_dish_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
skipped_dish_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
+3 -239
View File
@@ -24,23 +24,6 @@ def _yuan_to_cents(yuan: float | None) -> int | None:
return round(yuan * 100)
def _product_names_from_items(items: list | None) -> str | None:
"""下单商品 items([{name, qty, specs?}])→ 顿号分隔的商品名串(去重保序),
供 admin「商品」列展示 + 商品名 LIKE 搜索。空 / 无名 → None;超列宽(512)截断留余量。"""
if not items:
return None
names: list[str] = []
for it in items:
name = it.get("name") if isinstance(it, dict) else None
if not name:
continue
s = str(name).strip()
if s and s not in names:
names.append(s)
joined = "".join(names)
return joined[:500] or None
def _derive(payload: ComparisonRecordIn) -> dict:
"""从上报 payload 派生结构化列(best/saved/is_source_best/status)。"""
results = payload.comparison_results
@@ -91,19 +74,12 @@ def _derive(payload: ComparisonRecordIn) -> dict:
def upsert_record(
db: Session, *, user_id: int, payload: ComparisonRecordIn
) -> ComparisonRecord:
"""**trace_id** 幂等写入(唯一键已从 user_id+trace_id 改为 trace_id):已存在则合并
(回填 null user_id + 覆盖字段,但**不降级 success**),否则新建。
灰度期老客户端 POST /compare/record 走这条,与后端 harvest 按 trace_id reconcile;
新客户端不再 POST(改由 compare.py 透传壳 harvest 落库)。
"""
"""(user_id, trace_id) 幂等写入:已存在则覆盖(更完整的重试上报胜出),否则新建。"""
derived = _derive(payload)
items = [it.model_dump(exclude_none=True) for it in payload.items]
fields = dict(
device_id=payload.device_id,
business_type=payload.business_type,
store_name=payload.store_name,
product_names=_product_names_from_items(items),
source_platform_id=payload.source_platform_id,
source_platform_name=payload.source_platform_name,
source_package=payload.source_package,
@@ -112,7 +88,7 @@ def upsert_record(
trace_url=payload.trace_url,
total_dish_count=payload.total_dish_count,
skipped_dish_count=payload.skipped_dish_count,
items=items,
items=[it.model_dump(exclude_none=True) for it in payload.items],
comparison_results=[r.model_dump() for r in payload.comparison_results],
skipped_dish_names=list(payload.skipped_dish_names),
# 客户端环境 / 性能(debug,客户端上报;旧客户端为 None)
@@ -134,29 +110,14 @@ def upsert_record(
**derived,
)
# 按 trace_id 定位(唯一键已改 trace_id):后端 harvest 可能已建行,这里的客户端上报
# (灰度期老客户端 / 新客户端不再走这条)与之 reconcile。
existing = db.execute(
select(ComparisonRecord).where(
ComparisonRecord.user_id == user_id,
ComparisonRecord.trace_id == payload.trace_id,
)
).scalar_one_or_none()
if existing is not None:
# 不降级:harvest 或更早上报已落成 success,收尾期 fromFailure 的 cancelled/failed
# 不许把它盖回去(老 comparisonReported bug 的服务端兜底)——只补 user_id / trace_url。
if existing.status == "success" and fields.get("status") != "success":
if existing.user_id is None and user_id is not None:
existing.user_id = user_id
if fields.get("trace_url"):
existing.trace_url = fields["trace_url"]
db.commit()
db.refresh(existing)
return existing
# 只**填**空缺 user_id、不 reassign:trace_id 全局唯一,一条 trace 只属一个用户;
# 绝不把已有归属的记录改判给另一个上报者(仅 harvest 建的 null-user 行在此绑上)。
if existing.user_id is None:
existing.user_id = user_id
for k, v in fields.items():
setattr(existing, k, v)
db.commit()
@@ -178,203 +139,6 @@ def upsert_record(
return rec
# ============================================================
# 后端 harvestapp-server 从 pricebot 透传响应里直接落库(不靠客户端上报)。
# 帧0 建行(running) → 最终 done 更新(success/failed) → finalize 更新(aborted)。
# 全按 trace_id upsertsuccess 行永不被后到的 failed/aborted 降级。
# ============================================================
def _derive_from_results(results: list[dict]) -> dict:
"""从 done 帧 comparison_results(pricebot 原始 dict 列表)派生结构化列。
等价 _derive,但吃原始字段(is_source/price/rank/platform_id/store_name...)而非 pydantic 对象。"""
priced = [r for r in results if r.get("price") is not None]
best = None
if priced:
best = min(
priced,
key=lambda r: (r.get("rank") if r.get("rank") is not None else 10**9, r["price"]),
)
src_row = next((r for r in results if r.get("is_source")), None)
source_price_cents = None
if src_row is not None and src_row.get("price") is not None:
source_price_cents = _yuan_to_cents(src_row["price"])
best_price_cents = _yuan_to_cents(best["price"]) if best else None
saved_amount_cents = None
if source_price_cents is not None and best_price_cents is not None:
saved_amount_cents = source_price_cents - best_price_cents
has_valid_target = any(
(not r.get("is_source")) and r.get("price") is not None for r in results
)
return {
"source_platform_id": (src_row or {}).get("platform_id"),
"source_platform_name": (src_row or {}).get("platform_name"),
"source_package": (src_row or {}).get("package"),
"source_price_cents": source_price_cents,
"best_platform_id": best.get("platform_id") if best else None,
"best_platform_name": best.get("platform_name") if best else None,
"best_price_cents": best_price_cents,
"saved_amount_cents": saved_amount_cents,
"is_source_best": best.get("is_source") if best else None,
"store_name": (src_row or {}).get("store_name") or None,
"status": "success" if has_valid_target else "failed",
}
def _device_cols_from_info(device_info: dict | None) -> dict:
"""step 帧 device_info({locale,brand,model,android_version,rom_version})→ 表列。
step 帧只带这几项;rom_name / android_sdk / app_version / 耗时步数 harvest 拿不到 → 留 None
(灰度期老客户端 fromComparison 会补齐;要新客户端也全带需扩 collectDeviceInfo,后续)。"""
d = device_info or {}
rv = str(d.get("rom_version") or "")
return {
"device_model": d.get("model") or None,
"device_manufacturer": d.get("brand") or None,
"android_version": d.get("android_version") or None,
"rom_version": int(rv) if rv.isdigit() else None,
}
def _get_by_trace(db: Session, trace_id: str) -> ComparisonRecord | None:
return db.execute(
select(ComparisonRecord).where(ComparisonRecord.trace_id == trace_id)
).scalar_one_or_none()
def harvest_running(
db: Session,
*,
trace_id: str,
user_id: int | None,
business_type: str = "food",
device_id: str | None = None,
device_info: dict | None = None,
trace_url: str | None = None,
) -> ComparisonRecord:
"""帧0(或任一尚未建行的帧)建/补 running 行。幂等:已存在只补空缺(user_id/trace_url/
device_id/机型),绝不动已落定的 status / 结果。"""
rec = _get_by_trace(db, trace_id)
if rec is None:
rec = ComparisonRecord(
trace_id=trace_id,
user_id=user_id,
business_type=business_type or "food",
device_id=device_id,
status="running",
trace_url=trace_url,
created_at=datetime.now(CN_TZ).replace(tzinfo=None),
**_device_cols_from_info(device_info),
)
db.add(rec)
db.commit()
db.refresh(rec)
return rec
changed = False
if rec.user_id is None and user_id is not None:
rec.user_id = user_id
changed = True
if not rec.trace_url and trace_url:
rec.trace_url = trace_url
changed = True
if rec.device_id is None and device_id:
rec.device_id = device_id
changed = True
for k, v in _device_cols_from_info(device_info).items():
if getattr(rec, k) is None and v is not None:
setattr(rec, k, v)
changed = True
if changed:
db.commit()
db.refresh(rec)
return rec
def harvest_done(
db: Session,
*,
trace_id: str,
user_id: int | None,
done_params: dict,
business_type: str = "food",
device_id: str | None = None,
device_info: dict | None = None,
trace_url: str | None = None,
) -> tuple[ComparisonRecord, bool]:
"""最终 done 帧:running 行 → 终态(success/failed)+结果+trace_url。
返回 (记录, 是否本次**新**落成 success)——供调用方据此幂等发一次邀请奖。
行不存在(理论上帧0已建;防御)则新建。"""
results = done_params.get("comparison_results") or []
derived = _derive_from_results(results)
# 菜品:pricebot 已把源单菜品塞进 comparison_results[源行].items
items = next((r.get("items") or [] for r in results if r.get("is_source")), [])
fields = dict(
business_type=business_type or "food",
information=done_params.get("information") or None,
# best_deeplink 来自客户端剪贴板采集,harvest 拿不到 → 留空(灰度期 fromComparison 会补;
# 纯 harvest 行「再次比价」退化为按 package 拉起 App。要精确深链需客户端另传,后续)。
trace_url=trace_url or done_params.get("trace_url"),
total_dish_count=done_params.get("total_dish_count"),
skipped_dish_count=done_params.get("skipped_dish_count"),
skipped_dish_names=list(done_params.get("skipped_dish_names") or []),
comparison_results=results,
items=items,
product_names=_product_names_from_items(items),
raw_payload=done_params,
**derived,
**_device_cols_from_info(device_info),
)
rec = _get_by_trace(db, trace_id)
was_success = rec is not None and rec.status == "success"
if rec is None:
rec = ComparisonRecord(
trace_id=trace_id,
user_id=user_id,
created_at=datetime.now(CN_TZ).replace(tzinfo=None),
device_id=device_id,
**fields,
)
db.add(rec)
else:
if user_id is not None and rec.user_id is None:
rec.user_id = user_id
if device_id and rec.device_id is None:
rec.device_id = device_id
for k, v in fields.items():
setattr(rec, k, v)
db.commit()
db.refresh(rec)
newly_success = (rec.status == "success") and not was_success
return rec, newly_success
def harvest_abort(
db: Session,
*,
trace_id: str,
status: str,
reason: str | None,
trace_url: str | None = None,
) -> ComparisonRecord | None:
"""finalize(用户终止/超时/异常,无 done 帧):running 行 → aborted/failed + trace_url。
**不降级 success**:行已 success(收尾取消那种 finalize 后到)只 refresh trace_url。
行不存在(极少:帧0没建成)→ 返回 None,不凭空造。"""
rec = _get_by_trace(db, trace_id)
if rec is None:
return None
if trace_url and not rec.trace_url:
rec.trace_url = trace_url
if rec.status != "success":
rec.status = status or "cancelled"
if reason:
rec.information = reason
db.commit()
db.refresh(rec)
return rec
def _ordered_shop_names(db: Session, user_id: int) -> set[str]:
"""该用户「真实下单」(source='compare')覆盖到的店名集合,用来给比价记录打「已下单」。
-3
View File
@@ -304,9 +304,6 @@ def get_invitees(
"avatar_url": u.avatar_url or u.wechat_avatar_url or None,
"coins": rel.inviter_coin, # v3 起恒 0(邀请人收益改走邀请奖励金)
"invited_at": rel.created_at,
# 是否已完成过一次比价(= 已发过邀请奖励金)。客户端据此:好友列表分"去提醒/邀请成功"、
# 在途列表只取未比价(is_compared=False)、算在途好友数与在途收益(未比价数×2元)。
"is_compared": bool(rel.compare_reward_granted),
})
has_more = offset + len(rows) < int(total)
return items, int(total), has_more
+1 -16
View File
@@ -5,7 +5,7 @@ device_id 为空(老客户端 / 取不到 ANDROID_ID)一律按"未完成"处理,
"""
from __future__ import annotations
from sqlalchemy import delete, select
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -33,18 +33,3 @@ def mark_completed(db: Session, *, user_id: int, device_id: str) -> None:
except IntegrityError:
# 并发 / 重复提交撞唯一约束:已有行即视为成功。
db.rollback()
def delete_completion(db: Session, *, user_id: int, device_id: str) -> int:
"""删该 (账号, 设备) 的引导完成标记 → 下次登录 is_completed=False → 客户端重走。
与 [mark_completed] 互逆。device_id 为空忽略(返 0);无记录也幂等(返 0)。返回删除行数。"""
if not device_id:
return 0
result = db.execute(
delete(OnboardingCompletion).where(
OnboardingCompletion.user_id == user_id,
OnboardingCompletion.device_id == device_id,
)
)
db.commit()
return result.rowcount
+39 -56
View File
@@ -29,17 +29,6 @@ from app.core.rewards import CN_TZ
from app.models.comparison import ComparisonRecord
from app.models.ops_marquee_seed import OpsMarqueeSeed
from app.models.user import User
from app.repositories import app_config
# 首页轮播数据源模式(存 app_config.marquee_feed_mode):
# mixed=真实优先+种子补位+合成兜底(默认,原行为);real=只真实(不足则少/空);seed=只种子+合成兜底。
FEED_MODES = ("mixed", "real", "seed")
def get_feed_mode(db: Session) -> str:
"""读首页轮播数据源模式;非法/未配置回退 mixed(= 原行为)。"""
mode = app_config.get_value(db, "marquee_feed_mode")
return mode if mode in FEED_MODES else "mixed"
# feed 运行时随机源(每次请求结果不同 = 轮播想要的「鲜活感」)
_rng = random.Random()
@@ -223,56 +212,50 @@ def get_feed(db: Session, limit: int = 8) -> list[dict]:
展示时间统一「刷新」成相对现在的最近时刻(从 now 往前**随机抖动**递减),保证轮播永远像刚发生、
节奏自然不机械(真实用户/金额不变,只换展示时间——避免旧测试数据 / 低谷期记录显示成过时时间)。
"""
mode = get_feed_mode(db) # mixed / real / seed(运营可在「首页轮播种子」页切换)
# 真实条:取较多近期记录(带 ~30s 缓存)后按 user 去重;金额超上限的异常值已在查询剔除。
rows = _recent_real_rows(db)
items: list[dict] = []
used_names: set[str] = set()
seen_users: set[int] = set()
for uid, sc, nick in rows:
if uid in seen_users:
continue
seen_users.add(uid)
name = _mask_real(nick, uid)
used_names.add(name) # 真实名按昵称/id 稳定;偶发撞名可接受
items.append({"masked_user": name, "saved_amount_cents": int(sc)})
if len(items) >= limit:
break
# 真实条(mixed / real):取较多近期记录(带 ~30s 缓存)后按 user 去重;金额超上限的异常值已在查询剔除。
if mode != "seed":
rows = _recent_real_rows(db)
seen_users: set[int] = set()
for uid, sc, nick in rows:
if uid in seen_users:
continue
seen_users.add(uid)
name = _mask_real(nick, uid)
used_names.add(name) # 真实名按昵称/id 稳定;偶发撞名可接受
items.append({"masked_user": name, "saved_amount_cents": int(sc)})
if len(items) >= limit:
break
# 种子补位 + 合成兜底(mixed / seed):mixed 下补真实不足的部分,seed 下全量用种子/合成。
# real 模式**跳过**——只出真实,不掺任何假数据(真实不足则少于 limit,为 0 时返回空)。
if mode != "real":
need = limit - len(items)
if need > 0:
seeds = db.execute(
select(OpsMarqueeSeed).where(OpsMarqueeSeed.enabled.is_(True))
).scalars().all()
# 公平随机抽取 need 个(池子够大则不放回抽样;否则全用并打散),让所有启用种子都有机会露出。
chosen = _rng.sample(seeds, need) if len(seeds) > need else list(seeds)
_rng.shuffle(chosen)
for s in chosen:
# 用户名:旧的「用户****xxx」统一模板名 / 留空 → 一律走新混合合成(避开同屏撞名、自愈历史种子);
# 仅运营手动设的非模板真名才原样用。
fixed = (s.masked_user or "").strip()
name = fixed if fixed and not fixed.startswith("用户****") else _unique_name(used_names)
used_names.add(name)
# 金额:在 [min,max] 取长尾随机值(小额居多、偶尔大额;固定金额则 min==max)。
lo = max(0, int(s.min_cents))
hi = max(lo, int(s.max_cents))
items.append({"masked_user": name, "saved_amount_cents": _skewed_amount(lo, hi)})
# 兜底:真实 + 种子仍凑不满 limit(种子被全停用 / 数量太少)→ 用内置合成补满,
# 保证轮播既不空也不稀疏(稀疏的几条循环同样像假)。
while len(items) < limit:
name = _unique_name(used_names)
need = limit - len(items)
if need > 0:
seeds = db.execute(
select(OpsMarqueeSeed).where(OpsMarqueeSeed.enabled.is_(True))
).scalars().all()
# 公平随机抽取 need 个(池子够大则不放回抽样;否则全用并打散),让所有启用种子都有机会露出。
chosen = _rng.sample(seeds, need) if len(seeds) > need else list(seeds)
_rng.shuffle(chosen)
for s in chosen:
# 用户名:旧的「用户****xxx」统一模板名 / 留空 → 一律走新混合合成(避开同屏撞名、自愈历史种子);
# 仅运营手动设的非模板真名才原样用。
fixed = (s.masked_user or "").strip()
name = fixed if fixed and not fixed.startswith("用户****") else _unique_name(used_names)
used_names.add(name)
items.append({
"masked_user": name,
"saved_amount_cents": _skewed_amount(_FALLBACK_MIN_CENTS, _FALLBACK_MAX_CENTS),
})
# 金额:在 [min,max] 取长尾随机值(小额居多、偶尔大额;固定金额则 min==max)。
lo = max(0, int(s.min_cents))
hi = max(lo, int(s.max_cents))
items.append({"masked_user": name, "saved_amount_cents": _skewed_amount(lo, hi)})
# 兜底:真实 + 种子仍凑不满 limit(种子被全停用 / 数量太少)→ 用内置合成补满,
# 保证轮播既不空也不稀疏(稀疏的几条循环同样像假)。
while len(items) < limit:
name = _unique_name(used_names)
used_names.add(name)
items.append({
"masked_user": name,
"saved_amount_cents": _skewed_amount(_FALLBACK_MIN_CENTS, _FALLBACK_MAX_CENTS),
})
# 统一赋「最近」时间:从 now 往前**随机抖动**递减,避免固定节奏被看出规律。
# 首条几十秒前;其余多数 1~5 分钟,偶尔扎堆(20~55s)或较长(5~9 分钟),降序、像真实流水。
+2 -5
View File
@@ -318,14 +318,11 @@ def update_config(
row.random_current = _monotonic(row, random_initial)
row.random_last_tick_at = now
elif apply_now:
# 立即更新:不等钟点,马上刷新一次
# random:走一档增长;real/manual:直接落到「配置目标值」(manual_value / max(真实,保底)),
# **显式保存即所见即所得,绕过「只增不减」护栏**——运营手动改值就是要按配置显示(含调小)。
# (护栏仍作用于自动 tick 的 _refresh:防门面在两次保存之间被真实刷新/自增长悄悄缩水。)
# 立即更新:不等钟点,马上刷新一次
if row.mode == "random" and row.random_current is not None:
row.random_current = _grow(row, row.random_current, 1)
else:
row.random_current = _current_target(db, row)
row.random_current = _monotonic(row, _current_target(db, row))
row.random_last_tick_at = now
elif row.random_current is None:
# 首次无值:按当前模式播种,使配置后立即有合理展示值
-3
View File
@@ -296,9 +296,6 @@ def daily_auto_exchange(db: Session) -> dict:
- **逐用户独立事务**:单个用户异常 rollback 不影响其他人;exchange_coins_to_cash 内部按用户 commit。
返回统计 dict(scanned/converted/skipped_done/skipped_dust/failed/total_cents)。
"""
# ⚠️「今天」写死北京时(rewards.cn_today() = datetime.now(CN_TZ).date(), CN_TZ=+8),与服务器/用户
# 时区无关(用户 2026-07-01 硬约束:兑换一律北京 0 点日切)。**禁止**改成 date.today() / 裸
# datetime.now().date()——那会跟随进程本地时区,服务器非 CST 时会在错误的"天"兑/重复兑。
today = rewards.cn_today()
stats = {
"scanned": 0, "converted": 0, "skipped_done": 0,
-1
View File
@@ -69,7 +69,6 @@ class InviteeItem(BaseModel):
avatar_url: str | None = None # 头像 URL;null = 前端画默认色块
coins: int # 这次邀请给我(邀请人)发的金币
invited_at: datetime # 邀请绑定时间(前端转"今天/3天前")
is_compared: bool = False # 该好友是否已完成过一次比价(好友列表分"去提醒/邀请成功";在途列表只取 False)
class InviteeListOut(BaseModel):
-3
View File
@@ -19,9 +19,6 @@ Type=oneshot
User=root
WorkingDirectory=/opt/shaguabijia-app-server
Environment="PATH=/opt/shaguabijia-app-server/.venv/bin:/usr/bin:/bin"
# 写死北京时:兑换的"当天/0 点"一律按北京时,不随服务器 OS 时区漂(用户 2026-07-01 硬约束)。
# 脚本内的日期判断本就走 rewards.cn_today()(CN_TZ=+8),这里再把进程 TZ 也钉成北京,双保险。
Environment="TZ=Asia/Shanghai"
EnvironmentFile=/opt/shaguabijia-app-server/.env
ExecStart=/opt/shaguabijia-app-server/.venv/bin/python -m scripts.daily_auto_exchange --once
SyslogIdentifier=daily-exchange
+2 -4
View File
@@ -4,10 +4,8 @@
Description=Run daily auto-exchange coins->cash at midnight
[Timer]
# 每天【北京时】0 点跑,写死时区(用户 2026-07-01 硬约束:兑换一律北京 0 点,不随服务器本地时区漂)
# systemd OnCalendar 支持尾缀时区;不写时区会按服务器 OS 本地时区触发 → 服务器非 CST 时会在错误时刻兑。
# 客户端文案已注明「可能存在延迟」,可按需改 00:05 错开整点扎堆。
OnCalendar=*-*-* 00:00:00 Asia/Shanghai
# 每天 0 点跑。客户端文案已注明「可能存在延迟」,可按需改 00:05 错开整点扎堆
OnCalendar=*-*-* 00:00:00
# 服务器宕机/重启后,补跑错过的那一轮(而不是干等次日)。
Persistent=true
AccuracySec=1min
+47
View File
@@ -0,0 +1,47 @@
# OpenObserve 监控台反代(单独子域名)。前提:
# - DNS: observe.shaguabijia.com A 记录 → 本机公网 IP
# - 证书: /etc/nginx/ssl/observe.shaguabijia.com.pem|.key(同现有域名放法)
# - OpenObserve 只绑 127.0.0.1:5080(见 docker-compose.prod.yml)
# 三重防护:IP 白名单 + nginx TLS + OpenObserve 自身登录。
# 只你/少数人看的话,更省事的是 SSH 隧道(README「UI 访问」方案 A),不用域名/证书。
# 80 → 301 → 443
server {
listen 80;
listen [::]:80;
server_name observe.shaguabijia.com;
return 301 https://$host$request_uri;
}
# 443 SSL 反代到本地 OpenObserve (127.0.0.1:5080)
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name observe.shaguabijia.com;
ssl_certificate /etc/nginx/ssl/observe.shaguabijia.com.pem;
ssl_certificate_key /etc/nginx/ssl/observe.shaguabijia.com.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_session_cache shared:SSL:10m;
# ★ 只放行你的固定出口 IP(办公/家宽)。监控台不必对全网开。
allow 1.2.3.4; # ← 换成真实 IP,可多行
deny all;
client_max_body_size 10m;
location / {
proxy_pass http://127.0.0.1:5080;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# OpenObserve 有实时/流式,需透传 WebSocket
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 300s;
}
}
+4
View File
@@ -0,0 +1,4 @@
# OpenObserve 落盘数据(parquet/索引/元数据),运行时产生,不入库。
data/
# 生产 compose 的密码文件(OO_ROOT_PASSWORD),含机密,不入库。
.env
+126
View File
@@ -0,0 +1,126 @@
# OpenObserve 本地部署(接口 QPS / 耗时可观测)
app-server 通过中间件采集每个接口的 QPS + 耗时 + 错误率,批量上报到这里。
设计见 [../../docs/superpowers/specs/2026-07-06-openobserve-api-metrics-design.md](../../docs/superpowers/specs/2026-07-06-openobserve-api-metrics-design.md)。
## 启动
```bash
cd deploy/openobserve
docker compose up -d
```
- Web UIhttp://localhost:5080
- 登录:`admin@shaguabijia.local` / `Complexpass#123`(见 `docker-compose.yml`
- 数据落 `deploy/openobserve/data/`(已挂卷持久化;该目录已 gitignore)
## 让 app-server 上报
在项目根的 `.env` 打开观测(`OBSERVE_*`,账号密码与 compose 里 root 一致):
```dotenv
OBSERVE_ENABLED=true
OBSERVE_ENDPOINT=http://localhost:5080
OBSERVE_ORG=default
OBSERVE_STREAM=app_requests
OBSERVE_USER=admin@shaguabijia.local
OBSERVE_PASSWORD=Complexpass#123
```
重启 app-server,随便打几个接口。stream `app_requests` **首次上报自动创建**
在 UI 的 Logs → 选 `app_requests` 就能看到逐条请求事件(字段:`method` / `route` /
`status` / `duration_ms` / `service` / `env`)。
> 未开 `OBSERVE_ENABLED` 或缺账号密码时,中间件透传、worker 不启动,整套 no-op,不影响业务。
## 查询(Logs 页 SQL,或建 Dashboard 面板)
各接口 QPS(1 分钟分桶,面板里再除 60 得每秒):
```sql
SELECT route, histogram(_timestamp, '1 minute') AS ts, count(*) AS cnt
FROM app_requests GROUP BY route, ts ORDER BY ts
```
各接口 P95 耗时(毫秒):
```sql
SELECT route, approx_percentile_cont(duration_ms, 0.95) AS p95_ms
FROM app_requests GROUP BY route ORDER BY p95_ms DESC
```
各接口错误率(5xx 占比):
```sql
SELECT route,
count(*) FILTER (WHERE status >= 500) * 100.0 / count(*) AS err_pct
FROM app_requests GROUP BY route ORDER BY err_pct DESC
```
## 一键导入现成仪表盘(QPS / P95 / 分位 / 错误率)
备好了 [dashboard-api-metrics.json](dashboard-api-metrics.json),4 个面板:各接口每分钟请求数(QPS 源)、
P95 耗时折线、P50/P95/P99 分位表、5xx 错误率表。
- **UI 导入**Dashboards → 右上 **Import** → 选该 JSON 文件 → Import(每次导入新建,不覆盖)。
- **或 API 导入**
```bash
curl -u admin@shaguabijia.local:Complexpass#123 -H 'Content-Type: application/json' \
-X POST 'http://localhost:5080/api/default/dashboards?folder=default' \
--data-binary @deploy/openobserve/dashboard-api-metrics.json
```
导入后进仪表盘,右上角时间调到「最近 15 分钟 / 1 小时」、开自动刷新即可。低流量下 QPS 面板看「每分钟请求数」比「每秒」直观。
## 停止 / 清数据
```bash
docker compose down # 停止(保留数据)
docker compose down -v && rm -rf data # 停止并清空数据
```
## 生产部署(单机)+ UI 访问
前提:app-server 与 OpenObserve **同机**,app→OO 走 localhost(`127.0.0.1:5080`)、不出网、无需 TLS。
唯一要防的是**别把 :5080 裸暴露公网**。硬化版编排见 [docker-compose.prod.yml](docker-compose.prod.yml)。
### 部署步骤
```bash
# 1) 密码文件(本目录,已 gitignore)
echo "OO_ROOT_PASSWORD=$(python -c 'import secrets;print(secrets.token_urlsafe(24))')" > deploy/openobserve/.env
# 2) 起 OpenObserve(只绑 127.0.0.1、命名卷持久化、mem 1g)
cd deploy/openobserve && docker compose -f docker-compose.prod.yml up -d
sudo systemctl enable docker # 开机自起
```
3) app-server 的 `.env` 打开观测并**重启**(用非 root 的专用 ingest 账号):
```dotenv
OBSERVE_ENABLED=true
OBSERVE_ENDPOINT=http://127.0.0.1:5080
OBSERVE_ORG=default
OBSERVE_STREAM=app_requests
OBSERVE_USER=ingest@shaguabijia.com # UI → Users 建的非 root 账号
OBSERVE_PASSWORD=<该账号密码>
```
```bash
sudo systemctl restart shaguabijia-app-server # 日志出现 "observe worker started" 即生效
```
4) 两个必做收口(磁盘/安全):
- **保留期**:UI → Streams → `app_requests` → Data Retention 设 14/30 天(一请求一行,不封顶迟早撑爆盘)。
- **专用账号**:UI → Users 建非 root 账号给 app 上报,root 只留人工登 UI。
### UI 访问(二选一)
**A. SSH 隧道(推荐,零暴露、不用域名/证书):**
```bash
ssh -L 5080:127.0.0.1:5080 用户@服务器IP
# 然后本机浏览器开 http://localhost:5080
```
**B. nginx 子域名反代(要固定 URL / 团队常看):** 见 [../nginx/observe.shaguabijia.com.conf](../nginx/observe.shaguabijia.com.conf)。
需 DNS `observe.shaguabijia.com` → 本机 + 证书放 `/etc/nginx/ssl/`;含 IP 白名单 + TLS + WebSocket 透传。
> ⚠️ prod compose 必须保持 `127.0.0.1:5080:5080`;写成 `5080:5080`(绑 0.0.0.0)= 裸暴露公网,这是唯一真正的坑。
@@ -0,0 +1,301 @@
{
"version": 8,
"title": "接口监控 (QPS / 耗时 / 错误率)",
"description": "app-server 接口 QPS、P50/P95/P99 耗时、5xx 错误率。数据流 app_requests。",
"role": "",
"tabs": [
{
"tabId": "default",
"name": "Default",
"panels": [
{
"id": "panel_qps",
"type": "line",
"title": "各接口 每分钟请求数 (QPS 源)",
"description": "",
"config": {
"show_legends": true,
"legends_position": null,
"decimals": 2.0,
"axis_border_show": false,
"base_map": null,
"map_view": null
},
"queryType": "sql",
"queries": [
{
"query": "SELECT histogram(_timestamp, '1 minute') as ts, route, count(*) as reqs FROM app_requests GROUP BY ts, route ORDER BY ts",
"vrlFunctionQuery": "",
"customQuery": true,
"fields": {
"stream": "app_requests",
"stream_type": "logs",
"x": [
{
"label": "ts",
"alias": "ts",
"column": "ts",
"color": null,
"sortBy": "ASC"
}
],
"y": [
{
"label": "reqs",
"alias": "reqs",
"column": "reqs",
"color": null
}
],
"z": [],
"breakdown": [
{
"label": "route",
"alias": "route",
"column": "route",
"color": null
}
],
"filter": {
"filterType": "group",
"logicalOperator": "AND",
"conditions": []
}
},
"config": {
"promql_legend": "",
"layer_type": "scatter",
"weight_fixed": 1.0
}
}
],
"layout": {
"x": 0,
"y": 0,
"w": 24,
"h": 9,
"i": 1
}
},
{
"id": "panel_p95",
"type": "line",
"title": "各接口 P95 耗时 (ms)",
"description": "",
"config": {
"show_legends": true,
"legends_position": null,
"decimals": 2.0,
"axis_border_show": false,
"base_map": null,
"map_view": null
},
"queryType": "sql",
"queries": [
{
"query": "SELECT histogram(_timestamp, '1 minute') as ts, route, approx_percentile_cont(duration_ms, 0.95) as p95_ms FROM app_requests GROUP BY ts, route ORDER BY ts",
"vrlFunctionQuery": "",
"customQuery": true,
"fields": {
"stream": "app_requests",
"stream_type": "logs",
"x": [
{
"label": "ts",
"alias": "ts",
"column": "ts",
"color": null,
"sortBy": "ASC"
}
],
"y": [
{
"label": "p95_ms",
"alias": "p95_ms",
"column": "p95_ms",
"color": null
}
],
"z": [],
"breakdown": [
{
"label": "route",
"alias": "route",
"column": "route",
"color": null
}
],
"filter": {
"filterType": "group",
"logicalOperator": "AND",
"conditions": []
}
},
"config": {
"promql_legend": "",
"layer_type": "scatter",
"weight_fixed": 1.0
}
}
],
"layout": {
"x": 24,
"y": 0,
"w": 24,
"h": 9,
"i": 2
}
},
{
"id": "panel_pctl",
"type": "table",
"title": "各接口 耗时分位 P50/P95/P99 (ms)",
"description": "",
"config": {
"show_legends": true,
"legends_position": null,
"decimals": 2.0,
"axis_border_show": false,
"base_map": null,
"map_view": null
},
"queryType": "sql",
"queries": [
{
"query": "SELECT route, approx_percentile_cont(duration_ms,0.5) as p50, approx_percentile_cont(duration_ms,0.95) as p95, approx_percentile_cont(duration_ms,0.99) as p99, count(*) as cnt FROM app_requests GROUP BY route ORDER BY p95 DESC",
"vrlFunctionQuery": "",
"customQuery": true,
"fields": {
"stream": "app_requests",
"stream_type": "logs",
"x": [
{
"label": "route",
"alias": "route",
"column": "route",
"color": null
}
],
"y": [
{
"label": "p50",
"alias": "p50",
"column": "p50",
"color": null
},
{
"label": "p95",
"alias": "p95",
"column": "p95",
"color": null
},
{
"label": "p99",
"alias": "p99",
"column": "p99",
"color": null
},
{
"label": "cnt",
"alias": "cnt",
"column": "cnt",
"color": null
}
],
"z": [],
"breakdown": [],
"filter": {
"filterType": "group",
"logicalOperator": "AND",
"conditions": []
}
},
"config": {
"promql_legend": "",
"layer_type": "scatter",
"weight_fixed": 1.0
}
}
],
"layout": {
"x": 0,
"y": 9,
"w": 24,
"h": 9,
"i": 3
}
},
{
"id": "panel_err",
"type": "table",
"title": "各接口 错误率 (5xx %)",
"description": "",
"config": {
"show_legends": true,
"legends_position": null,
"decimals": 2.0,
"axis_border_show": false,
"base_map": null,
"map_view": null
},
"queryType": "sql",
"queries": [
{
"query": "SELECT route, count(*) FILTER (WHERE status >= 500) * 100.0 / count(*) as err_pct, count(*) as cnt FROM app_requests GROUP BY route ORDER BY err_pct DESC",
"vrlFunctionQuery": "",
"customQuery": true,
"fields": {
"stream": "app_requests",
"stream_type": "logs",
"x": [
{
"label": "route",
"alias": "route",
"column": "route",
"color": null
}
],
"y": [
{
"label": "err_pct",
"alias": "err_pct",
"column": "err_pct",
"color": null
},
{
"label": "cnt",
"alias": "cnt",
"column": "cnt",
"color": null
}
],
"z": [],
"breakdown": [],
"filter": {
"filterType": "group",
"logicalOperator": "AND",
"conditions": []
}
},
"config": {
"promql_legend": "",
"layer_type": "scatter",
"weight_fixed": 1.0
}
}
],
"layout": {
"x": 24,
"y": 9,
"w": 24,
"h": 9,
"i": 4
}
}
]
}
],
"variables": {
"list": []
}
}
@@ -0,0 +1,27 @@
# 生产用 OpenObserve(单机)。相对本地版 docker-compose.yml 的区别:
# - 端口只绑 127.0.0.1 → 公网/外网都到不了(UI 访问走 SSH 隧道或 nginx 反代,见 README)
# - root 密码走环境变量(放同目录 .env,已 gitignore,勿提交)
# - 数据用 docker 命名卷(免 bind-mount 权限坑)+ 内存上限(与 app/PG 共存防抢内存)
#
# 用法:
# 1) 本目录建 .env(已 gitignore):
# OO_ROOT_PASSWORD=<强随机串> # 生成: python -c "import secrets;print(secrets.token_urlsafe(24))"
# 2) docker compose -f docker-compose.prod.yml up -d
# 3) 开机自起: sudo systemctl enable docker
services:
openobserve:
image: public.ecr.aws/zinclabs/openobserve:latest
container_name: openobserve
ports:
- "127.0.0.1:5080:5080" # ★只绑本机;写成 5080:5080 = 裸暴露公网,别这么干
environment:
ZO_ROOT_USER_EMAIL: "admin@shaguabijia.com"
ZO_ROOT_USER_PASSWORD: "${OO_ROOT_PASSWORD:?请先在 deploy/openobserve/.env 里设 OO_ROOT_PASSWORD}"
ZO_DATA_DIR: "/data"
volumes:
- openobserve_data:/data # docker 命名卷;想固定到某块盘改成 bind: /your/disk:/data
restart: unless-stopped
mem_limit: 1g # 按机器内存调;查询重可给 2g
volumes:
openobserve_data:
+16
View File
@@ -0,0 +1,16 @@
# 本地开发用 OpenObserve(单容器 = local 模式)。用于接收 app-server 的接口指标(QPS/耗时/错误率)。
# 启动: cd deploy/openobserve && docker compose up -d
# Web UI: http://localhost:5080 (账号见下方 env)
services:
openobserve:
image: public.ecr.aws/zinclabs/openobserve:latest
container_name: openobserve
ports:
- "5080:5080"
environment:
ZO_ROOT_USER_EMAIL: "admin@shaguabijia.local"
ZO_ROOT_USER_PASSWORD: "Complexpass#123"
ZO_DATA_DIR: "/data"
volumes:
- ./data:/data
restart: unless-stopped
+4 -4
View File
@@ -2,10 +2,10 @@
> 所属:比价记录组(前缀 `/api/v1/compare` | 鉴权:Bearer | [← 返回 API 索引](../README.md)
比价 `done` 帧后上报一条比价结果,落 `comparison_record` 表,作为「我的比价记录」数据源 + 用户级行为画像。
比价 `done` 帧后,客户端用**带 JWT 的通道**上报一条比价结果,落 `comparison_record` 表,作为「我的比价记录」数据源 + 用户级行为画像。
> ⚠️ **灰度定位(2026-07)**:写 `comparison_record` 现以透传壳 `compare.py` 的**后端 harvest** 为主(帧0 建 `running` 行 → done/finalize 落终态,新客户端不再 POST)。本接口降级为**老客户端兼容**通道,与 harvest 按 `trace_id` reconcile(**success 不被降级**);新版覆盖够高后可下线本 POST
> 与软鉴权透传端点 [`/api/v1/price/step`](./compare-price-step.md) 不同:那是转发壳(顺带 harvest 落库),本接口按用户维度显式上报,**必须鉴权**
> ⚠️ 与不鉴权的透传端点 [`/api/v1/price/step`](./compare-price-step.md) 不同:那是转发壳,本接口按用户维度落库,**必须鉴权**
> 本轮只做 server 端;客户端在 done 帧后调本接口的改动另起一轮(见 [待办与技术债.md](../guides/待办与技术债.md) P1
## 入参(JSON body
@@ -13,7 +13,7 @@
| 字段 | 类型 | 必填 | 默认 | 说明 |
|---|---|---|---|---|
| `trace_id` | string | ✅ | — | 一次比价的唯一标识(app-server 帧0 签发)。**幂等键(trace_id 单列唯一)**同 trace_id 重复上报覆盖、返回同一 id;与 harvest 行按它 reconcile |
| `trace_id` | string | ✅ | — | pricebot 侧 trace_id。**幂等键**:同用户同 trace_id 重复上报覆盖、返回同一 id |
| `business_type` | string | ❌ | `food` | `food`(外卖,当前唯一接通) / `ecom`(电商) / `coupon`(领券) |
| `device_id` | string \| null | ❌ | null | 设备号 |
| `store_name` | string \| null | ❌ | null | 店铺名(外卖,来自 calibration.result |
+6 -7
View File
@@ -2,13 +2,13 @@
> 模型 `app/models/comparison.py` · 仓库 `app/repositories/comparison.py` · 接口 [compare-record-report](../api/compare-record-report.md) / [compare-records](../api/compare-records.md) / [compare-record-detail](../api/compare-record-detail.md) · [← 索引](./README.md) · [总览](./OVERVIEW.md)
每完成一次比价(外卖/电商/领券)记一行。**写入以 app-server 后端 harvest 为主**(2026-07 起):比价透传壳 `compare.py` 在帧0(pricebot 出 trace_id)即建 `running` 行,随 done / `trace/finalize` 逐步补全成终态,客户端不再主动 POST 记录;老客户端仍可走**带 JWT** 的 `POST /compare/record` 兜底(灰度期两条写路径按 `trace_id` reconcile)。App「我的比价记录」列表/详情的数据源,也是比价战绩里程碑解锁进度的计数源(`status='success'` 条数),还被「上报更低价」反查原最低价。
每完成一次比价(外卖/电商/领券),客户端在 done 帧后用**带 JWT** 的通道上报一条。App「我的比价记录」列表/详情的数据源,也是比价战绩里程碑解锁进度的计数源(`status='success'` 条数),还被「上报更低价」反查原最低价。
> 与 `savings_record` 的区别:本表是「每一次**比价行为**的完整明细」(不省钱、甚至失败也记);`savings_record` 是「真正**下单成交**省了多少」。两表独立、互不喂数据。
> 与 [`price_observation`](./price_observation.md) / `store_mapping` 的区别:本表是**用户视角**(登录后按 `user_id` 存「我的比价记录」);后两张是 server 侧无条件沉淀的**平台/门店视角客观事实**(价格事实 / 跨平台店铺身份映射),与本表 `trace_id` 同源但不互相 join,各存各的视角。
## 用在哪 / 增删改查
- **C / U(harvest 为主,按 `trace_id` 幂等)**:透传壳 `compare.py` 三段式落库(`app/repositories/comparison.py`)——`harvest_running`(帧0 建 `running` 行)→ `harvest_done`(done 帧转 `success`/`failed` + 派生 `best_*`/`saved_amount_cents` + 返 `newly_success` 供幂等发邀请奖)→ `harvest_abort`(`trace/finalize``cancelled`/`failed`,**不降级已 success**)。老客户端仍可 `POST /compare/record`(`upsert_record`,按 `trace_id`、整行覆盖)兜底`best_*`/`saved_amount_cents`/`is_source_best`/`status` 一律`_derive``comparison_results` 算出(协议已按 price 升序、rank=1 最便宜),不信客户端自算。
- **C / U(upsert,幂等)**:`POST /compare/record`(`upsert_record`)。按 `(user_id, trace_id)`:不存在→新建;已存在→整行覆盖(客户端重试/重复上报时,**更完整的那次胜出**)`best_*`/`saved_amount_cents`/`is_source_best`/`status``_derive``comparison_results` 算出(协议已按 price 升序、rank=1 最便宜),不信客户端自算。
- **D**:无(关联的 `price_report` 也只把 `comparison_record_id` 置空,不删本表)。
- **R**:`GET /compare/records`(列表,`created_at` 倒序游标 + 「已下单」标记)、`GET /compare/records/{id}`(详情,限本人);`count_success` 给里程碑;`get_stats`(`status='success'` 计数 + `saved_amount_cents` 求和)给 [`GET /compare/stats`](../api/compare-stats.md) 喂「我的」页省钱战绩卡(完成比价 + 累计发现可省,**比价口径**);`report.py` 反查 `best_*`;admin 大盘/明细。
@@ -16,10 +16,10 @@
| 列 | 类型 | 约束 / 默认 | 说明(取值 / join) |
|---|---|---|---|
| `id` | Integer | PK, autoincrement | 被 `price_report.comparison_record_id` 引用 |
| `user_id` | Integer | FK→user.id, index, **nullable**(2026-07 从 NOT NULL 放开) | 归属用户。后端 harvest 帧0 建行时(软鉴权 / 老客户端匿名)可能暂缺 → 可空;C 端「我的比价记录」按 `user_id` 过滤天然排除 null 行,admin 全看(含孤儿行) |
| `user_id` | Integer | FK→user.id, index, NOT NULL | 归属用户 |
| `device_id` | String(64) | nullable | 设备号(多设备区分 / 与不鉴权期对账) |
| `business_type` | String(16) | NOT NULL, default `food`, index | 取值:`food`(当前唯一接通)/ `ecom` / `coupon` |
| `trace_id` | String(64) | NOT NULL, **UNIQUE** | 一次比价的唯一标识。**由 app-server 帧0 用 uuid 签发**(注入转发 body + 回填响应顶层给客户端;老客户端自带),全局唯一 = harvest upsert 键 + 关联 pricebot 调试落盘 |
| `trace_id` | String(64) | NOT NULL | pricebot 侧 trace_id(关联调试落盘 + 幂等键) |
| `trace_url` | String(512) | nullable | 本次比价公网调试链接(`price.shaguabijia.com/traces/{dir}/`);dir 名含 pricebot 落盘时分秒,前端/server 拼不出必须存。查看接口按 `user.debug_trace_enabled`(或本机 agent 调试 `include_trace`)决定返不返回。旧记录 / 未开上云为 null |
| `source_platform_id` / `_name` | String(32) | nullable | 源平台代号 / 中文名 |
| `source_package` | String(128) | nullable | 源平台 Android 包名 |
@@ -31,7 +31,7 @@
| `is_source_best` | Boolean | nullable | 源平台就是最便宜(= 这次没省到) |
| `store_name` | String(128) | nullable | 店铺名。**与 `savings_record.shop_name` 按字符串相等关联**,给本记录打「已下单」 |
| `total_dish_count` / `skipped_dish_count` | Integer | nullable | 菜品总数 / 目标平台没找到被跳过数 |
| `status` | String(16) | NOT NULL, default `success` | 取值:`running`(harvest 帧0 建行、比价进行中)/ `success`(有非源且有价的目标结果)/ `failed`(出错/没采到目标价)/ `cancelled`(用户终止 / Phase1 未识别,`harvest_abort` 写,**不降级已 success**)。**里程碑只数 success** |
| `status` | String(16) | NOT NULL, default `success` | 取值:`success`(有非源且有价的目标结果)/ `failed`(出错/没采到目标价)。**里程碑只数 success** |
| `information` | String(256) | nullable | done 帧文案;成功=摘要,失败=具体原因(前端失败时当原因展示) |
| `items` | JSON(PG: JSONB) | NOT NULL, default [] | 下单菜品 `[{name, qty, specs?}]` |
| `comparison_results` | JSON(PG: JSONB) | NOT NULL, default [] | 逐平台对比 `[{platform_id,platform_name,package,price(元),is_source,rank,coupon_saved(元),coupon_name,applied_coupons}]`;`coupon_saved`=该平台主优惠额(美团红包/淘宝平台红包/京东百亿补贴,只取一笔),`coupon_name`=优惠来源名(展示用),`applied_coupons`=`[{name,amount}]` 多券明细 |
@@ -50,8 +50,7 @@
- 被 `comparison_milestone_claim` 间接依赖:解锁进度 = 本表 `status='success'` 计数。
## 索引与约束
- PK `id`;index `user_id``business_type``created_at`;复合 index `ix_comparison_status_created`(`status`, `created_at`)(按 `status='success'` 过滤 + 近期排序的聚合/轮播,避免随数据量退化为全表扫);UNIQUE(`trace_id`) = `uq_comparison_trace`(harvest 按它 upsert,一次比价一行)。
> 2026-07 迁移 `comparison_record_trace_unique`:唯一键从复合 `uq_comparison_user_trace`(`user_id`,`trace_id`)改为 `trace_id` 单列——harvest 帧0 建行时 user_id 可能暂缺,不能再用复合键去重。上线前须确认历史无重复 `trace_id`(`SELECT trace_id,COUNT(*) c FROM comparison_record GROUP BY trace_id HAVING c>1`),否则建单列唯一会失败。
- PK `id`;index `user_id``business_type``created_at`;复合 index `ix_comparison_status_created`(`status`, `created_at`)(按 `status='success'` 过滤 + 近期排序的聚合/轮播,避免随数据量退化为全表扫);UNIQUE(`user_id`, `trace_id`) = `uq_comparison_user_trace`(幂等覆盖)。
## 注意
- 4 个 JSON 列用 `JSON().with_variant(JSONB(),"postgresql")`(SQLite 退化 JSON)。结构化金额列存「分」,`comparison_results.price`/`coupon_saved` 原样存「元」。
+9 -8
View File
@@ -16,9 +16,9 @@
## 函数 / 异常
| 函数 | 行为 |
|---|---|
| `send_code(phone) -> int` | 防刷检查(冷却)→ `secrets` 生成 N 位码 → **预占**(冷却/存码)→ mock 打日志 / real 调极光 → 失败**回滚预占**。返回距下次可发秒数 |
| `send_code(phone) -> int` | 防刷检查(冷却 + 每日上限)→ `secrets` 生成 N 位码 → **预占**(冷却/计数/存码)→ mock 打日志 / real 调极光 → 失败**回滚预占**。返回距下次可发秒数 |
| `verify_code(phone, code) -> bool` | mock 放行任意 6 位;real 比对存码,匹配即作废,失败累计到上限作废 |
| `SmsError(msg, status_code)` | `status_code` 决定 HTTP 码:过频 **429**、供应商失败 **503**、号码无效 **400** |
| `SmsError(msg, status_code)` | `status_code` 决定 HTTP 码:过频/每日超限 **429**、供应商失败 **503**、号码无效 **400** |
## 配置
| 配置项 | 默认 | 说明 |
@@ -30,16 +30,17 @@
| `SMS_SIGN_ID` | 31729 | 极光签名 ID |
| `SMS_TEMPLATE_ID` | 1 | 极光模板 ID(变量名 `code`) |
| `SMS_CODE_LENGTH` | 6 | 验证码位数(本服务生成;前端 code 4-8 位兼容) |
| `SMS_DAILY_LIMIT_PER_PHONE` | 10 | 单号每日发送上限(防刷 + 控费) |
| `SMS_MAX_VERIFY_ATTEMPTS` | 5 | 单码最多校验失败次数,超过作废(防爆破) |
> **鉴权复用极光一键登录**:`/v1/messages``base64(JG_APP_KEY:JG_MASTER_SECRET)` 做 HTTP Basic Auth——短信与一键登录是**同一个极光应用**(同 AppKey)。**上线不需要额外凭证,只需 `SMS_MOCK=false`**(`JG_*` 一键登录已配)。
## 防刷(短信花钱 + `/sms/send` 在登录前无法 JWT 鉴权)
> 2026-07-03 精简:登录风控只留「单号冷却 + 单设备频控」两道主策略(删单号每日上限、删登录纯 IP `rate_limit`);单码失败上限属验证码安全底线,保留。
1. 单号 `SMS_SEND_INTERVAL_SEC` 冷却(单号维度)
2. 单设备(`device_id` + IP)每小时频控:`/sms/send` `SMS_SEND_MAX_PER_HOUR_PER_DEVICE`(5)`/sms/login` `SMS_LOGIN_MAX_PER_HOUR`(5)——堵「换号绕开单号冷却」+ 挡登录撞库,在 `app/api/v1/auth.py``enforce_rate_limit`
3. 单码校验失败 `SMS_MAX_VERIFY_ATTEMPTS` 次即作废 + 验过即作废(一次性)
4. 运维侧建议在极光控制台叠加:**IP 白名单**(只许服务器 IP)+ **防轰炸设置**
1. 单号 `SMS_SEND_INTERVAL_SEC` 冷却
2. 单号每日 `SMS_DAILY_LIMIT_PER_PHONE` 条上限
3. 单 IP 频控:`/sms/send` `rate_limit(10,60)``/sms/login` `rate_limit(20,60)`
4. 单码校验失败 `SMS_MAX_VERIFY_ATTEMPTS` 次即作废
5. 运维侧建议在极光控制台叠加:**IP 白名单**(只许服务器 IP)+ **防轰炸设置**
## 极光错误码(节选,映射在 `_send_via_jiguang`)
| code | 含义 | 处理 |
@@ -55,4 +56,4 @@
3. 真机发一条验证:收到短信 + 能登录
## 已知局限
**验证码存进程内存**:单 worker uvicorn 够用;重启丢码(用户重发即可);**多 worker / 多机不共享 → 冷却 / 校验失效**,扩 worker 前迁移到 DB/Redis。见 [待办与技术债](../guides/待办与技术债.md)。
**验证码存进程内存**:单 worker uvicorn 够用;重启丢码(用户重发即可);**多 worker / 多机不共享 → 冷却 / 每日上限 / 校验失效**,扩 worker 前迁移到 DB/Redis。见 [待办与技术债](../guides/待办与技术债.md)。
@@ -0,0 +1,897 @@
# 接口 QPS + 耗时可观测(OpenObserve)实现计划
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** 给 app-server 每个接口采集 QPS + 耗时 + 错误率,经轻量 ASGI 中间件 + 后台 worker 批量直采到本地 Docker 的 OpenObserve。
**Architecture:** 纯 ASGI 中间件测每请求耗时/抓路由模板+状态码 → 非阻塞入有界队列(满则丢、绝不阻塞)→ 后台 asyncio worker 批量 POST 到 OpenObserve `_json` ingest 端点。请求路径零 I/O;未配置观测则整套 no-op;上报失败丢批不重试。
**Tech Stack:** FastAPI / Starlette ASGI 中间件、`asyncio.Queue``httpx.AsyncClient`(已有依赖)、pydantic-settings、OpenObserveDocker)。
参考 spec[docs/superpowers/specs/2026-07-06-openobserve-api-metrics-design.md](2026-07-06-openobserve-api-metrics-design.md)
---
## 文件结构
| 文件 | 职责 |
|---|---|
| `app/core/config.py`(改) | 新增 `OBSERVE_*` 配置 + `observe_configured` 门槛属性 |
| `app/core/observe.py`(新) | 有界事件队列 + `record_event` + 路由模板解析 + `RequestMetricsMiddleware` |
| `app/core/observe_worker.py`(新) | 后台批量上报 worker:`_collect_batch` / `_post_batch` / `start_*` / `stop_*` |
| `app/main.py`(改) | 挂中间件(最外层)+ lifespan 启停 worker |
| `.env.example`(改) | 新增 `OBSERVE_*` 注释段 |
| `deploy/openobserve/docker-compose.yml`(新) | 本地 OpenObserve 容器 |
| `deploy/openobserve/README.md`(新) | 部署步骤 + 查询/仪表盘 SQL |
| `tests/test_observe.py`(新) | 配置门槛 / 队列 / 中间件 / worker 单测 |
**关键接口契约(跨任务一致,勿改名):**
- `app.core.observe.get_queue() -> asyncio.Queue[dict]`
- `app.core.observe.record_event(event: dict) -> None`
- `app.core.observe.take_dropped() -> int`
- `app.core.observe.RequestMetricsMiddleware`ASGI class`__init__(self, app)`
- 事件字段:`_timestamp`(µs int) / `service` / `env` / `method` / `route` / `status` / `duration_ms`(float)
- `app.core.observe_worker.start_observe_worker() -> asyncio.Task | None`
- `app.core.observe_worker.stop_observe_worker(task) -> None`
- `settings.observe_configured -> bool`
---
## Task 1: 配置项 `OBSERVE_*` + `observe_configured`
**Files:**
- Modify: `app/core/config.py`(在 `cors_origins_list` property 之后、`is_prod` property 之前插入)
- Test: `tests/test_observe.py`(新建)
- [ ] **Step 1: 写失败测试**
新建 `tests/test_observe.py`
```python
"""接口指标可观测(observe)单测:配置门槛 / 队列 / 中间件 / worker。
沿用仓库约定:TestClient + monkeypatch,绝不打真网络。observe 默认关(conftest 未设
OBSERVE_*),需要开启的用例用 monkeypatch 改 settings 单例属性。
"""
from __future__ import annotations
from app.core.config import settings
def test_observe_configured_requires_switch_and_creds(monkeypatch):
# 开关开 + endpoint(默认 localhost)+ user + password 齐全 → True
monkeypatch.setattr(settings, "OBSERVE_ENABLED", True)
monkeypatch.setattr(settings, "OBSERVE_USER", "u")
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "p")
assert settings.observe_configured is True
# 缺密码 → False
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "")
assert settings.observe_configured is False
# 开关关 → False(即便凭证齐全)
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "p")
monkeypatch.setattr(settings, "OBSERVE_ENABLED", False)
assert settings.observe_configured is False
```
- [ ] **Step 2: 运行,确认失败**
Run: `pytest tests/test_observe.py::test_observe_configured_requires_switch_and_creds -q`
Expected: FAIL —— `AttributeError``settings``OBSERVE_ENABLED` / 无 `observe_configured`
- [ ] **Step 3: 实现配置**
`app/core/config.py``cors_origins_list` property 之后、`is_prod` property 之前插入:
```python
# ===== 可观测(OpenObserve 接口指标)=====
# 采集每个接口的 QPS + 耗时 + 错误率,批量直采到 OpenObserve(本地 Docker)。
# 默认关(prod 安全):未开启 → 中间件透传、worker 不启动,整套 no-op。
# 开启需 ENABLED=true 且 ENDPOINT/USER/PASSWORD 齐全(见 observe_configured)。
OBSERVE_ENABLED: bool = False
OBSERVE_ENDPOINT: str = "http://localhost:5080" # OpenObserve base URL
OBSERVE_ORG: str = "default" # 组织名
OBSERVE_STREAM: str = "app_requests" # stream 名(首次上报自动建)
OBSERVE_USER: str = "" # Basic auth 邮箱
OBSERVE_PASSWORD: str = "" # Basic auth 密码/token
OBSERVE_FLUSH_INTERVAL_SEC: float = 5.0 # worker 最长攒批间隔
OBSERVE_BATCH_MAX: int = 200 # 单批最大事件数
OBSERVE_QUEUE_MAX: int = 10000 # 有界队列上限,满则丢
OBSERVE_TIMEOUT_SEC: float = 5.0 # 上报 HTTP 超时
@property
def observe_configured(self) -> bool:
"""观测上报可用 = 总开关开 且 endpoint/账号/密码齐全(缺则整套 no-op)。"""
return bool(
self.OBSERVE_ENABLED
and self.OBSERVE_ENDPOINT
and self.OBSERVE_USER
and self.OBSERVE_PASSWORD
)
```
- [ ] **Step 4: 运行,确认通过**
Run: `pytest tests/test_observe.py::test_observe_configured_requires_switch_and_creds -q`
Expected: PASS
- [ ] **Step 5: 提交**
```bash
git add app/core/config.py tests/test_observe.py
git commit -m "feat(observe): 加 OBSERVE_* 配置与 observe_configured 门槛"
```
---
## Task 2: 事件队列 + `record_event` + `take_dropped`
**Files:**
- Create: `app/core/observe.py`
- Test: `tests/test_observe.py`(追加)
- [ ] **Step 1: 写失败测试**
`tests/test_observe.py` 顶部 import 区补 `import asyncio``from app.core import observe`,并追加:
```python
def test_record_event_enqueues(monkeypatch):
q = asyncio.Queue(maxsize=10)
monkeypatch.setattr(observe, "_queue", q)
observe.record_event({"route": "/x"})
assert q.get_nowait() == {"route": "/x"}
def test_record_event_drops_when_full(monkeypatch):
q = asyncio.Queue(maxsize=1)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(observe, "_dropped", 0)
observe.record_event({"n": 1}) # 占满
observe.record_event({"n": 2}) # 满 → 丢弃当前,不抛异常
assert observe.take_dropped() == 1
assert observe.take_dropped() == 0 # 取出后清零
assert q.get_nowait() == {"n": 1} # 保留的是先到的
```
- [ ] **Step 2: 运行,确认失败**
Run: `pytest tests/test_observe.py -q -k record_event`
Expected: FAIL —— `ModuleNotFoundError: app.core.observe` 或无 `record_event`
- [ ] **Step 3: 实现 `app/core/observe.py`(先只放队列部分)**
> 注意:本步只放队列相关代码。中间件用到的 `os`/`time`/`Match``_SKIP_PATHS`/`_UNMATCHED`/`_SERVICE` 常量放到 Task 3 一并加入——否则本步提交时 ruff 会报 F401 未用导入。
新建 `app/core/observe.py`
```python
"""接口指标埋点:有界事件队列 + 纯 ASGI 中间件。
每个 HTTP 请求测总耗时、抓路由模板 + 状态码,非阻塞塞进有界队列;由 observe_worker
后台批量上报到 OpenObserve。请求路径上无任何 I/O。未配置观测时中间件直接透传。
"""
from __future__ import annotations
import asyncio
from app.core.config import settings
# 有界事件队列(懒创建,见 get_queue):首次取用时在运行中的 loop 里建,避免 import 期
# 无 loop 的边角问题;put_nowait/get_nowait 不需运行中的 loop → 可在无 loop 下测试。
_queue: asyncio.Queue[dict] | None = None
# 队列满时的丢弃计数,worker 定期取出打日志。
_dropped = 0
def get_queue() -> asyncio.Queue[dict]:
"""返回全局有界事件队列(懒创建)。测试可 monkeypatch 模块级 _queue 换成小队列。"""
global _queue
if _queue is None:
_queue = asyncio.Queue(maxsize=settings.OBSERVE_QUEUE_MAX)
return _queue
def take_dropped() -> int:
"""取出并清零累计丢弃数(供 worker 打点)。"""
global _dropped
n, _dropped = _dropped, 0
return n
def record_event(event: dict) -> None:
"""非阻塞入队;队列满则丢弃当前事件并计数。永不抛异常、永不阻塞请求。"""
global _dropped
try:
get_queue().put_nowait(event)
except asyncio.QueueFull:
_dropped += 1
```
- [ ] **Step 4: 运行,确认通过**
Run: `pytest tests/test_observe.py -q -k record_event`
Expected: PASS
- [ ] **Step 5: 提交**
```bash
git add app/core/observe.py tests/test_observe.py
git commit -m "feat(observe): 加有界事件队列与 record_event(满则丢)"
```
---
## Task 3: `RequestMetricsMiddleware`(路由模板 + 状态码 + 耗时)
**Files:**
- Modify: `app/core/observe.py`(追加 `_resolve_route` 和中间件 class
- Test: `tests/test_observe.py`(追加)
- [ ] **Step 1: 写失败测试**
`tests/test_observe.py` 顶部 import 区补:
```python
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
```
并追加:
```python
def _make_probe_app() -> FastAPI:
"""独立最小 app:只挂中间件 + 两个无鉴权路由,不碰真业务 DB/auth。"""
app = FastAPI()
app.add_middleware(observe.RequestMetricsMiddleware)
@app.get("/things/{tid}")
def get_thing(tid: str):
return {"tid": tid}
@app.get("/health")
def health():
return {"ok": True}
return app
@pytest.fixture
def observe_on(monkeypatch):
"""开启观测 + 换一个干净小队列,返回该队列供断言。"""
monkeypatch.setattr(settings, "OBSERVE_ENABLED", True)
monkeypatch.setattr(settings, "OBSERVE_USER", "u")
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "p")
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
return q
def test_middleware_records_route_template(observe_on):
client = TestClient(_make_probe_app())
r = client.get("/things/42")
assert r.status_code == 200
evt = observe_on.get_nowait()
assert evt["route"] == "/things/{tid}" # 模板,不是 /things/42
assert evt["method"] == "GET"
assert evt["status"] == 200
assert evt["duration_ms"] >= 0
assert evt["service"] and "env" in evt and isinstance(evt["_timestamp"], int)
def test_middleware_skips_health(observe_on):
client = TestClient(_make_probe_app())
client.get("/health")
assert observe_on.empty()
def test_middleware_unmatched_route_is_normalized(observe_on):
client = TestClient(_make_probe_app())
r = client.get("/definitely-not-a-route")
assert r.status_code == 404
evt = observe_on.get_nowait()
assert evt["route"] == "__unmatched__"
assert evt["status"] == 404
def test_middleware_noop_when_disabled(monkeypatch):
monkeypatch.setattr(settings, "OBSERVE_ENABLED", False)
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
client = TestClient(_make_probe_app())
client.get("/things/1")
assert q.empty() # 未配置观测 → 零入队
```
- [ ] **Step 2: 运行,确认失败**
Run: `pytest tests/test_observe.py -q -k middleware`
Expected: FAIL —— `AttributeError: module 'app.core.observe' has no attribute 'RequestMetricsMiddleware'`
- [ ] **Step 3a: 给 `app/core/observe.py` 补中间件用的导入与常量**
把顶部 import 段从
```python
from __future__ import annotations
import asyncio
from app.core.config import settings
```
改成
```python
from __future__ import annotations
import asyncio
import os
import time
from starlette.routing import Match
from app.core.config import settings
# 不采集的路径(纯噪音):健康检查。
_SKIP_PATHS = frozenset({"/health"})
# 未匹配路由(404/扫描器)归一到此,防维度爆炸。
_UNMATCHED = "__unmatched__"
# service 字段:与 logging.py 同源(LOG_SERVICE_NAME),默认 app-server。
_SERVICE = os.getenv("LOG_SERVICE_NAME", "app-server")
```
`_queue` / `_dropped` / `get_queue` / `take_dropped` / `record_event` 保持不动。)
- [ ] **Step 3b: 实现中间件(追加到 `app/core/observe.py` 末尾)**
```python
def _resolve_route(scope) -> str:
"""从 scope 取路由模板(如 /things/{tid})。优先 scope['route'](现代 Starlette
路由后写入);取不到则手动匹配一次(老版本兜底);仍无 → __unmatched__(404/扫描器)。"""
route = scope.get("route")
path = getattr(route, "path", None)
if path:
return path
app_ = scope.get("app")
router = getattr(app_, "router", None)
for candidate in getattr(router, "routes", []):
try:
match, _ = candidate.matches(scope)
except Exception: # noqa: BLE001 - 匹配兜底,任一路由异常不影响整体
continue
if match == Match.FULL and getattr(candidate, "path", None):
return candidate.path
return _UNMATCHED
class RequestMetricsMiddleware:
"""纯 ASGI 中间件:测每个 http 请求耗时,记 method/route/status/duration。
放在最外层(main.py 里 CORS 之后 add),测到含 CORS 的完整耗时。未配置观测 → 透传。
"""
def __init__(self, app) -> None:
self.app = app
async def __call__(self, scope, receive, send) -> None:
if scope["type"] != "http" or not settings.observe_configured:
await self.app(scope, receive, send)
return
if scope.get("path") in _SKIP_PATHS:
await self.app(scope, receive, send)
return
start = time.perf_counter()
status_holder = {"status": 500} # 下游异常未产出 response 时兜底 500
async def send_wrapper(message) -> None:
if message["type"] == "http.response.start":
status_holder["status"] = message["status"]
await send(message)
try:
await self.app(scope, receive, send_wrapper)
finally:
duration_ms = (time.perf_counter() - start) * 1000.0
record_event({
"_timestamp": int(time.time() * 1_000_000), # µs,OpenObserve 时间列
"service": _SERVICE,
"env": settings.APP_ENV,
"method": scope.get("method", ""),
"route": _resolve_route(scope),
"status": status_holder["status"],
"duration_ms": round(duration_ms, 3),
})
```
- [ ] **Step 4: 运行,确认通过**
Run: `pytest tests/test_observe.py -q -k middleware`
Expected: PASS4 个中间件用例全过)
> 若 `test_middleware_records_route_template` 拿到的是 `/things/42` 而非模板,说明该 Starlette 版本未在 `scope["route"]` 写模板——此时 `_resolve_route` 的手动匹配兜底应已生效并返回模板;若仍不对,检查兜底分支是否被 import 顺序影响。
- [ ] **Step 5: 提交**
```bash
git add app/core/observe.py tests/test_observe.py
git commit -m "feat(observe): 加 RequestMetricsMiddleware(路由模板+状态码+耗时)"
```
---
## Task 4: 后台上报 worker
**Files:**
- Create: `app/core/observe_worker.py`
- Test: `tests/test_observe.py`(追加)
- [ ] **Step 1: 写失败测试**
`tests/test_observe.py` 顶部 import 区补:
```python
import httpx
from app.core import observe_worker
```
并追加:
```python
async def test_collect_batch_drains_up_to_batch_max(monkeypatch):
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(settings, "OBSERVE_FLUSH_INTERVAL_SEC", 0.1)
monkeypatch.setattr(settings, "OBSERVE_BATCH_MAX", 200)
for i in range(3):
q.put_nowait({"n": i})
batch = await observe_worker._collect_batch()
assert [e["n"] for e in batch] == [0, 1, 2]
async def test_collect_batch_timeout_returns_empty(monkeypatch):
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(settings, "OBSERVE_FLUSH_INTERVAL_SEC", 0.05)
batch = await observe_worker._collect_batch()
assert batch == []
async def test_post_batch_hits_json_ingest_url(monkeypatch):
monkeypatch.setattr(settings, "OBSERVE_ORG", "default")
monkeypatch.setattr(settings, "OBSERVE_STREAM", "app_requests")
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["url"] = str(request.url)
captured["json"] = request.content
return httpx.Response(200, json={"code": 200})
client = httpx.AsyncClient(
base_url="http://oo", transport=httpx.MockTransport(handler)
)
await observe_worker._post_batch(client, [{"route": "/x", "status": 200}])
await client.aclose()
assert captured["url"] == "http://oo/api/default/app_requests/_json"
assert b"/x" in captured["json"]
def test_start_observe_worker_noop_when_not_configured(monkeypatch):
monkeypatch.setattr(settings, "OBSERVE_ENABLED", False)
assert observe_worker.start_observe_worker() is None
```
- [ ] **Step 2: 运行,确认失败**
Run: `pytest tests/test_observe.py -q -k "collect_batch or post_batch or start_observe"`
Expected: FAIL —— `ModuleNotFoundError: app.core.observe_worker`
- [ ] **Step 3: 实现 `app/core/observe_worker.py`**
新建 `app/core/observe_worker.py`
```python
"""接口指标后台上报 worker:批量 drain 事件队列 → POST 到 OpenObserve。
对齐 heartbeat_monitor_worker 等的 start_*/stop_* 形态。best-effort 遥测:catch 全部
异常,上报失败直接丢批不重试。未配置观测 → start 返回 None(不启动),整套 no-op。
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
import httpx
from app.core.config import settings
from app.core.observe import get_queue, take_dropped
logger = logging.getLogger("shagua.observe")
# 上报用的 httpx client,start 时建、stop 时关。
_client: httpx.AsyncClient | None = None
async def _collect_batch() -> list[dict]:
"""等到 ≥1 条(或到 flush 间隔)后,连抽到 BATCH_MAX 条或抽空。超时且空 → 返回 []。"""
queue = get_queue()
batch: list[dict] = []
try:
first = await asyncio.wait_for(
queue.get(), timeout=settings.OBSERVE_FLUSH_INTERVAL_SEC
)
except asyncio.TimeoutError:
return batch
batch.append(first)
while len(batch) < settings.OBSERVE_BATCH_MAX:
try:
batch.append(queue.get_nowait())
except asyncio.QueueEmpty:
break
return batch
async def _post_batch(client: httpx.AsyncClient, batch: list[dict]) -> None:
"""POST 一批事件到 OpenObserve 的 _json ingest 端点。非 2xx 仅告警。"""
url = f"/api/{settings.OBSERVE_ORG}/{settings.OBSERVE_STREAM}/_json"
resp = await client.post(url, json=batch)
if resp.status_code >= 300:
logger.warning(
"observe ingest failed status=%s body=%s",
resp.status_code,
resp.text[:200],
)
async def _run_loop(client: httpx.AsyncClient) -> None:
try:
while True:
batch = await _collect_batch()
dropped = take_dropped()
if dropped:
logger.warning("observe dropped %d events (queue full)", dropped)
if not batch:
continue
try:
await _post_batch(client, batch)
except Exception: # noqa: BLE001 - best-effort 遥测,失败丢批不重试、不退出
logger.warning(
"observe post batch failed, dropped %d events",
len(batch),
exc_info=True,
)
except asyncio.CancelledError:
logger.info("observe worker stopped")
raise
def start_observe_worker() -> asyncio.Task | None:
"""启动上报 worker。未配置观测 → 返回 None(no-op)。"""
global _client
if not settings.observe_configured:
return None
_client = httpx.AsyncClient(
base_url=settings.OBSERVE_ENDPOINT,
auth=(settings.OBSERVE_USER, settings.OBSERVE_PASSWORD),
timeout=settings.OBSERVE_TIMEOUT_SEC,
)
logger.info(
"observe worker started endpoint=%s org=%s stream=%s",
settings.OBSERVE_ENDPOINT,
settings.OBSERVE_ORG,
settings.OBSERVE_STREAM,
)
return asyncio.create_task(_run_loop(_client), name="observe-worker")
async def stop_observe_worker(task: asyncio.Task | None) -> None:
"""收尾:cancel worker → best-effort 发最后一批 → 关 client。"""
global _client
if task is None:
return
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
if _client is not None:
# worker 已停,安全 drain 剩余并 best-effort 发最后一批(短超时,不卡关停)。
try:
queue = get_queue()
final: list[dict] = []
while len(final) < settings.OBSERVE_BATCH_MAX:
try:
final.append(queue.get_nowait())
except asyncio.QueueEmpty:
break
if final:
await asyncio.wait_for(
_post_batch(_client, final), timeout=settings.OBSERVE_TIMEOUT_SEC
)
except Exception: # noqa: BLE001 - 关停期尽力而为,失败忽略
pass
await _client.aclose()
_client = None
```
- [ ] **Step 4: 运行,确认通过**
Run: `pytest tests/test_observe.py -q -k "collect_batch or post_batch or start_observe"`
Expected: PASS
- [ ] **Step 5: 提交**
```bash
git add app/core/observe_worker.py tests/test_observe.py
git commit -m "feat(observe): 加后台批量上报 worker(失败丢批不重试)"
```
---
## Task 5: 接线到 `app/main.py`(挂中间件 + lifespan 启停)
**Files:**
- Modify: `app/main.py`import 区、lifespan、CORS 之后)
- Test: `tests/test_observe.py`(追加)
- [ ] **Step 1: 写失败测试**
`tests/test_observe.py` 追加:
```python
def test_app_has_metrics_middleware():
from app.main import app
names = [m.cls.__name__ for m in app.user_middleware]
assert "RequestMetricsMiddleware" in names
```
- [ ] **Step 2: 运行,确认失败**
Run: `pytest tests/test_observe.py::test_app_has_metrics_middleware -q`
Expected: FAIL —— 断言失败(中间件尚未挂载)
- [ ] **Step 3: 实现接线**
3a. 在 `app/main.py` import 区(`withdraw_reconcile_worker` import 块之后)加:
```python
from app.core.observe import RequestMetricsMiddleware
from app.core.observe_worker import (
start_observe_worker,
stop_observe_worker,
)
```
3b. lifespan 里加启停(现有 `daily_exchange_task = start_daily_exchange_worker()` 之后、`try:` 之前加一行;`finally` 里在 `stop_daily_exchange_worker` 之后加一行):
```python
daily_exchange_task = start_daily_exchange_worker()
observe_task = start_observe_worker()
try:
yield
finally:
await stop_heartbeat_monitor(heartbeat_task)
await stop_withdraw_reconcile_worker(reconcile_task)
await stop_daily_exchange_worker(daily_exchange_task)
await stop_observe_worker(observe_task)
await aclose_pricebot_client()
logger.info("shutting down")
```
3c. 挂中间件——在 CORS 的 `if settings.cors_origins_list:` 整块之后加(使其成为最外层,测到含 CORS 的完整耗时):
```python
# 接口指标埋点(最外层:测含 CORS 的完整耗时)。未配置观测时中间件自 no-op。
app.add_middleware(RequestMetricsMiddleware)
```
- [ ] **Step 4: 运行,确认通过**
Run: `pytest tests/test_observe.py::test_app_has_metrics_middleware -q`
Expected: PASS
- [ ] **Step 5: 跑整套 observe 测试 + 全量回归,确认无破坏**
Run: `pytest tests/test_observe.py -q && pytest -q`
Expected: 全 PASS(现有用例不受影响:conftest 未设 `OBSERVE_*` → 观测关 → worker no-op、中间件透传)
- [ ] **Step 6: 提交**
```bash
git add app/main.py tests/test_observe.py
git commit -m "feat(observe): main.py 挂中间件 + lifespan 启停上报 worker"
```
---
## Task 6: OpenObserve 本地部署(compose + README + .env.example
**Files:**
- Create: `deploy/openobserve/docker-compose.yml`
- Create: `deploy/openobserve/README.md`
- Modify: `.env.example`(追加 `OBSERVE_*` 段)
- [ ] **Step 1: 写 docker-compose**
新建 `deploy/openobserve/docker-compose.yml`
```yaml
# 本地开发用 OpenObserve(单容器 = local 模式)。用于接收 app-server 的接口指标。
# 启动: cd deploy/openobserve && docker compose up -d
# Web UI: http://localhost:5080 (账号见下方 env)
services:
openobserve:
image: public.ecr.aws/zinclabs/openobserve:latest
container_name: openobserve
ports:
- "5080:5080"
environment:
ZO_ROOT_USER_EMAIL: "admin@shaguabijia.local"
ZO_ROOT_USER_PASSWORD: "Complexpass#123"
ZO_DATA_DIR: "/data"
volumes:
- ./data:/data
restart: unless-stopped
```
- [ ] **Step 2: 写 README**
新建 `deploy/openobserve/README.md`
````markdown
# OpenObserve 本地部署(接口 QPS / 耗时可观测)
app-server 通过中间件采集每个接口的 QPS + 耗时 + 错误率,批量上报到这里。
## 启动
```bash
cd deploy/openobserve
docker compose up -d
```
- Web UIhttp://localhost:5080
- 登录:`admin@shaguabijia.local` / `Complexpass#123`(见 `docker-compose.yml`
- 数据落 `deploy/openobserve/data/`(已挂卷持久化;`data/` 建议 gitignore
## 让 app-server 上报
在项目根的 `.env` 打开观测(`OBSERVE_*`,账号密码与 compose 里 root 一致):
```dotenv
OBSERVE_ENABLED=true
OBSERVE_ENDPOINT=http://localhost:5080
OBSERVE_ORG=default
OBSERVE_STREAM=app_requests
OBSERVE_USER=admin@shaguabijia.local
OBSERVE_PASSWORD=Complexpass#123
```
重启 app-server,随便打几个接口。stream `app_requests` **首次上报自动创建**
在 UI 的 Logs → 选 `app_requests` 就能看到逐条请求事件。
## 查询(Logs 页 SQL,或建 Dashboard 面板)
各接口 QPS(1 分钟分桶,面板里再除 60 得每秒):
```sql
SELECT route, histogram(_timestamp, '1 minute') AS ts, count(*) AS cnt
FROM app_requests GROUP BY route, ts ORDER BY ts
```
各接口 P95 耗时(毫秒):
```sql
SELECT route, approx_percentile_cont(duration_ms, 0.95) AS p95_ms
FROM app_requests GROUP BY route ORDER BY p95_ms DESC
```
各接口错误率(5xx 占比):
```sql
SELECT route,
count(*) FILTER (WHERE status >= 500) * 100.0 / count(*) AS err_pct
FROM app_requests GROUP BY route ORDER BY err_pct DESC
```
## 停止 / 清数据
```bash
docker compose down # 停止(保留数据)
docker compose down -v && rm -rf data # 停止并清空数据
```
> 生产部署(持久化规格、独立 ingest 账号、鉴权收紧)见 spec 第 9 节,本期不做。
````
- [ ] **Step 3: 追加 `.env.example`**
`.env.example` 末尾追加:
```dotenv
# ===== 可观测(OpenObserve 接口指标)=====
# 采集每个接口 QPS + 耗时 + 错误率,批量直采到 OpenObserve(本地 Docker,见 deploy/openobserve/)。
# 默认关;开启需 ENABLED=true 且填 USER/PASSWORD(与 docker-compose 里 root 账号一致)。
OBSERVE_ENABLED=false
OBSERVE_ENDPOINT=http://localhost:5080
OBSERVE_ORG=default
OBSERVE_STREAM=app_requests
OBSERVE_USER=admin@shaguabijia.local
OBSERVE_PASSWORD=Complexpass#123
# 进阶(一般不用改):攒批间隔秒 / 单批最大条数 / 有界队列上限(满则丢) / 上报超时秒
OBSERVE_FLUSH_INTERVAL_SEC=5
OBSERVE_BATCH_MAX=200
OBSERVE_QUEUE_MAX=10000
OBSERVE_TIMEOUT_SEC=5
```
- [ ] **Step 4: 校验 compose 语法(不需真拉镜像)**
Run: `docker compose -f deploy/openobserve/docker-compose.yml config`
Expected: 打印规整后的配置、无报错(若本机无 docker,可跳过,标注为手动验证项)
- [ ] **Step 5: 提交**
```bash
git add deploy/openobserve/docker-compose.yml deploy/openobserve/README.md .env.example
git commit -m "feat(observe): 加 OpenObserve 本地 compose + README + .env.example"
```
---
## Task 7: 端到端手动验证 + 全量 lint/test 收尾
**Files:** 无(验证 + 收尾)
- [ ] **Step 1: 起 OpenObserve**
Run: `cd deploy/openobserve && docker compose up -d`
Expected: 容器起来,浏览器打开 http://localhost:5080 能登录
- [ ] **Step 2: 本地开观测起 app-server**
在根 `.env``OBSERVE_ENABLED=true` + `OBSERVE_USER/PASSWORD`(同 compose),然后:
Run: `./run.sh`Windows 用 `python -m uvicorn app.main:app --port 8770 --reload --reload-dir app`
Expected: 启动日志出现 `observe worker started endpoint=http://localhost:5080 ...`
- [ ] **Step 3: 打几个接口产生数据**
Run: `curl http://localhost:8770/health && curl http://localhost:8770/things-does-not-exist -i`(或正常业务接口若干)
Expected: 稍等 ≤5sflush 间隔),OpenObserve UI 的 Logs → `app_requests` 出现事件;`/health` 不应出现;不存在的路径 route 为 `__unmatched__`
- [ ] **Step 4: 验证三条查询**
在 OpenObserve UI 分别粘贴 README 里的 QPS / P95 / 错误率 SQL,确认能出数。
- [ ] **Step 5: lint(仅本改动涉及文件)+ 全量测试**
> 说明:仓库基线有 ~558 个既有 ruff 错误、且未强制 ruff 通过。不要去清历史欠债(范围蔓延)。只要求**本次新增/改动的文件**零 ruff 错误。
Run: `ruff check app/core/observe.py app/core/observe_worker.py tests/test_observe.py && python -m pytest -q`
Expected: 上述三个新文件 ruff 无错;测试里 `tests/test_observe.py` 全 PASS,且**全量失败数不超过基线的 4 个**(test_compare_proxy ×2 / test_coupon_proxy ×1 / test_invite ×1,均与本功能无关)。
额外确认我对既有文件的改动没有引入**新的** ruff 错误:`ruff check app/core/config.py app/main.py`(数量应与基线一致,不因本改动增加)。
- [ ] **Step 6: 关观测复跑一次,确认降级**
`.env``OBSERVE_ENABLED` 改回 `false``ruff check .` 不涉及,直接 `pytest -q`
Expected: 全 PASS(验证 observe 关闭时零副作用)
- [ ] **Step 7: 收尾提交(如有 .env 之外的改动)**
```bash
git add -A
git commit -m "chore(observe): 端到端验证与收尾" --allow-empty
```
> `.env` 不入 git(已 gitignore);本任务只验证,不提交 `.env`
---
## Self-Review(写完计划后自查)
- **Spec 覆盖**Docker 部署→Task 6/7;事件 schema→Task 3`record_event` 事件字段);中间件→Task 3worker→Task 4;配置→Task 1main 接线→Task 5;查询/仪表盘→Task 6 README;测试→Task 1-5;决策(a)队列满丢→Task 2(b)失败不重试→Task 4(c)跳过 /health→Task 3。全覆盖。
- **占位符**:无 TBD/TODO;每个代码步骤含完整代码。
- **类型/命名一致**`get_queue` / `record_event` / `take_dropped` / `RequestMetricsMiddleware` / `start_observe_worker` / `stop_observe_worker` / `observe_configured` / 事件字段名,跨 Task 1-5 与文件结构表一致。
@@ -0,0 +1,236 @@
# 接口 QPS + 耗时可观测(OpenObserve)设计
- **日期**2026-07-06
- **状态**:已评审通过,待写实现计划
- **范围**:仅 app-server8770);admin8771)暂不接入
- **方案**:A —— 轻量自研 ASGI 中间件 + 后台 worker 批量直采到 OpenObserve
## 1. 背景与目标
app-server 目前除 CORS 外无任何中间件,也无接口级可观测。需要按**每个接口**采集:
- **QPS**(每秒请求数,可按接口/时间分桶)
- **耗时**P50/P95/P99 等分位)
顺带低成本拿到**错误率**`status >= 500` 占比)。落地目标是:本地 Docker 跑一个 OpenObserve 实例接收数据,服务侧加埋点上报,在 OpenObserve 仪表盘上看各接口 QPS + 耗时。
### 非目标(YAGNI
- 不做分布式 trace / span 关联(只要接口聚合指标)。
- 不引入 OpenTelemetry / Prometheus 客户端等重依赖。
- 不采集请求体 / query / 用户身份等,任何 PII 都不进上报。
- admin(8771)本期不接(中间件写成可复用,未来一行挂载即可)。
- 上报失败不做持久化重试 / 落盘补偿(best-effort)。
## 2. 方案选型
对比过三条路(详见评审记录):
- **A 轻量自研中间件 + JSON 直采**(选中):零新依赖(`httpx` 已在依赖里),完全贴合本仓库「后台 worker + JSON 事件 + `*_configured` 优雅降级」的既有习惯,恰好满足「每接口 QPS + 耗时 + 错误率」并保留原始事件下钻能力。
- B OpenTelemetry 自动埋点 + OTLP:行业标准、顺带 trace,但多 5–6 个依赖、概念多、数据量/成本高于需求,与精简代码库风格相悖。
- C Prometheus 进程内聚合 + remote_write/抓取:数据量最小,但 remote_write 编码复杂或需额外抓取进程,丢失单请求下钻,最不贴合 OpenObserve 的 log-first 强项。
**结论:A。**
## 3. 架构与数据流
```
每个 HTTP 请求
→ RequestMetricsMiddleware(最外层:测总耗时 / 抓路由模板 + 状态码)
→ record_event() 非阻塞入队(有界队列,满则丢最旧,绝不阻塞、绝不 OOM)
→ observe_worker(后台 asyncio.Task,随 lifespan 启停)批量 drain
→ httpx POST {ENDPOINT}/api/{ORG}/{STREAM}/_json → OpenObserve
→ 仪表盘 SQL 聚合出 QPS / 分位耗时 / 错误率
```
**核心不变量**
1. 请求路径上只做「测时 + 构建一个小 dict + `put_nowait`」,**无任何网络/磁盘 I/O**。
2. 所有上报 I/O 在后台 worker;worker 捕获全部异常,绝不让埋点影响请求。
3. 未配置观测(`observe_configured=False`)→ 中间件透传、worker 不启动,整套 no-op。
4. OpenObserve 不可用 → 队列填满后丢弃事件 + 限流告警,业务零影响。
## 4. 组件设计
### 4.1 OpenObserve 本地部署 —— `deploy/openobserve/docker-compose.yml`(新增)
```yaml
services:
openobserve:
image: public.ecr.aws/zinclabs/openobserve:latest
container_name: openobserve
ports: ["5080:5080"]
environment:
ZO_ROOT_USER_EMAIL: "admin@shaguabijia.local"
ZO_ROOT_USER_PASSWORD: "Complexpass#123"
ZO_DATA_DIR: "/data"
volumes: ["./data:/data"]
restart: unless-stopped
```
- `docker compose up -d` 启动;Web UI `http://localhost:5080`,用上面邮箱/密码登录。
- 单容器 = local 模式,数据落 `./data`(已挂卷持久化)。
- **stream 首次上报自动创建**,无需预建 `app_requests`
- 上报鉴权:HTTP Basic auth`email:password`),本地直接用 root 账号;生产应另建仅具 ingest 权限的用户/服务账号(本期不涉及)。
### 4.2 事件 schema(一请求一行 JSON
```json
{
"_timestamp": 1720000000000000, // 微秒(µs)整数,请求完成时刻。OpenObserve 默认时间列 _timestamp 以微秒计
"service": "app-server", // 取 LOG_SERVICE_NAME / 固定值
"env": "dev", // settings.APP_ENV
"method": "POST",
"route": "/api/v1/coupon/step", // 路由模板(非实际 path)
"status": 200,
"duration_ms": 42.7 // float 毫秒
}
```
- **只存路由模板**(如 `/c/{code}``/media` 静态归一),避免 path 参数把维度打爆。
- 未匹配路由(404 / 扫描器)归一到常量 `__unmatched__`
- 只采 method / route / status / duration —— 无 body、无 query、无 PII。
### 4.3 埋点中间件 —— `app/core/observe.py`(新增)
**纯 ASGI 中间件**(比 `BaseHTTPMiddleware` 开销低;能可靠读到路由与最终状态码;scope 按引用透传,内层 router 的 `scope["route"]` 外层可见)。
职责:
1. 非 `http` 请求、或 `not settings.observe_configured` → 直接透传,不测。
2. `perf_counter()` 记起点;包一层 `send``http.response.start``status`(默认兜底 500,覆盖下游抛异常未产出 response 的情况)。
3. `finally` 里算 `duration_ms`,从 `scope` 取路由模板(见下),构建事件,调 `record_event()`
4. 跳过路径集合 `_SKIP_PATHS = {"/health"}`(纯噪音)。
**路由模板解析(跨 Starlette 版本稳健)**
```python
route = scope.get("route")
template = getattr(route, "path", None)
if template is None: # 未匹配 / 老版本未写 scope["route"]
template = "__unmatched__"
```
(若实测某 Starlette 版本不写 `scope["route"]`,回退用 `request.app.router.routes` 逐个 `route.matches(scope)==Match.FULL` 找模板;实现时以实际版本为准,优先 `scope["route"]`。)
**入队(`record_event`**:模块级 `asyncio.Queue(maxsize=OBSERVE_QUEUE_MAX)`。用 `put_nowait``QueueFull` 则丢弃并累加一个 `_dropped` 计数(每累计 N 条限流打一条 WARNING)。**永不 `await put()`、永不阻塞请求**。
> 决策(a):队列满 → **丢弃**(不阻塞请求)。
### 4.4 上报 worker —— `app/core/observe_worker.py`(新增)
对齐现有 `heartbeat_monitor_worker.py` / `daily_exchange_worker.py` / `withdraw_reconcile_worker.py``start_*` / `stop_*` 形态。
- `start_observe_worker() -> asyncio.Task | None`
- `not settings.observe_configured` → 返回 `None`no-op)。
- 否则建专用 `httpx.AsyncClient``base_url=ENDPOINT``auth=(USER, PASSWORD)``timeout=OBSERVE_TIMEOUT_SEC`),起 `_run_loop` task。
- `_run_loop()`:循环
1. `_collect_batch()``await asyncio.wait_for(queue.get(), timeout=FLUSH_INTERVAL)` 拿到首条(超时且空 → 返回空,continue);再 `get_nowait()` 连抽到 `BATCH_MAX` 条或抽空。
2. `POST /api/{ORG}/{STREAM}/_json`body 为事件数组。
3. **catch 所有异常**:失败限流打 WARNING,**直接丢弃该批,不重试**。
- `stop_observe_worker(task)`best-effort 收尾 flush(短超时)→ `task.cancel()``await`(吞 `CancelledError`)→ 关 client。
> 决策(b):上报失败 → **直接丢弃,不重试**best-effort 遥测)。
### 4.5 配置 —— `app/core/config.py`(改)
新增一段 `# ===== 可观测(OpenObserve 接口指标)=====`,默认全关(prod 安全):
| 配置 | 默认 | 说明 |
|---|---|---|
| `OBSERVE_ENABLED` | `False` | 总开关;默认关,opt-in |
| `OBSERVE_ENDPOINT` | `http://localhost:5080` | OpenObserve base URL |
| `OBSERVE_ORG` | `default` | 组织名 |
| `OBSERVE_STREAM` | `app_requests` | stream 名 |
| `OBSERVE_USER` | `""` | Basic auth 邮箱 |
| `OBSERVE_PASSWORD` | `""` | Basic auth 密码/token |
| `OBSERVE_FLUSH_INTERVAL_SEC` | `5.0` | worker 最长攒批间隔 |
| `OBSERVE_BATCH_MAX` | `200` | 单批最大事件数 |
| `OBSERVE_QUEUE_MAX` | `10000` | 有界队列上限,满则丢 |
| `OBSERVE_TIMEOUT_SEC` | `5.0` | 上报 HTTP 超时 |
```python
@property
def observe_configured(self) -> bool:
return bool(self.OBSERVE_ENABLED and self.OBSERVE_ENDPOINT
and self.OBSERVE_USER and self.OBSERVE_PASSWORD)
```
`.env.example` 同步补一段带注释的 `OBSERVE_*`(沿用该文件重注释风格),`OBSERVE_ENABLED=false`
### 4.6 接线 —— `app/main.py`(改)
- import `RequestMetricsMiddleware``start_observe_worker` / `stop_observe_worker`
- `app.add_middleware(RequestMetricsMiddleware)`:放在 CORS `add_middleware` **之后** → 成为最外层,测到含 CORS 的完整耗时。无条件挂载(内部自 no-op)。
- `lifespan`:启动 `observe_task = start_observe_worker()``finally``await stop_observe_worker(observe_task)`,与现有 worker 并列。
### 4.7 OpenObserve 查询 / 仪表盘 —— `deploy/openobserve/README.md`(新增)
含:compose 启停、登录、stream 自动创建说明、`.env` 接线,以及可直接粘的示例 SQL
- **各接口 QPS**1 分钟分桶):
```sql
SELECT route, histogram(_timestamp, '1 minute') AS ts, count(*) AS cnt
FROM app_requests GROUP BY route, ts ORDER BY ts
```
(面板按 `cnt/60` 展示每秒;或用 OpenObserve 图表的 rate 能力。)
- **各接口 P95 耗时**
```sql
SELECT route, approx_percentile_cont(duration_ms, 0.95) AS p95_ms
FROM app_requests GROUP BY route ORDER BY p95_ms DESC
```
- **各接口错误率**
```sql
SELECT route,
count(*) FILTER (WHERE status >= 500) * 100.0 / count(*) AS err_pct
FROM app_requests GROUP BY route ORDER BY err_pct DESC
```
## 5. 关键设计决策汇总
- **(a) 队列满 → 丢弃**(不阻塞请求):遥测让路于业务可用性。
- **(b) 上报失败 → 不重试**best-effort;避免 poison batch 堆积与队列无限增长。
- **(c) 跳过 `/health`**:健康检查是纯噪音,硬编码在 `_SKIP_PATHS`
- **只存路由模板 + `__unmatched__`**:防维度爆炸。
- **默认 OFF、opt-in**:prod 安全默认;开启后仍全异步 + 有界。
- **纯 ASGI 中间件 + `perf_counter`**:请求路径开销微秒级,无 I/O。
## 6. 安全 / 性能保证
- 请求路径新增开销 ≈ 一次 `perf_counter` 差 + 一个小 dict + 一次 `put_nowait`(微秒级),无锁竞争的显著热点。
- 失败隔离:入队丢弃 + worker 全异常捕获;OpenObserve 宕机不影响任何请求。
- 有界内存:队列 `maxsize` 封顶,最坏丢事件不涨内存。
- 无 PII:仅 method / route / status / duration。
## 7. 测试策略 —— `tests/test_observe.py`(新增)
沿用仓库约定(`TestClient` + `monkeypatch`,绝不打真网络;`conftest` 在 import 前设 env):
1. 埋点入队字段正确:模板路由、`status``duration_ms > 0`
2. 参数化路由 → 取到**模板**而非实际 path。
3. 未匹配路径(404)→ `route == "__unmatched__"`
4. `OBSERVE_ENABLED=false` → 零入队、零 HTTP(现有测试不受影响)。
5. 队列满 → `record_event` 不抛异常(走丢弃分支)。
6. worker 批量 POST 的 URL / payload 正确(monkeypatch httpx client / `_post`,不打网络)。
7. `/health` 被跳过 → 不入队。
> `settings``lru_cache` 单例;需要开启观测的用例通过 monkeypatch `settings` 属性或直接调 `record_event` / 中间件并 patch `observe_configured` 实现,避免全局 env 改动波及他用例。
## 8. 文件清单
| 文件 | 动作 |
|---|---|
| `deploy/openobserve/docker-compose.yml` | 新增(OpenObserve 容器)|
| `deploy/openobserve/README.md` | 新增(部署步骤 + 查询/仪表盘)|
| `app/core/observe.py` | 新增(中间件 + 有界队列 + `record_event` + 路由解析)|
| `app/core/observe_worker.py` | 新增(后台批量上报 worker)|
| `app/core/config.py` | 改(`OBSERVE_*` + `observe_configured`|
| `app/main.py` | 改(挂中间件 + lifespan 启停 worker|
| `.env.example` | 改(新增 `OBSERVE_*` 注释段)|
| `tests/test_observe.py` | 新增 |
## 9. 未来工作(本期不做)
- admin(8771)接入同一套中间件(`service` 字段区分)。
- 生产部署 OpenObserve(持久化、独立 ingest 账号、资源规格、鉴权收紧)。
- 上报字段扩展(如按 user/设备维度、上游 pricebot 透传耗时拆分)。
+1 -1
View File
@@ -169,7 +169,7 @@ POST /api/v1/auth/sms/login { phone, code } → 任意 6 位通过 → upsert
短信冷却表存进程内存(`--workers 1` 下够用,多 worker/重启即失效)。
**real 模式(`SMS_MOCK=false`,生产)**:自定义验证码——本服务 `secrets` 生成 6 位码 → 极光 `/v1/messages` 只负责发 → 本地校验(一次性 / 防爆破),鉴权**复用 `JG_APP_KEY`/`JG_MASTER_SECRET`**(短信与一键登录同一极光应用,**上线只需 `SMS_MOCK=false`**)。防刷(单号冷却 + 单设备频控 + 单码失败次数;2026-07-03 精简删单号每日上限与登录纯 IP 限流)+ 错误码 429/503/400。详见 [integrations/sms](./integrations/sms.md)。
**real 模式(`SMS_MOCK=false`,生产)**:自定义验证码——本服务 `secrets` 生成 6 位码 → 极光 `/v1/messages` 只负责发 → 本地校验(一次性 / 防爆破),鉴权**复用 `JG_APP_KEY`/`JG_MASTER_SECRET`**(短信与一键登录同一极光应用,**上线只需 `SMS_MOCK=false`**)。防刷四层(单号冷却 + 单号每日上限 + 单 IP `rate_limit` + 单码失败次数)+ 错误码 429/503/400。详见 [integrations/sms](./integrations/sms.md)。
### 4.3 Token 与刷新
+1 -8
View File
@@ -68,14 +68,7 @@ def test_list_config(admin_client: TestClient, token: str) -> None:
r = admin_client.get("/admin/api/config", headers=_auth(token))
assert r.status_code == 200, r.text
items = {i["key"]: i for i in r.json()}
# 非 hidden 项照常返回;看广告组保留可见的:每日上限 / 单次金币上限 / 关闭后冷却。
assert "signin_rewards" in items and "ad_daily_limit" in items and "ad_cooldown_sec" in items
# hidden 项(任务/里程碑、首页轮播数据源、看广告组的单次金币/每轮次数/信息流广告开关)不在配置页返回。
for hidden_key in (
"task_rewards", "record_milestones", "marquee_feed_mode",
"ad_reward_coin", "ad_round_count", "comparing_ad_enabled",
):
assert hidden_key not in items, f"{hidden_key} 应被 hidden 过滤"
assert "signin_rewards" in items and "ad_daily_limit" in items
assert items["signin_rewards"]["value"] == [
200, 200, 300, 200, 400, 400, 800,
]
-152
View File
@@ -1,152 +0,0 @@
"""Admin RBAC 角色/权限测试:可见页下发、角色 CRUD 仅 super_admin、内建/在用保护。"""
from __future__ import annotations
import pytest
from fastapi.testclient import TestClient
from app.admin.main import admin_app
from app.admin.repositories import admin_user as admin_repo
from app.core.security import hash_password
from app.db.session import SessionLocal
@pytest.fixture()
def admin_client() -> TestClient:
return TestClient(admin_app)
def _token(username: str, role: str) -> str:
db = SessionLocal()
try:
a = admin_repo.get_by_username(db, username)
if a is None:
admin_repo.create_admin(db, username=username, password="pass1234", role=role)
else:
a.password_hash = hash_password("pass1234")
a.role = role
a.status = "active"
db.commit()
finally:
db.close()
c = TestClient(admin_app)
return c.post(
"/admin/api/auth/login", json={"username": username, "password": "pass1234"}
).json()["access_token"]
@pytest.fixture()
def super_token() -> str:
return _token("r_super", "super_admin")
@pytest.fixture()
def operator_token() -> str:
return _token("r_operator", "operator")
def _auth(t: str) -> dict:
return {"Authorization": f"Bearer {t}"}
def test_super_pages_all_operator_limited(admin_client, super_token, operator_token) -> None:
su = admin_client.get("/admin/api/auth/me", headers=_auth(super_token)).json()
assert "admins" in su["pages"] and "dashboard" in su["pages"] # 超管全页
op = admin_client.get("/admin/api/auth/me", headers=_auth(operator_token)).json()
assert "dashboard" in op["pages"] and "admins" not in op["pages"] # 运营看不到管理员页
def test_roles_endpoints_super_only(admin_client, super_token, operator_token) -> None:
assert admin_client.get("/admin/api/roles", headers=_auth(super_token)).status_code == 200
assert admin_client.get("/admin/api/roles", headers=_auth(operator_token)).status_code == 403
def test_role_crud(admin_client, super_token) -> None:
cat = admin_client.get("/admin/api/roles/catalog", headers=_auth(super_token)).json()
assert any(g["group"] == "看板" for g in cat)
r = admin_client.post(
"/admin/api/roles", json={"name": "审核", "pages": ["dashboard", "feedbacks", "xx-bad"]},
headers=_auth(super_token),
)
assert r.status_code == 200, r.text
rid = r.json()["id"]
assert set(r.json()["pages"]) == {"dashboard", "feedbacks"} # 非法 key 被过滤
# 重名 409
assert admin_client.post(
"/admin/api/roles", json={"name": "审核", "pages": []}, headers=_auth(super_token)
).status_code == 409
# 改可见页
u = admin_client.patch(
f"/admin/api/roles/{rid}", json={"pages": ["dashboard"]}, headers=_auth(super_token)
)
assert u.status_code == 200 and u.json()["pages"] == ["dashboard"]
# 删
assert admin_client.delete(f"/admin/api/roles/{rid}", headers=_auth(super_token)).status_code == 200
def test_builtin_super_admin_protected(admin_client, super_token) -> None:
roles = admin_client.get("/admin/api/roles", headers=_auth(super_token)).json()
sa = next(r for r in roles if r["name"] == "super_admin")
assert sa["is_builtin"] and len(sa["pages"]) >= 10 # 全部页
assert admin_client.patch(
f"/admin/api/roles/{sa['id']}", json={"pages": []}, headers=_auth(super_token)
).status_code == 400
assert admin_client.delete(
f"/admin/api/roles/{sa['id']}", headers=_auth(super_token)
).status_code == 400
def test_delete_role_in_use_blocked(admin_client, super_token) -> None:
admin_client.post(
"/admin/api/admins",
json={"username": "role_holder", "password": "pass1234", "role": "operator"},
headers=_auth(super_token),
)
roles = admin_client.get("/admin/api/roles", headers=_auth(super_token)).json()
op = next(r for r in roles if r["name"] == "operator")
assert op["in_use"] >= 1
assert admin_client.delete(
f"/admin/api/roles/{op['id']}", headers=_auth(super_token)
).status_code == 400
def test_builtin_roles_labels_and_pages(admin_client, super_token) -> None:
roles = {r["name"]: r for r in admin_client.get("/admin/api/roles", headers=_auth(super_token)).json()}
# 中文展示名(key 不变)
assert roles["super_admin"]["label"] == "管理员"
assert roles["operator"]["label"] == "运营"
assert roles["finance"]["label"] == "财务"
assert roles["tech"]["label"] == "技术"
# 页集对齐 Prototypes/dashboard/permissions.md 的 ROLES
assert set(roles["finance"]["pages"]) == {"dashboard", "ad-revenue-report", "cps", "withdraws"}
assert set(roles["tech"]["pages"]) == {
"dashboard", "device-liveness", "config", "ad-revenue", "event-logs", "audit-logs",
}
def test_ui_created_admin_shows_password_script_admin_hidden(admin_client, super_token) -> None:
# UI 建号:账号列表回显明文登录密码(供权限管理页编辑复看)
admin_client.post(
"/admin/api/admins",
json={"username": "pw_show_user", "password": "pass1234", "role": "operator"},
headers=_auth(super_token),
)
admins = {a["username"]: a for a in admin_client.get("/admin/api/admins", headers=_auth(super_token)).json()}
assert admins["pw_show_user"]["password"] == "pass1234"
# 经 repo/脚本直建的账号(r_super,无留存明文)→ password 为 None,前端不显示
assert admins["r_super"]["password"] is None
# 登录 / me 不下发明文(保持 None)
me = admin_client.get("/admin/api/auth/me", headers=_auth(super_token)).json()
assert me.get("password") is None
def test_create_admin_rejects_unknown_role(admin_client, super_token) -> None:
r = admin_client.post(
"/admin/api/admins",
json={"username": "bad_role_user", "password": "pass1234", "role": "不存在的角色"},
headers=_auth(super_token),
)
assert r.status_code == 400
+22 -2
View File
@@ -17,6 +17,7 @@ def _reset(phone: str) -> None:
"""清该号的进程内存状态,隔离 real 模式用例。"""
sms._codes.pop(phone, None)
sms._last_sent.pop(phone, None)
sms._daily_count.pop(phone, None)
class _OkResp:
@@ -75,7 +76,7 @@ def test_sms_send_too_frequent(client) -> None:
def test_sms_send_device_ip_rate_limit(client, monkeypatch) -> None:
"""发码防刷:同一设备(device_id) + 同一 IP 每小时最多 N 次发码,超出 429。
用不同手机号(绕开单号 60s 冷却)证明限流按设备封顶 换号绕开单号冷却的洞
用不同手机号(绕开单号 60s 冷却)证明限流按设备封顶 换号绕开单号冷却/每日上限的洞
conftest 默认关限流;本用例临时打开 + 调小阈值便于测 + 清计数隔离"""
from app.api.v1 import auth
from app.core import ratelimit
@@ -154,6 +155,21 @@ def test_phone_format_validation(client) -> None:
assert r.status_code == 422
def test_sms_daily_limit(monkeypatch) -> None:
"""单号每日上限(send 的防刷逻辑 mock/real 都跑;函数级绕开 60s 冷却)。"""
phone = "13511135000"
_reset(phone)
limit = 3
monkeypatch.setattr(sms.settings, "SMS_DAILY_LIMIT_PER_PHONE", limit)
for _ in range(limit):
sms.send_code(phone)
sms._last_sent.pop(phone, None) # 绕开冷却,只测每日上限
with pytest.raises(sms.SmsError, match="上限"):
sms.send_code(phone)
# ============================ real 路径(不真发)============================
def test_sms_real_send_calls_jiguang(monkeypatch) -> None:
@@ -239,20 +255,24 @@ def test_sms_real_balance_error_keeps_cooldown(monkeypatch) -> None:
def test_sms_gc_purges_stale_only(monkeypatch) -> None:
"""GC 清过期码 / 旧冷却,但不动今天有效的(阈值设 0 强制每次扫)。"""
"""GC 清过期码 / 旧冷却 / 隔日计数,但不动今天有效的(阈值设 0 强制每次扫)。"""
monkeypatch.setattr(sms, "_GC_THRESHOLD", 0)
sms._codes.clear()
sms._last_sent.clear()
sms._daily_count.clear()
now = time.time()
sms._codes["stale"] = sms._CodeRecord(code="111111", expires_at=now - 1)
sms._codes["fresh"] = sms._CodeRecord(code="222222", expires_at=now + 999)
sms._last_sent["old"] = now - 99999
sms._last_sent["recent"] = now
sms._daily_count["yesterday"] = ("2000-01-01", 3)
sms._daily_count["today"] = (sms._today(), 1)
sms._gc(now)
assert "stale" not in sms._codes and "fresh" in sms._codes
assert "old" not in sms._last_sent and "recent" in sms._last_sent
assert "yesterday" not in sms._daily_count and "today" in sms._daily_count
# ============================ 用户名 / 默认昵称 ============================
-258
View File
@@ -1,258 +0,0 @@
"""后端 harvest 落库测试(比价记录改后端 harvest 后新增)。
覆盖:
repo :harvest_running(建行/幂等回填)harvest_done(派生+newly_success 幂等)
harvest_abort(夭折 / **不降级 success**)upsert_record 不降级
端点层:price/step 首帧 mint trace_id + 回传 + running ;done 帧落 success;
trace/finalize cancelled;软鉴权( token 也放行user_id 暂空)
pricebot httpx mock,不真连( test_compare_proxy)
"""
from __future__ import annotations
import json
import uuid
from unittest.mock import MagicMock, patch
import httpx
from app.db.session import SessionLocal
from app.models.comparison import ComparisonRecord
from app.repositories import comparison as crud
from app.schemas.compare_record import ComparisonRecordIn
from sqlalchemy import select
def _tid() -> str:
return uuid.uuid4().hex
def _done_params() -> dict:
"""一份典型 done 帧 params:美团 25 元 vs 源淘宝闪购 30 元 → 省 5 元、success。"""
return {
"comparison_results": [
{"platform_id": "meituan", "platform_name": "美团", "package": "com.sankuai.meituan",
"price": 25.0, "is_source": False, "rank": 1, "items": []},
{"platform_id": "taobao_flash", "platform_name": "淘宝闪购",
"package": "com.taobao.taobao", "price": 30.0, "is_source": True, "rank": 2,
"store_name": "测试店", "items": [{"name": "肥牛饭", "qty": 1}]},
],
"information": "美团更便宜",
"trace_url": "https://price.shaguabijia.com/traces/done/",
}
def _get(db, trace_id: str) -> ComparisonRecord | None:
return db.execute(
select(ComparisonRecord).where(ComparisonRecord.trace_id == trace_id)
).scalar_one_or_none()
# ============================================================
# repo 层
# ============================================================
def test_harvest_running_creates_row(client) -> None:
tid = _tid()
with SessionLocal() as db:
crud.harvest_running(
db, trace_id=tid, user_id=None, device_id="dev-1",
device_info={"brand": "vivo", "model": "V2309A", "android_version": "14",
"rom_version": "14"},
trace_url="https://price.shaguabijia.com/traces/run/",
)
rec = _get(db, tid)
assert rec is not None
assert rec.status == "running"
assert rec.user_id is None
assert rec.device_id == "dev-1"
assert rec.device_model == "V2309A"
assert rec.device_manufacturer == "vivo"
assert rec.rom_version == 14
assert rec.trace_url.endswith("/run/")
def test_harvest_running_idempotent_backfills(client) -> None:
tid = _tid()
with SessionLocal() as db:
crud.harvest_running(db, trace_id=tid, user_id=None, trace_url=None)
# 第二帧带上 trace_url + user_id → 回填空缺,不新建、不改 status
crud.harvest_running(db, trace_id=tid, user_id=None,
trace_url="https://price.shaguabijia.com/traces/late/")
rows = db.execute(
select(ComparisonRecord).where(ComparisonRecord.trace_id == tid)
).scalars().all()
assert len(rows) == 1 # 幂等:仍一行
assert rows[0].status == "running"
assert rows[0].trace_url.endswith("/late/") # 空缺被回填
def test_harvest_done_derives_and_newly_success_once(client) -> None:
tid = _tid()
with SessionLocal() as db:
crud.harvest_running(db, trace_id=tid, user_id=None)
rec, newly = crud.harvest_done(db, trace_id=tid, user_id=None,
done_params=_done_params())
assert newly is True # running → success 是"新落成"
assert rec.status == "success"
assert rec.source_platform_id == "taobao_flash"
assert rec.source_price_cents == 3000
assert rec.best_platform_id == "meituan"
assert rec.best_price_cents == 2500
assert rec.saved_amount_cents == 500 # 30 - 25
assert rec.is_source_best is False
assert rec.store_name == "测试店"
assert rec.information == "美团更便宜"
assert rec.items == [{"name": "肥牛饭", "qty": 1}]
assert rec.trace_url.endswith("/done/")
# 再来一次(重试 done)→ 已 success,newly_success=False(发奖不重复触发)
_rec2, newly2 = crud.harvest_done(db, trace_id=tid, user_id=None,
done_params=_done_params())
assert newly2 is False
def test_harvest_abort_cancels_running(client) -> None:
tid = _tid()
with SessionLocal() as db:
crud.harvest_running(db, trace_id=tid, user_id=None)
rec = crud.harvest_abort(db, trace_id=tid, status="cancelled",
reason="用户终止比价",
trace_url="https://price.shaguabijia.com/traces/ab/")
assert rec is not None
assert rec.status == "cancelled"
assert rec.information == "用户终止比价"
assert rec.trace_url.endswith("/ab/")
def test_harvest_abort_no_downgrade_success(client) -> None:
"""已 success 的行,finalize 后到(收尾取消)→ 只 refresh trace_url,status 不降级。"""
tid = _tid()
with SessionLocal() as db:
crud.harvest_running(db, trace_id=tid, user_id=None)
crud.harvest_done(db, trace_id=tid, user_id=None, done_params=_done_params())
rec = crud.harvest_abort(db, trace_id=tid, status="cancelled", reason="收尾误触")
assert rec.status == "success" # 不降级
assert rec.information == "美团更便宜" # 不被 abort 的 reason 覆盖
def test_harvest_abort_missing_row_returns_none(client) -> None:
with SessionLocal() as db:
assert crud.harvest_abort(db, trace_id=_tid(), status="cancelled",
reason=None) is None
def test_upsert_record_no_downgrade_after_harvest_success(client) -> None:
"""harvest 落 success 后,老客户端 fromFailure 的 cancelled 上报不许把它盖回去。"""
tid = _tid()
with SessionLocal() as db:
crud.harvest_done(db, trace_id=tid, user_id=None, done_params=_done_params())
payload = ComparisonRecordIn(
trace_id=tid, business_type="food", status="cancelled",
information="用户终止", comparison_results=[],
)
# 用一个不会与顺序自增用户撞的合成 id(SQLite 测试库 FK 不强制;别用小整数,
# 否则会撞上别的测试 login 出来的真实 user_id → 记录混进那个用户的列表)。
rec = crud.upsert_record(db, user_id=987654, payload=payload)
assert rec.status == "success" # 不降级
assert rec.user_id == 987654 # 但补上了 user_id(原为 None)
# ============================================================
# 端点层(mock pricebot)
# ============================================================
def _mock_pricebot(resp_json: dict):
"""patch httpx.AsyncClient.post 返回给定响应;捕获转发的 content。"""
captured: dict = {}
async def fake_post(self, url, content=None, **kw):
captured["url"] = url
captured["content"] = content
m = MagicMock()
m.status_code = 200
m.json = lambda: dict(resp_json)
return m
return patch.object(httpx.AsyncClient, "post", fake_post), captured
def _stub_body(**over) -> dict:
body = {
"device_id": "dev-x",
"step": 0,
"query": "海底捞",
"device_info": {"brand": "OPPO", "model": "PEXM00", "android_version": "13",
"rom_version": "13"},
"screen_state": {"screen": {"width": 1080, "height": 2340, "density": 3.0},
"foreground": {"package": "com.taobao.taobao", "activity": ""},
"windows": []},
}
body.update(over)
return body
def test_price_step_mints_trace_id_and_creates_running(client) -> None:
"""首帧不带 trace_id → app-server 签发 + 回传;并建 running 行(带机型/设备)。"""
wait_frame = {"success": True, "action": {"command": "wait", "params": {"duration_ms": 500}},
"continue": True, "trace_url": "https://price.shaguabijia.com/traces/mint/"}
p, _cap = _mock_pricebot(wait_frame)
with p:
r = client.post("/api/v1/price/step", json=_stub_body()) # 无 trace_id、无 token
assert r.status_code == 200, r.text
tid = r.json().get("trace_id")
assert tid and len(tid) >= 16 # 回传了签发的 trace_id
with SessionLocal() as db:
rec = _get(db, tid)
assert rec is not None and rec.status == "running"
assert rec.user_id is None # 无 token → 软鉴权放行、user_id 暂空
assert rec.device_model == "PEXM00"
assert rec.trace_url.endswith("/mint/")
def test_price_step_done_harvests_success(client) -> None:
tid = _tid()
done_frame = {"success": True, "continue": False,
"action": {"command": "done", "params": _done_params()},
"trace_url": "https://price.shaguabijia.com/traces/done2/"}
p, _cap = _mock_pricebot(done_frame)
with p:
r = client.post("/api/v1/price/step", json=_stub_body(trace_id=tid, step=8))
assert r.status_code == 200
with SessionLocal() as db:
rec = _get(db, tid)
assert rec is not None and rec.status == "success"
assert rec.best_platform_id == "meituan"
assert rec.saved_amount_cents == 500
def test_trace_finalize_harvests_abort(client) -> None:
tid = _tid()
with SessionLocal() as db: # 先有 running 行(帧0建的)
crud.harvest_running(db, trace_id=tid, user_id=None)
p, _cap = _mock_pricebot({"trace_url": "https://price.shaguabijia.com/traces/fin/"})
with p:
r = client.post("/api/v1/trace/finalize",
json={"trace_id": tid, "status": "cancelled", "reason": "用户终止"})
assert r.status_code == 200
with SessionLocal() as db:
rec = _get(db, tid)
assert rec is not None and rec.status == "cancelled"
assert rec.trace_url.endswith("/fin/")
def test_price_step_binds_user_when_authed(client) -> None:
"""带 JWT 的首帧 → running 行绑上 user_id。"""
client.post("/api/v1/auth/sms/send", json={"phone": "13800009001"})
token = client.post("/api/v1/auth/sms/login",
json={"phone": "13800009001", "code": "123456"}).json()["access_token"]
wait_frame = {"success": True, "action": {"command": "wait", "params": {}},
"continue": True, "trace_url": "https://price.shaguabijia.com/traces/auth/"}
p, _cap = _mock_pricebot(wait_frame)
with p:
r = client.post("/api/v1/price/step", json=_stub_body(),
headers={"Authorization": f"Bearer {token}"})
tid = r.json()["trace_id"]
with SessionLocal() as db:
rec = _get(db, tid)
assert rec is not None and rec.user_id is not None # 绑上了登录用户
+2 -4
View File
@@ -20,10 +20,8 @@ def _auth(token: str) -> dict[str, str]:
def _report_success(client, token: str, trace_id: str) -> None:
"""上报一条成功比价(有非源有效价 → status 派生 success)。"""
# trace_id 现全局唯一(prod 由 app-server 签发 UUID);测试共用 session DB,故按 token
# (每用户不同)加前缀防跨用户/跨文件复用 "s-1" 撞车(见 comparison_record_trace_unique 迁移)。
payload = {
"trace_id": f"{token[-16:]}:{trace_id}",
"trace_id": trace_id,
"business_type": "food",
"store_name": "测试店",
"source_platform_id": "taobao_flash",
@@ -42,7 +40,7 @@ def _report_success(client, token: str, trace_id: str) -> None:
def _report_failed(client, token: str, trace_id: str) -> None:
"""上报一条失败比价(只有源 → status 派生 failed,不计入解锁)。"""
payload = {
"trace_id": f"{token[-16:]}:{trace_id}",
"trace_id": trace_id,
"business_type": "food",
"comparison_results": [
{"platform_id": "taobao_flash", "platform_name": "淘宝闪购", "price": 30.0,
+6 -13
View File
@@ -9,7 +9,6 @@ mock 掉对 pricebot 的 httpx 调用,验证(两个端点参数化同跑):
"""
from __future__ import annotations
import json
from unittest.mock import MagicMock, patch
import httpx
@@ -42,8 +41,7 @@ def _stub_body() -> dict:
@pytest.mark.parametrize("path,upstream", ENDPOINTS)
def test_passes_body_through(client, path, upstream) -> None:
"""无 token(软鉴权放行)+ pricebot 200 → 响应透传(+ app-server 注入 trace_id),
body 原样转发(自带 trace_id 时不重签),URL 去掉 /v1"""
"""无 token + pricebot 200 → 响应原样透传,body 原样转发,URL 去掉 /v1。"""
fake_resp = {
"success": True,
"action": {"command": "wait", "params": {"duration_ms": 500}},
@@ -51,25 +49,20 @@ def test_passes_body_through(client, path, upstream) -> None:
}
captured: dict = {}
async def fake_post(self, url, content=None, **kw):
# 透传壳用 content=raw(bytes)转发,不是 json=;这里捕获 content 解回来核对
async def fake_post(self, url, json=None, **kw):
captured["url"] = url
captured["content"] = content
captured["json"] = json
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json = lambda: dict(fake_resp) # 每次新副本(端点会 setdefault trace_id 进去)
mock_resp.json = lambda: fake_resp
return mock_resp
with patch.object(httpx.AsyncClient, "post", fake_post):
r = client.post(path, json=_stub_body())
assert r.status_code == 200, r.text
body = r.json()
assert body["success"] is True
assert body["action"] == fake_resp["action"]
assert body["trace_id"] == "test-trace-1" # 自带 trace_id → 原样回传, 不 mint
# body 原样转发(content=raw bytes;自带 trace_id 时不重新序列化)
assert json.loads(captured["content"]) == _stub_body()
assert r.json() == fake_resp
assert captured["json"] == _stub_body() # body 原样转发(不鉴权,无需 token)
assert captured["url"].endswith(upstream) # /api/v1/xxx → /api/xxx
+234
View File
@@ -0,0 +1,234 @@
"""接口指标可观测(observe)单测:配置门槛 / 队列 / 中间件 / worker。
沿用仓库约定:TestClient + monkeypatch,绝不打真网络observe 默认关(conftest 未设
OBSERVE_*),需要开启的用例用 monkeypatch settings 单例属性
"""
from __future__ import annotations
import asyncio
import httpx
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app.core import observe, observe_worker
from app.core.config import settings
def test_observe_configured_requires_switch_and_creds(monkeypatch):
# 开关开 + endpoint(默认 localhost)+ user + password 齐全 → True
monkeypatch.setattr(settings, "OBSERVE_ENABLED", True)
monkeypatch.setattr(settings, "OBSERVE_USER", "u")
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "p")
assert settings.observe_configured is True
# 缺密码 → False
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "")
assert settings.observe_configured is False
# 缺用户名 → False
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "p")
monkeypatch.setattr(settings, "OBSERVE_USER", "")
assert settings.observe_configured is False
# 开关关 → False(即便凭证齐全)
monkeypatch.setattr(settings, "OBSERVE_USER", "u")
monkeypatch.setattr(settings, "OBSERVE_ENABLED", False)
assert settings.observe_configured is False
def test_record_event_enqueues(monkeypatch):
q = asyncio.Queue(maxsize=10)
monkeypatch.setattr(observe, "_queue", q)
observe.record_event({"route": "/x"})
assert q.get_nowait() == {"route": "/x"}
def test_record_event_drops_when_full(monkeypatch):
q = asyncio.Queue(maxsize=1)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(observe, "_dropped", 0)
observe.record_event({"n": 1}) # 占满
observe.record_event({"n": 2}) # 满 → 丢弃当前,不抛异常
assert observe.take_dropped() == 1
assert observe.take_dropped() == 0 # 取出后清零
assert q.get_nowait() == {"n": 1} # 保留的是先到的
def _make_probe_app() -> FastAPI:
"""独立最小 app:只挂中间件 + 两个无鉴权路由,不碰真业务 DB/auth。"""
app = FastAPI()
app.add_middleware(observe.RequestMetricsMiddleware)
@app.get("/things/{tid}")
def get_thing(tid: str):
return {"tid": tid}
@app.get("/health")
def health():
return {"ok": True}
return app
@pytest.fixture
def observe_on(monkeypatch):
"""开启观测 + 换一个干净小队列,返回该队列供断言。"""
monkeypatch.setattr(settings, "OBSERVE_ENABLED", True)
monkeypatch.setattr(settings, "OBSERVE_USER", "u")
monkeypatch.setattr(settings, "OBSERVE_PASSWORD", "p")
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
return q
def test_middleware_records_route_template(observe_on):
client = TestClient(_make_probe_app())
r = client.get("/things/42")
assert r.status_code == 200
evt = observe_on.get_nowait()
assert evt["route"] == "/things/{tid}" # 模板,不是 /things/42
assert evt["method"] == "GET"
assert evt["status"] == 200
assert evt["duration_ms"] >= 0
assert evt["service"] and "env" in evt and isinstance(evt["_timestamp"], int)
def test_middleware_skips_health(observe_on):
client = TestClient(_make_probe_app())
client.get("/health")
assert observe_on.empty()
def test_middleware_unmatched_route_is_normalized(observe_on):
client = TestClient(_make_probe_app())
r = client.get("/definitely-not-a-route")
assert r.status_code == 404
evt = observe_on.get_nowait()
assert evt["route"] == "__unmatched__"
assert evt["status"] == 404
def test_middleware_noop_when_disabled(monkeypatch):
monkeypatch.setattr(settings, "OBSERVE_ENABLED", False)
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
client = TestClient(_make_probe_app())
client.get("/things/1")
assert q.empty() # 未配置观测 → 零入队
async def test_collect_batch_drains_up_to_batch_max(monkeypatch):
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(settings, "OBSERVE_FLUSH_INTERVAL_SEC", 0.1)
monkeypatch.setattr(settings, "OBSERVE_BATCH_MAX", 200)
for i in range(3):
q.put_nowait({"n": i})
batch = await observe_worker._collect_batch()
assert [e["n"] for e in batch] == [0, 1, 2]
async def test_collect_batch_timeout_returns_empty(monkeypatch):
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(settings, "OBSERVE_FLUSH_INTERVAL_SEC", 0.05)
batch = await observe_worker._collect_batch()
assert batch == []
async def test_post_batch_hits_json_ingest_url(monkeypatch):
monkeypatch.setattr(settings, "OBSERVE_ORG", "default")
monkeypatch.setattr(settings, "OBSERVE_STREAM", "app_requests")
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["url"] = str(request.url)
captured["json"] = request.content
return httpx.Response(200, json={"code": 200})
client = httpx.AsyncClient(
base_url="http://oo", transport=httpx.MockTransport(handler)
)
await observe_worker._post_batch(client, [{"route": "/x", "status": 200}])
await client.aclose()
assert captured["url"] == "http://oo/api/default/app_requests/_json"
assert b"/x" in captured["json"]
def test_start_observe_worker_noop_when_not_configured(monkeypatch):
monkeypatch.setattr(settings, "OBSERVE_ENABLED", False)
assert observe_worker.start_observe_worker() is None
async def test_run_loop_survives_post_failure(monkeypatch):
"""_post_batch 抛异常时,loop 不崩溃、继续处理后续批次(best-effort 契约)。"""
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(settings, "OBSERVE_FLUSH_INTERVAL_SEC", 0.02)
monkeypatch.setattr(settings, "OBSERVE_BATCH_MAX", 200)
seen: list[list[int]] = []
async def boom(client, batch):
seen.append([e["n"] for e in batch])
raise RuntimeError("boom")
monkeypatch.setattr(observe_worker, "_post_batch", boom)
q.put_nowait({"n": 1})
task = asyncio.create_task(observe_worker._run_loop(None))
try:
for _ in range(50): # 轮询直到第 1 批被处理(失败),最多等 0.5s
await asyncio.sleep(0.01)
if seen:
break
q.put_nowait({"n": 2})
for _ in range(50): # 第 2 批被处理 → 证明失败后 loop 仍存活
await asyncio.sleep(0.01)
if len(seen) >= 2:
break
finally:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
assert seen == [[1], [2]]
async def test_stop_flushes_remaining_and_closes_client(monkeypatch):
"""stop:cancel 后把剩余事件 best-effort 发出最后一批,并关闭 + 置空 client。"""
q = asyncio.Queue(maxsize=100)
monkeypatch.setattr(observe, "_queue", q)
monkeypatch.setattr(settings, "OBSERVE_ORG", "default")
monkeypatch.setattr(settings, "OBSERVE_STREAM", "app_requests")
monkeypatch.setattr(settings, "OBSERVE_BATCH_MAX", 200)
q.put_nowait({"n": 1})
q.put_nowait({"n": 2})
posted: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
posted["body"] = request.content
return httpx.Response(200, json={"code": 200})
client = httpx.AsyncClient(
base_url="http://oo", transport=httpx.MockTransport(handler)
)
monkeypatch.setattr(observe_worker, "_client", client)
async def _noop() -> None:
return None
task = asyncio.create_task(_noop())
await observe_worker.stop_observe_worker(task)
assert b'"n"' in posted["body"] # 关停时把剩余事件发了出去
assert observe_worker._client is None # client 已关闭并置空
def test_app_has_metrics_middleware():
from app.main import app
names = [m.cls.__name__ for m in app.user_middleware]
assert "RequestMetricsMiddleware" in names
-25
View File
@@ -65,28 +65,3 @@ def test_onboarding_missing_device_id_treated_as_incomplete(client) -> None:
phone = "13800138003"
data = _login(client, phone, device_id=None)
assert data["onboarding_completed"] is False
def _reset(client, access: str, device_id: str):
return client.post(
"/api/v1/user/onboarding/reset",
json={"device_id": device_id},
headers={"Authorization": f"Bearer {access}"},
)
def test_onboarding_reset_makes_relogin_reonboard(client) -> None:
"""重置(删完成标记)后,同 (账号, 设备) 再登录 → onboarding_completed=False,重走引导。
对应客户端开发设置 重置新手引导:先删后端行,再退登录重开"""
phone = "13800138004"
dev = "android-id-reset"
# ① 走完引导 → 标记完成 → 再登录已完成
access = _login(client, phone, dev)["access_token"]
assert _mark_complete(client, access, dev).status_code == 200
assert _login(client, phone, dev)["onboarding_completed"] is True
# ② 重置(幂等:重复重置也 200)→ 再登录变未完成 → 重走引导
assert _reset(client, access, dev).status_code == 200
assert _reset(client, access, dev).json()["ok"] is True
assert _login(client, phone, dev)["onboarding_completed"] is False