From f7d86011c1c76114c0ffbd1ae127dbec783f58c6 Mon Sep 17 00:00:00 2001 From: ouzhou Date: Mon, 15 Jun 2026 23:13:14 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat(ad-revenue):=20admin=20=E5=B9=BF?= =?UTF-8?q?=E5=91=8A=E6=94=B6=E7=9B=8A=E6=8A=A5=E8=A1=A8(=E6=8C=89=20?= =?UTF-8?q?=E7=94=A8=E6=88=B7/=E6=97=A5=E6=9C=9F/=E7=B1=BB=E5=9E=8B/?= =?UTF-8?q?=E5=BA=94=E7=94=A8/=E4=BB=A3=E7=A0=81=E4=BD=8D=20=E8=81=9A?= =?UTF-8?q?=E5=90=88)=20(#54)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 GET /admin/api/ad-revenue-report:展示条数/收益 + 复用金币审计逐条复算做发奖对账 - ad_ecpm/ad_reward/ad_feed_reward 各加 app_env + our_code_id 两列(alembic 迁移) - ecpm-report / feed-reward 接收并落库 app_env/our_code_id;激励发奖按 ad_session_id 回填 - ad_audit 抽出 audit_rows,报表与逐条审计复用同一复算口径 - 组级 matched 改「组内逐条全一致」,避免应发和==实发和的互相抵消掩盖错误 - list_feedbacks 改 offset 分页并返回 total(配合 admin 页码分页) - 反馈正文上限 _CONTENT_MAX 2000→200 - 文档:新增 admin-ad-revenue-report,更新 ecpm/feed-reward/feedback 及对应 db docs Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: OuYingJun1024 <1034284404@qq.com> Reviewed-on: https://gitea.shaguabijia.com/WonderableAI/shaguabijia-app-server/pulls/54 Co-authored-by: ouzhou Co-committed-by: ouzhou --- alembic/versions/ad_revenue_report_cols.py | 39 +++ ...merge_ad_revenue_report_cols_and_store_.py | 26 ++ app/admin/main.py | 2 + app/admin/repositories/ad_audit.py | 30 ++- app/admin/repositories/ad_revenue.py | 239 ++++++++++++++++++ app/admin/repositories/queries.py | 14 +- app/admin/routers/ad_revenue.py | 75 ++++++ app/admin/routers/feedback.py | 6 +- app/admin/schemas/ad_revenue.py | 90 +++++++ app/api/v1/ad.py | 36 ++- app/api/v1/feedback.py | 2 +- app/models/ad_ecpm.py | 5 + app/models/ad_feed_reward.py | 3 + app/models/ad_reward.py | 4 + app/repositories/ad_ecpm.py | 10 +- app/repositories/ad_feed_reward.py | 57 ++++- app/repositories/ad_reward.py | 66 ++++- app/schemas/ad.py | 54 +++- docs/api/README.md | 2 + docs/api/ad-ecpm-report.md | 10 +- docs/api/ad-feed-reward.md | 21 +- docs/api/ad-reward-noshow.md | 41 +++ docs/api/admin-ad-revenue-report.md | 108 ++++++++ docs/api/feedback.md | 4 +- docs/database/ad_ecpm_record.md | 10 +- docs/database/ad_feed_reward_record.md | 12 +- docs/database/ad_reward_record.md | 10 +- 27 files changed, 915 insertions(+), 61 deletions(-) create mode 100644 alembic/versions/ad_revenue_report_cols.py create mode 100644 alembic/versions/d4e68464761d_merge_ad_revenue_report_cols_and_store_.py create mode 100644 app/admin/repositories/ad_revenue.py create mode 100644 app/admin/routers/ad_revenue.py create mode 100644 app/admin/schemas/ad_revenue.py create mode 100644 docs/api/ad-reward-noshow.md create mode 100644 docs/api/admin-ad-revenue-report.md diff --git a/alembic/versions/ad_revenue_report_cols.py b/alembic/versions/ad_revenue_report_cols.py new file mode 100644 index 0000000..afbc8b4 --- /dev/null +++ b/alembic/versions/ad_revenue_report_cols.py @@ -0,0 +1,39 @@ +"""ad revenue report columns: app_env + our_code_id + +给 ad_ecpm_record / ad_reward_record / ad_feed_reward_record 各加两列: +- app_env:我们的穿山甲应用环境(prod=傻瓜比价正式 / test=测试应用) +- our_code_id:我们在穿山甲后台配置的代码位 ID(104xxx,非底层 mediation rit) + +供「广告收益报表」按 用户/日期/广告类型/应用/代码位 聚合 展示条数/收益/金币。 +旧数据这两列为 NULL(报表里来源列留空),新数据由客户端上报/发奖时回填。 + +Revision ID: ad_revenue_report_cols +Revises: coupon_engage_per_package +Create Date: 2026-06-15 +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "ad_revenue_report_cols" +down_revision = "coupon_engage_per_package" +branch_labels = None +depends_on = None + + +_TABLES = ("ad_ecpm_record", "ad_reward_record", "ad_feed_reward_record") + + +def upgrade() -> None: + for table in _TABLES: + op.add_column(table, sa.Column("app_env", sa.String(length=16), nullable=True)) + op.add_column(table, sa.Column("our_code_id", sa.String(length=64), nullable=True)) + + +def downgrade() -> None: + for table in _TABLES: + op.drop_column(table, "our_code_id") + op.drop_column(table, "app_env") diff --git a/alembic/versions/d4e68464761d_merge_ad_revenue_report_cols_and_store_.py b/alembic/versions/d4e68464761d_merge_ad_revenue_report_cols_and_store_.py new file mode 100644 index 0000000..16fc94c --- /dev/null +++ b/alembic/versions/d4e68464761d_merge_ad_revenue_report_cols_and_store_.py @@ -0,0 +1,26 @@ +"""merge ad_revenue_report_cols and store_mapping_tb_dl_invalid heads + +Revision ID: d4e68464761d +Revises: ad_revenue_report_cols, store_mapping_tb_dl_invalid +Create Date: 2026-06-15 21:55:58.115692 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd4e68464761d' +down_revision: Union[str, Sequence[str], None] = ('ad_revenue_report_cols', 'store_mapping_tb_dl_invalid') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/app/admin/main.py b/app/admin/main.py index a422885..c65aa9e 100644 --- a/app/admin/main.py +++ b/app/admin/main.py @@ -14,6 +14,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from app.admin.routers.ad_audit import router as ad_audit_router +from app.admin.routers.ad_revenue import router as ad_revenue_router from app.admin.routers.admins import router as admins_router from app.admin.routers.audit import router as audit_router from app.admin.routers.auth import router as auth_router @@ -90,3 +91,4 @@ admin_app.include_router(admins_router) admin_app.include_router(audit_router) admin_app.include_router(config_router) admin_app.include_router(ad_audit_router) +admin_app.include_router(ad_revenue_router) diff --git a/app/admin/repositories/ad_audit.py b/app/admin/repositories/ad_audit.py index e08ae68..7cb2941 100644 --- a/app/admin/repositories/ad_audit.py +++ b/app/admin/repositories/ad_audit.py @@ -67,6 +67,8 @@ def _reward_video_rows( "scene": "reward_video", "record_id": rec.id, "user_id": rec.user_id, + "app_env": rec.app_env, + "our_code_id": rec.our_code_id, "created_at": rec.created_at, "status": rec.status, "ecpm": rec.ecpm_raw, @@ -86,6 +88,8 @@ def _reward_video_rows( "scene": "reward_video", "record_id": rec.id, "user_id": rec.user_id, + "app_env": rec.app_env, + "our_code_id": rec.our_code_id, "created_at": rec.created_at, "status": rec.status, "ecpm": rec.ecpm_raw, @@ -150,6 +154,8 @@ def _feed_rows(db: Session, *, date: str, user_id: int | None) -> list[dict]: "scene": "feed", "record_id": rec.id, "user_id": rec.user_id, + "app_env": rec.app_env, + "our_code_id": rec.our_code_id, "created_at": rec.created_at, "status": rec.status, "ecpm": rec.ecpm_raw, @@ -168,6 +174,8 @@ def _feed_rows(db: Session, *, date: str, user_id: int | None) -> list[dict]: "scene": "feed", "record_id": rec.id, "user_id": rec.user_id, + "app_env": rec.app_env, + "our_code_id": rec.our_code_id, "created_at": rec.created_at, "status": rec.status, "ecpm": rec.ecpm_raw, @@ -184,6 +192,22 @@ def _feed_rows(db: Session, *, date: str, user_id: int | None) -> list[dict]: return rows +def audit_rows( + db: Session, *, date: str, user_id: int | None, scene: str | None = None +) -> list[dict]: + """当日逐条发奖复算行(未排序)。scene: None=两类 / "reward_video" / "feed"。 + + 每行含 `app_env`/`our_code_id`/`expected_coin`/`actual_coin` 等,供金币审计逐条对账, + 也供广告收益报表把「应发/实发」按 用户×类型×应用×代码位 聚合(见 ad_revenue,复用同一复算口径)。 + """ + rows: list[dict] = [] + if scene in (None, "reward_video"): + rows.extend(_reward_video_rows(db, date=date, user_id=user_id)) + if scene in (None, "feed"): + rows.extend(_feed_rows(db, date=date, user_id=user_id)) + return rows + + def ad_coin_audit( db: Session, *, @@ -200,11 +224,7 @@ def ad_coin_audit( 影响;`items` 才是展示集(only_mismatch 时只取 ✗ 行)按 created_at 倒序截断到 limit。 份序号在全天数据上已算好,limit 只影响展示条数、不影响 expected 复算正确性。 """ - rows: list[dict] = [] - if scene in (None, "reward_video"): - rows.extend(_reward_video_rows(db, date=date, user_id=user_id)) - if scene in (None, "feed"): - rows.extend(_feed_rows(db, date=date, user_id=user_id)) + rows = audit_rows(db, date=date, user_id=user_id, scene=scene) rows.sort(key=lambda r: (r["created_at"], r["record_id"]), reverse=True) total = len(rows) diff --git a/app/admin/repositories/ad_revenue.py b/app/admin/repositories/ad_revenue.py new file mode 100644 index 0000000..206726b --- /dev/null +++ b/app/admin/repositories/ad_revenue.py @@ -0,0 +1,239 @@ +"""admin 广告收益报表:按 用户 / 日期 / 广告类型 / 应用 / 代码位 聚合(单表含发奖对账)。 + +只读。聚合键 = user_id × ad_type × app_env × our_code_id;每组一行同时给出: +- 展示条数 + 收益:`ad_ecpm_record`(每行 = 客户端一次广告展示;收益 = Σ eCPM元 ÷ 1000)。 + 激励视频每次展示上报一行;信息流轮播每条展示各上报一行(每条独立 id,不复用会话)。 +- 应发金币 / 实发金币:复用金币审计的**逐条复算**(`ad_audit.audit_rows`,与正式发奖同一公式口径, + 不另写公式),把每条发奖记录的 expected/actual 按同维度求和;`matched` = 组内**逐条**全部一致 + (任一条不符该组即不符,不用「应发和==实发和」以免互相抵消掩盖错误)。**不改发奖逻辑**,只读复算。 + +展示与发奖来自不同表,做并集:有展示无发奖(用户中途关 / 未达发奖)、有发奖无展示 +(未上报 eCPM)都各自成行。app_env/our_code_id 旧数据为 NULL → 归到「来源未知」组。 + +⚠️ 局限:① 历史 Draw 发奖混在 ad_feed_reward_record 无类型标记,金币侧统一记 `feed`(迁移后 Draw +不再产生新数据)。② 聚合级只能看出「某组应发≠实发」,定位到具体哪条仍需逐条审计接口(ad-coin-audit)。 +""" +from __future__ import annotations + +from datetime import date as _date, datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.admin.repositories import ad_audit +from app.core import rewards +from app.models.ad_ecpm import AdEcpmRecord + + +def _cn_hour(dt: datetime) -> int: + """created_at(UTC 口径)→ 北京时间小时(0–23)。naive 当 UTC 处理(sqlite),tz-aware 直接换算(pg)。""" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(rewards.CN_TZ).hour + + +def _key( + report_date: str, + user_id: int, + ad_type: str, + app_env: str | None, + our_code_id: str | None, + hour: int | None, +) -> tuple: + return (report_date, user_id, ad_type, app_env or None, our_code_id or None, hour) + + +def _date_range(date_from: str, date_to: str) -> list[str]: + """闭区间内逐日 'YYYY-MM-DD' 串(含首尾)。date_from > date_to 时返回空。""" + d0 = _date.fromisoformat(date_from) + d1 = _date.fromisoformat(date_to) + out: list[str] = [] + d = d0 + while d <= d1: + out.append(d.isoformat()) + d += timedelta(days=1) + return out + + +# 审计行的 scene 与报表 ad_type 一一对应 +_SCENE_TO_AD_TYPE = {"reward_video": "reward_video", "feed": "feed"} + + +def ad_revenue_report( + db: Session, + *, + date_from: str, + date_to: str, + user_id: int | None = None, + ad_type: str | None = None, + granularity: str = "day", + limit: int = 500, +) -> dict: + """日期区间(北京时间,闭区间)广告收益聚合 + 发奖对账。单日时 date_from==date_to。 + + 聚合键含**日期**:report_date × user × ad_type × app_env × our_code_id(× 北京小时,granularity=hour)。 + ad_type: None=全部 / reward_video / feed / draw。 + granularity: "day"=按天 / "hour"=按小时(聚合键再加北京小时 0–23,每组一行)。 + limit 只截断展示明细,total 与 total_* / daily 在全量上统计(不受 limit 影响),数字始终可信。 + + 返回额外含 `daily`(按日期汇总的展示/收益/应发/实发,供前端按天趋势图;不受 limit 影响)。 + + 注:按小时下,展示按 ecpm 记录的小时、金币按发奖记录的小时各自归桶——S2S 回调可能比展示晚 + 一会儿,故同一次广告的展示与金币偶尔落相邻小时(按天则一致)。 + """ + by_hour = granularity == "hour" + groups: dict[tuple, dict] = {} + + def _grp(key: tuple) -> dict: + g = groups.get(key) + if g is None: + rdate, uid, atype, app_env, code_id, hour = key + g = { + "report_date": rdate, + "user_id": uid, + "ad_type": atype, + "app_env": app_env, + "our_code_id": code_id, + "hour": hour, + "impressions": 0, + "revenue_yuan": 0.0, + "expected_coin": 0, + "actual_coin": 0, + "adns": set(), + "impression_records": [], # 该组逐条展示明细(展开下钻用) + "records": [], # 该组逐条发奖复算明细(展开下钻用) + } + groups[key] = g + return g + + # 1) 展示条数 + 收益 ← ad_ecpm_record(report_date 闭区间;字符串 YYYY-MM-DD 字典序即日期序) + stmt = select(AdEcpmRecord).where( + AdEcpmRecord.report_date >= date_from, + AdEcpmRecord.report_date <= date_to, + ) + if user_id is not None: + stmt = stmt.where(AdEcpmRecord.user_id == user_id) + if ad_type is not None: + stmt = stmt.where(AdEcpmRecord.ad_type == ad_type) + for rec in db.execute(stmt).scalars(): + hour = _cn_hour(rec.created_at) if by_hour else None + g = _grp(_key(rec.report_date, rec.user_id, rec.ad_type, rec.app_env, rec.our_code_id, hour)) + g["impressions"] += 1 + # 单次展示收益(元) = eCPM元 ÷ 1000(每千次→单次);用与发奖同源的解析,口径一致。 + rev = rewards.parse_ecpm_yuan(rec.ecpm_raw) / 1000.0 + g["revenue_yuan"] += rev + if rec.adn: + g["adns"].add(rec.adn) + g["impression_records"].append({ + "id": rec.id, + "created_at": rec.created_at, + "ecpm": rec.ecpm_raw, + "revenue_yuan": round(rev, 6), + "adn": rec.adn, + "slot_id": rec.slot_id, + }) + + # 2) 应发 / 实发金币 ← 复用金币审计逐条复算(同一公式口径),按同维度求和。 + # audit_rows 是单日的,区间逐日调用,每天的行归到当天 report_date(语义与单日报表完全一致)。 + # ad_type=draw 时审计无对应记录(scene 只有 reward_video/feed),金币侧自然为空。 + audit_scene = _SCENE_TO_AD_TYPE.get(ad_type) if ad_type is not None else None + if ad_type is None or audit_scene is not None: + for d in _date_range(date_from, date_to): + for row in ad_audit.audit_rows(db, date=d, user_id=user_id, scene=audit_scene): + atype = _SCENE_TO_AD_TYPE.get(row["scene"], row["scene"]) + hour = _cn_hour(row["created_at"]) if by_hour else None + g = _grp(_key(d, row["user_id"], atype, row.get("app_env"), row.get("our_code_id"), hour)) + g["expected_coin"] += int(row["expected_coin"]) + g["actual_coin"] += int(row["actual_coin"]) + # 逐条明细(eCPM/因子1/份数/LT/因子2/应发/实发/一致)——前端展开该组时下钻展示。 + g["records"].append({ + "record_id": row["record_id"], + "created_at": row["created_at"], + "status": row["status"], + "ecpm": row["ecpm"], + "ecpm_factor": row["ecpm_factor"], + "units": row["units"], + "lt_index_start": row["lt_index_start"], + "lt_index_end": row["lt_index_end"], + "lt_factor_start": row["lt_factor_start"], + "lt_factor_end": row["lt_factor_end"], + "expected_coin": row["expected_coin"], + "actual_coin": row["actual_coin"], + "matched": row["matched"], + }) + + rows = list(groups.values()) + rows.sort( + key=lambda r: ( + r["report_date"], + r["user_id"], + r["hour"] if r["hour"] is not None else -1, + r["ad_type"] or "", + r["our_code_id"] or "", + ) + ) + + total_impressions = sum(r["impressions"] for r in rows) + total_expected_coin = sum(r["expected_coin"] for r in rows) + total_actual_coin = sum(r["actual_coin"] for r in rows) + total_revenue_yuan = round(sum(r["revenue_yuan"] for r in rows), 6) + + # 按日期汇总(全量,不受 limit):供前端按天趋势图。 + daily_map: dict[str, dict] = {} + for r in rows: + d = daily_map.get(r["report_date"]) + if d is None: + d = { + "date": r["report_date"], + "impressions": 0, + "revenue_yuan": 0.0, + "expected_coin": 0, + "actual_coin": 0, + } + daily_map[r["report_date"]] = d + d["impressions"] += r["impressions"] + d["revenue_yuan"] += r["revenue_yuan"] + d["expected_coin"] += r["expected_coin"] + d["actual_coin"] += r["actual_coin"] + daily = [ + {**d, "revenue_yuan": round(d["revenue_yuan"], 6)} + for d in sorted(daily_map.values(), key=lambda x: x["date"]) + ] + + items = [ + { + "report_date": r["report_date"], + "user_id": r["user_id"], + "ad_type": r["ad_type"], + "app_env": r["app_env"], + "our_code_id": r["our_code_id"], + "hour": r["hour"], + "impressions": r["impressions"], + "revenue_yuan": round(r["revenue_yuan"], 6), + "expected_coin": r["expected_coin"], + "actual_coin": r["actual_coin"], + # 组内**逐条**全部一致才记一致——不能用「应发和==实发和」,否则一条多发+一条少发会互相 + # 抵消、求和相等被误判为 ✓,掩盖真实发奖错误。纯展示无发奖记录的组 all([]) → True。 + "matched": all(rec["matched"] for rec in r["records"]), + "adns": sorted(r["adns"]), + "impression_records": sorted( + r["impression_records"], key=lambda x: (x["created_at"], x["id"]) + ), + "records": sorted(r["records"], key=lambda x: (x["created_at"], x["record_id"])), + } + for r in rows[:limit] + ] + + return { + "total": len(rows), + "truncated": len(rows) > limit, + "total_impressions": total_impressions, + "total_revenue_yuan": total_revenue_yuan, + "total_expected_coin": total_expected_coin, + "total_actual_coin": total_actual_coin, + "mismatch_count": sum( + 1 for r in rows if not all(rec["matched"] for rec in r["records"]) + ), + "daily": daily, + "items": items, + } diff --git a/app/admin/repositories/queries.py b/app/admin/repositories/queries.py index a4721e0..6e39192 100644 --- a/app/admin/repositories/queries.py +++ b/app/admin/repositories/queries.py @@ -286,10 +286,11 @@ def list_feedbacks( sort_order: str = "desc", limit: int = 20, cursor: int | None = None, -) -> tuple[list[Feedback], int | None]: +) -> tuple[list[Feedback], int | None, int]: """反馈工单列表。支持 状态 / 用户ID / 内容模糊 / 提交时间范围 筛选,按 id·提交时间排序。 **offset 分页**(cursor=offset):任意列排序下游标语义统一(同 [list_users]),代价是翻页期间 - 数据变动可能错位一条——admin 低频场景可接受。created_at 为 timestamptz,日期入参统一转 tz-aware UTC 比较。""" + 数据变动可能错位一条——admin 低频场景可接受。返回 (items, next_cursor, total),total 供页码分页。 + created_at 为 timestamptz,日期入参统一转 tz-aware UTC 比较。""" stmt = select(Feedback) if status: stmt = stmt.where(Feedback.status == status) @@ -306,14 +307,7 @@ def list_feedbacks( sort_col = sort_cols.get(sort_by, Feedback.id) order_fn = asc if sort_order == "asc" else desc id_order = asc(Feedback.id) if sort_order == "asc" else desc(Feedback.id) - stmt = stmt.order_by(order_fn(sort_col), id_order) - - offset = max(cursor or 0, 0) - rows = list(db.execute(stmt.offset(offset).limit(limit + 1)).scalars().all()) - has_more = len(rows) > limit - items = rows[:limit] - next_cursor = offset + limit if has_more else None - return items, next_cursor + return offset_paginate(db, stmt, (order_fn(sort_col), id_order), limit=limit, cursor=cursor) def get_withdraw_by_out_bill_no(db: Session, out_bill_no: str) -> WithdrawOrder | None: diff --git a/app/admin/routers/ad_revenue.py b/app/admin/routers/ad_revenue.py new file mode 100644 index 0000000..1a67270 --- /dev/null +++ b/app/admin/routers/ad_revenue.py @@ -0,0 +1,75 @@ +"""admin 广告收益报表:按 用户/日期/广告类型/应用/代码位 聚合 展示条数 / 收益 / 金币。 + +任意已登录 admin 可看(只读,不涉及资金操作)。聚合逻辑在 app/admin/repositories/ad_revenue.py。 +""" +from __future__ import annotations + +from datetime import date as _date +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query + +from app.admin.deps import AdminDb, get_current_admin +from app.admin.repositories import ad_revenue +from app.admin.schemas.ad_revenue import AdRevenueDaily, AdRevenueReportOut, AdRevenueRow +from app.core.rewards import cn_today + +router = APIRouter( + prefix="/admin/api/ad-revenue-report", + tags=["admin-ad-revenue-report"], + dependencies=[Depends(get_current_admin)], +) + +# 区间最大跨度(天);超出拒绝,避免审计页一次拉过多天(逐日审计 + 大查询)拖垮接口。 +_MAX_RANGE_DAYS = 92 + + +def _parse_day(value: str | None, *, field: str, default: _date) -> _date: + if value is None: + return default + try: + return _date.fromisoformat(value) + except ValueError as e: + raise HTTPException(status_code=422, detail=f"{field} 需为 YYYY-MM-DD") from e + + +@router.get("", response_model=AdRevenueReportOut, summary="广告收益报表(按 日期区间/用户/类型/应用/代码位 聚合)") +def get_ad_revenue_report( + db: AdminDb, + date_from: Annotated[str | None, Query(description="起始日 北京时间 YYYY-MM-DD,默认今天")] = None, + date_to: Annotated[str | None, Query(description="结束日 北京时间 YYYY-MM-DD,闭区间,默认=date_from")] = None, + user_id: Annotated[int | None, Query(description="只看某用户;不传=全部用户")] = None, + ad_type: Annotated[ + str | None, + Query(description="reward_video / feed / draw;不传=全部类型"), + ] = None, + granularity: Annotated[ + str, Query(description="day=按天 / hour=按小时(北京时间);区间>1 天建议用 day") + ] = "day", + limit: Annotated[int, Query(ge=1, le=1000)] = 500, +) -> AdRevenueReportOut: + today = cn_today() + d_from = _parse_day(date_from, field="date_from", default=today) + d_to = _parse_day(date_to, field="date_to", default=d_from) + if d_to < d_from: + raise HTTPException(status_code=422, detail="date_to 不能早于 date_from") + if (d_to - d_from).days + 1 > _MAX_RANGE_DAYS: + raise HTTPException(status_code=422, detail=f"区间最长 {_MAX_RANGE_DAYS} 天") + + result = ad_revenue.ad_revenue_report( + db, date_from=d_from.isoformat(), date_to=d_to.isoformat(), + user_id=user_id, ad_type=ad_type, granularity=granularity, limit=limit, + ) + return AdRevenueReportOut( + date_from=d_from.isoformat(), + date_to=d_to.isoformat(), + daily=[AdRevenueDaily(**d) for d in result["daily"]], + total=result["total"], + truncated=result["truncated"], + total_impressions=result["total_impressions"], + total_revenue_yuan=result["total_revenue_yuan"], + total_expected_coin=result["total_expected_coin"], + total_actual_coin=result["total_actual_coin"], + mismatch_count=result["mismatch_count"], + items=[AdRevenueRow(**r) for r in result["items"]], + ) diff --git a/app/admin/routers/feedback.py b/app/admin/routers/feedback.py index 51a2a29..249378d 100644 --- a/app/admin/routers/feedback.py +++ b/app/admin/routers/feedback.py @@ -34,7 +34,7 @@ def list_feedbacks( limit: Annotated[int, Query(ge=1, le=100)] = 20, cursor: Annotated[int | None, Query()] = None, ) -> CursorPage[FeedbackOut]: - items, next_cursor = queries.list_feedbacks( + items, next_cursor, total = queries.list_feedbacks( db, status=status, user_id=user_id, @@ -47,7 +47,9 @@ def list_feedbacks( cursor=cursor, ) return CursorPage( - items=[FeedbackOut.model_validate(f) for f in items], next_cursor=next_cursor, + items=[FeedbackOut.model_validate(f) for f in items], + next_cursor=next_cursor, + total=total, ) diff --git a/app/admin/schemas/ad_revenue.py b/app/admin/schemas/ad_revenue.py new file mode 100644 index 0000000..beeb132 --- /dev/null +++ b/app/admin/schemas/ad_revenue.py @@ -0,0 +1,90 @@ +"""广告收益报表 schemas。 + +按 用户 / 日期 / 广告类型 / 应用 / 代码位 聚合的只读报表:展示条数、收益(元)、金币、来源。 +字段 snake_case;收益按元(float),金币按整数。 +""" +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, Field + + +class AdRevenueImpression(BaseModel): + """聚合行下钻的单条**展示**明细(每次广告展示一条,展开该组时展示)。""" + + id: int = Field(..., description="ad_ecpm_record 主键") + created_at: datetime + ecpm: str = Field(..., description="本次展示 eCPM 原始值(分/千次展示)") + revenue_yuan: float = Field(..., description="本次展示预估收益(元)= eCPM元 ÷ 1000") + adn: str | None = Field(None, description="实际填充 ADN 子渠道(pangle/gdt…)") + slot_id: str | None = Field(None, description="底层 mediation rit(非我们配置的广告位 ID)") + + +class AdRevenueRecord(BaseModel): + """聚合行下钻的单条发奖复算明细(与金币审计同源,展开该组时展示)。""" + + record_id: int + created_at: datetime + status: str = Field(..., description="granted / capped / ecpm_missing") + ecpm: str | None = Field(None, description="本次采用的 eCPM 原始值(分/千次展示)") + ecpm_factor: float | None = Field(None, description="因子1(eCPM 档);非 granted 为空") + units: int = Field(..., description="折算份数:激励视频恒 1;信息流 = 满 10 秒份数") + lt_index_start: int | None = Field(None, description="本条占用「账号累计第几份」的起") + lt_index_end: int | None = Field(None, description="本条占用「账号累计第几份」的止;激励视频 = 起") + lt_factor_start: float | None = Field(None, description="因子2(LT)起值") + lt_factor_end: float | None = Field(None, description="因子2(LT)止值;激励视频 = 起") + expected_coin: int = Field(..., description="按公式复算应发金币") + actual_coin: int = Field(..., description="实际入账金币") + matched: bool = Field(..., description="复算与实发是否一致") + + +class AdRevenueDaily(BaseModel): + """按日期汇总的一天(供前端按天趋势图;全量,不受 limit 影响)。""" + + date: str = Field(..., description="北京时间 YYYY-MM-DD") + impressions: int = Field(..., description="当天展示条数合计") + revenue_yuan: float = Field(..., description="当天预估收益合计(元)") + expected_coin: int = Field(..., description="当天应发金币合计") + actual_coin: int = Field(..., description="当天实发金币合计") + + +class AdRevenueRow(BaseModel): + """一个聚合组(report_date × user × ad_type × app_env × our_code_id)的汇总。""" + + report_date: str = Field(..., description="该组所属日期(北京时间 YYYY-MM-DD)") + user_id: int + ad_type: str = Field(..., description="reward_video(激励视频) / feed(信息流) / draw(历史 Draw 信息流)") + app_env: str | None = Field(None, description="我们的应用:prod(傻瓜比价正式) / test(测试应用);旧数据为空") + our_code_id: str | None = Field(None, description="我们后台配置的代码位 ID(104xxx);旧数据为空") + hour: int | None = Field(None, description="北京时间小时 0–23(granularity=hour 时有值;按天为 null)") + impressions: int = Field(..., description="展示条数(每条广告展示一条;轮播每条各计一次)") + revenue_yuan: float = Field(..., description="收益(元)= Σ(eCPM元 ÷ 1000);测试应用多为 0") + expected_coin: int = Field(..., description="应发金币(按公式复算,与金币审计同源)") + actual_coin: int = Field(..., description="实发金币(实际入账,按现发奖算法)") + matched: bool = Field(..., description="该组应发==实发(组内任一条不符则 false)") + adns: list[str] = Field(default_factory=list, description="实际填充的底层 ADN 子渠道集合(如 pangle/gdt)") + impression_records: list[AdRevenueImpression] = Field( + default_factory=list, + description="该组逐条展示明细(时间/eCPM/收益/adn);展开下钻用,无发奖也有(只要有展示)", + ) + records: list[AdRevenueRecord] = Field( + default_factory=list, + description="该组逐条发奖复算明细(eCPM/因子1/份数/LT/因子2/应发/实发/一致);展开下钻用,纯展示无发奖记录的组为空", + ) + + +class AdRevenueReportOut(BaseModel): + """报表响应:全量统计 + 按天趋势 + 聚合明细。""" + + date_from: str = Field(..., description="报表起始日期(北京时间 YYYY-MM-DD)") + date_to: str = Field(..., description="报表结束日期(北京时间 YYYY-MM-DD,闭区间;单日时与 date_from 相同)") + daily: list[AdRevenueDaily] = Field(..., description="按日期汇总序列(全量,供按天趋势图)") + total: int = Field(..., description="聚合组总数(全量,不受 limit 影响)") + truncated: bool = Field(..., description="明细是否被 limit 截断") + total_impressions: int = Field(..., description="全量展示条数合计") + total_revenue_yuan: float = Field(..., description="全量收益合计(元)") + total_expected_coin: int = Field(..., description="全量应发金币合计") + total_actual_coin: int = Field(..., description="全量实发金币合计") + mismatch_count: int = Field(..., description="应发≠实发的组数(=0 说明全部按公式发放)") + items: list[AdRevenueRow] = Field(..., description="聚合明细(按 用户→类型→代码位 排序)") diff --git a/app/api/v1/ad.py b/app/api/v1/ad.py index 0c7e351..9508061 100644 --- a/app/api/v1/ad.py +++ b/app/api/v1/ad.py @@ -32,6 +32,8 @@ from app.schemas.ad import ( FeedRewardIn, FeedRewardOut, PangleCallbackOut, + RewardNoShowIn, + RewardNoShowOut, TestGrantIn, TestGrantOut, WatchReportIn, @@ -242,10 +244,12 @@ def ecpm_report(payload: EcpmReportIn, user: CurrentUser, db: DbSession) -> Ecpm ad_type=payload.ad_type, ecpm_raw=payload.ecpm, ad_session_id=payload.ad_session_id, adn=payload.adn, slot_id=payload.slot_id, + app_env=payload.app_env, our_code_id=payload.our_code_id, ) logger.info( - "ad ecpm report user_id=%d type=%s session=%s ecpm=%s adn=%s slot=%s", + "ad ecpm report user_id=%d type=%s session=%s ecpm=%s adn=%s slot=%s app=%s code=%s", user.id, payload.ad_type, payload.ad_session_id, payload.ecpm, payload.adn, payload.slot_id, + payload.app_env, payload.our_code_id, ) return EcpmReportOut(ok=True) @@ -359,6 +363,9 @@ def feed_reward(payload: FeedRewardIn, user: CurrentUser, db: DbSession) -> Feed ad_session_id=payload.ad_session_id, adn=payload.adn, slot_id=payload.slot_id, + app_env=payload.app_env, + our_code_id=payload.our_code_id, + aborted=payload.aborted, ) logger.info( "feed ad reward user_id=%d event=%s status=%s units=%d coin=%d", @@ -371,3 +378,30 @@ def feed_reward(payload: FeedRewardIn, user: CurrentUser, db: DbSession) -> Feed unit_count=rec.unit_count, daily_limit=rewards.get_ad_daily_limit(db), ) + + +@router.post( + "/reward-noshow", + response_model=RewardNoShowOut, + summary="激励视频提前关闭/未发奖留痕", + dependencies=[Depends(rate_limit(120, 60, "ad-reward-noshow"))], +) +def reward_noshow(payload: RewardNoShowIn, user: CurrentUser, db: DbSession) -> RewardNoShowOut: + """激励视频展示了但用户提前关/跳过、未触发 S2S 发奖时,客户端 best-effort 上报一条留痕, + 让广告收益报表能呈现「有展示、没发金币」的原因。不发金币;同一 session 已发奖则跳过。 + """ + rec = crud_ad.record_reward_noshow( + db, + user.id, + ad_session_id=payload.ad_session_id, + ecpm=payload.ecpm, + adn=payload.adn, + slot_id=payload.slot_id, + app_env=payload.app_env, + our_code_id=payload.our_code_id, + ) + logger.info( + "ad reward noshow user_id=%d session=%s watched=%ds -> status=%s", + user.id, payload.ad_session_id, payload.watched_seconds, rec.status, + ) + return RewardNoShowOut(ok=True, status=rec.status) diff --git a/app/api/v1/feedback.py b/app/api/v1/feedback.py index c085780..e59fb54 100644 --- a/app/api/v1/feedback.py +++ b/app/api/v1/feedback.py @@ -21,7 +21,7 @@ logger = logging.getLogger("shagua.feedback") router = APIRouter(prefix="/api/v1/feedback", tags=["feedback"]) _MAX_IMAGES = 6 -_CONTENT_MAX = 2000 +_CONTENT_MAX = 200 _CONTACT_MAX = 128 diff --git a/app/models/ad_ecpm.py b/app/models/ad_ecpm.py index 0f46b51..c1aa029 100644 --- a/app/models/ad_ecpm.py +++ b/app/models/ad_ecpm.py @@ -35,6 +35,11 @@ class AdEcpmRecord(Base): adn: Mapped[str | None] = mapped_column(String(32), nullable=True) # 实际展示用的代码位(底层 mediation rit,非客户端配置位) slot_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + # 我们的穿山甲应用环境:prod(傻瓜比价正式应用) / test(测试应用)。客户端按 AdConfig.useProductionApp 上报。 + # 与底层 adn 不同:这是「我们用的是哪个 App」,adn 是「聚合后实际填充的子渠道」。旧数据为 NULL。 + app_env: Mapped[str | None] = mapped_column(String(16), nullable=True) + # 我们在穿山甲后台配置的代码位 ID(AdConfig.feedCodeId/rewardCodeId 返回的 104xxx,**非** slot_id 的底层 rit)。旧数据为 NULL。 + our_code_id: Mapped[str | None] = mapped_column(String(64), nullable=True) # 客户端上报的 eCPM 原始字符串(单位:分/千次展示,SDK getEcpm 原值,原样存) ecpm_raw: Mapped[str] = mapped_column(String(32), nullable=False) # 北京时间日期串 'YYYY-MM-DD',按它等值做"按天聚合"(不在 SQL 里做跨时区 date 比较) diff --git a/app/models/ad_feed_reward.py b/app/models/ad_feed_reward.py index 99f261e..fb1ff82 100644 --- a/app/models/ad_feed_reward.py +++ b/app/models/ad_feed_reward.py @@ -30,6 +30,9 @@ class AdFeedRewardRecord(Base): ecpm_raw: Mapped[str] = mapped_column(String(32), nullable=False) adn: Mapped[str | None] = mapped_column(String(32), nullable=True) slot_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + # 来源(广告收益报表用):我们的应用环境 prod/test + 我们配置的代码位 104xxx,由客户端 feed-reward 上报带上。旧数据为 NULL。 + app_env: Mapped[str | None] = mapped_column(String(16), nullable=True) + our_code_id: Mapped[str | None] = mapped_column(String(64), nullable=True) coin: Mapped[int] = mapped_column(Integer, nullable=False, default=0) status: Mapped[str] = mapped_column(String(16), nullable=False, default="granted") diff --git a/app/models/ad_reward.py b/app/models/ad_reward.py index c9e8e9f..d45be76 100644 --- a/app/models/ad_reward.py +++ b/app/models/ad_reward.py @@ -33,6 +33,10 @@ class AdRewardRecord(Base): ad_session_id: Mapped[str | None] = mapped_column(String(64), index=True, nullable=True) # 本次发奖采用的 eCPM 原始值(回调自带或按 ad_session_id 匹配的客户端上报) ecpm_raw: Mapped[str | None] = mapped_column(String(32), nullable=True) + # 来源(广告收益报表用):我们的应用环境 prod/test + 我们配置的代码位 104xxx。 + # S2S 回调本身不带这俩,发奖时按 ad_session_id 匹配 ad_ecpm_record 回填(查不到为 NULL)。 + app_env: Mapped[str | None] = mapped_column(String(16), nullable=True) + our_code_id: Mapped[str | None] = mapped_column(String(64), nullable=True) # 北京时间日期串 'YYYY-MM-DD',按它等值统计当日发奖次数 reward_date: Mapped[str] = mapped_column(String(10), index=True, nullable=False) # 穿山甲上报的奖励名(参考,不作发奖依据) diff --git a/app/repositories/ad_ecpm.py b/app/repositories/ad_ecpm.py index 9841ae9..adfc7fd 100644 --- a/app/repositories/ad_ecpm.py +++ b/app/repositories/ad_ecpm.py @@ -23,8 +23,14 @@ def create_ecpm_record( ad_session_id: str | None = None, adn: str | None = None, slot_id: str | None = None, + app_env: str | None = None, + our_code_id: str | None = None, ) -> AdEcpmRecord: - """落一条 eCPM 上报记录。report_date 用北京时间当天,供按天聚合。""" + """落一条 eCPM 上报记录。report_date 用北京时间当天,供按天聚合。 + + app_env(prod/test)与 our_code_id(我们后台配置的 104xxx 代码位)供广告收益报表按 + 应用/代码位聚合;与 adn(实际填充子渠道)/slot_id(底层 rit)是两组不同口径。 + """ if ad_session_id: existing = find_by_session(db, user_id=user_id, ad_session_id=ad_session_id) if existing is not None: @@ -35,6 +41,8 @@ def create_ecpm_record( ad_session_id=ad_session_id, adn=adn, slot_id=slot_id, + app_env=app_env, + our_code_id=our_code_id, ecpm_raw=ecpm_raw, report_date=cn_today().isoformat(), ) diff --git a/app/repositories/ad_feed_reward.py b/app/repositories/ad_feed_reward.py index c2e367c..64fe277 100644 --- a/app/repositories/ad_feed_reward.py +++ b/app/repositories/ad_feed_reward.py @@ -69,12 +69,19 @@ def grant_feed_reward( ad_session_id: str | None = None, adn: str | None = None, slot_id: str | None = None, + app_env: str | None = None, + our_code_id: str | None = None, + aborted: bool = False, ) -> AdFeedRewardRecord: - """完成一条信息流广告后结算奖励。client_event_id 幂等,同号重试不重复发。 + """比价/领券一整场信息流(轮播多条)结束后结算奖励。client_event_id 幂等,同号重试不重复发。 - 一期 eCPM/时长均由客户端上报,故服务端两道硬闸防刷:时长钳到 FEED_MAX_DURATION_SECONDS - 限单事件份数,eCPM 在 rewards.calculate_ad_reward_coin 内钳到 AD_ECPM_MAX_FEN 限单份金额; - 叠加每日 get_ad_daily_limit 条数上限,把单用户日产出锁进有限区间。 + 发奖规则:**比价全程不关广告才发**,金额按整场**总观看时长**折份(每 10 秒 1 份)。 + - aborted=True(用户中途 ✕ 关闭):整场不发,记 status='closed_early' 留痕(原因可查)。 + - 总时长不足 10 秒(unit_count==0):记 status='too_short' 不发。 + - 命中当日条数上限:记 status='capped' 不发。 + duration_seconds 是整场累计秒数。一期 eCPM/时长均由客户端上报,故服务端两道硬闸防刷:时长钳到 + FEED_MAX_DURATION_SECONDS 限单场份数,eCPM 在 rewards.calculate_ad_reward_coin 内钳到 + AD_ECPM_MAX_FEN 限单份金额;叠加每日 get_ad_daily_limit 条数上限,把单用户日产出锁进有限区间。 """ existing = _find_by_event(db, client_event_id) if existing is not None: @@ -85,6 +92,25 @@ def grant_feed_reward( safe_duration = max(0, min(duration_seconds, FEED_MAX_DURATION_SECONDS)) unit_count = safe_duration // FEED_REWARD_UNIT_SECONDS + # 用户中途关闭广告:整场不发(全程不关才发),留一条 closed_early 记录原因。优先级最高。 + if aborted: + rec = AdFeedRewardRecord( + client_event_id=client_event_id, + user_id=user_id, + reward_date=today, + duration_seconds=safe_duration, + unit_count=unit_count, + ad_session_id=ad_session_id, + ecpm_raw=ecpm, + adn=adn, + slot_id=slot_id, + app_env=app_env, + our_code_id=our_code_id, + coin=0, + status="closed_early", + ) + return _commit_record(db, rec, client_event_id) + if _granted_today(db, user_id, today) >= rewards.get_ad_daily_limit(db): rec = AdFeedRewardRecord( client_event_id=client_event_id, @@ -96,11 +122,32 @@ def grant_feed_reward( ecpm_raw=ecpm, adn=adn, slot_id=slot_id, + app_env=app_env, + our_code_id=our_code_id, coin=0, status="capped", ) return _commit_record(db, rec, client_event_id) + # 整场总时长不足 10 秒,凑不满一份 → 不发,记 too_short 留痕。 + if unit_count == 0: + rec = AdFeedRewardRecord( + client_event_id=client_event_id, + user_id=user_id, + reward_date=today, + duration_seconds=safe_duration, + unit_count=0, + ad_session_id=ad_session_id, + ecpm_raw=ecpm, + adn=adn, + slot_id=slot_id, + app_env=app_env, + our_code_id=our_code_id, + coin=0, + status="too_short", + ) + return _commit_record(db, rec, client_event_id) + coin = _unit_reward_total(db, user_id, ecpm, unit_count) if coin > 0: crud_wallet.grant_coins( @@ -118,6 +165,8 @@ def grant_feed_reward( ecpm_raw=ecpm, adn=adn, slot_id=slot_id, + app_env=app_env, + our_code_id=our_code_id, coin=coin, status="granted", ) diff --git a/app/repositories/ad_reward.py b/app/repositories/ad_reward.py index 7c433d5..f9fb16f 100644 --- a/app/repositories/ad_reward.py +++ b/app/repositories/ad_reward.py @@ -90,6 +90,16 @@ def grant_ad_reward( today = cn_today().isoformat() + # 按 ad_session_id 匹配客户端 eCPM 上报:既用于缺 eCPM 时回退取值,也把「来源」 + # (我们的应用 app_env + 我们配置的代码位 our_code_id)回填到发奖记录,供广告收益报表聚合。 + # S2S 回调本身不带这俩;查不到(未上报 eCPM)则留空。 + ecpm_rec = ( + crud_ecpm.find_by_session(db, user_id=user_id, ad_session_id=ad_session_id) + if ad_session_id else None + ) + src_app_env = ecpm_rec.app_env if ecpm_rec is not None else None + src_code_id = ecpm_rec.our_code_id if ecpm_rec is not None else None + # #3 每日上限:当前产品只保留发奖次数上限(默认 500 次)。旧的观看时长闸保留字段, # 但 DAILY_AD_WATCH_SECONDS_LIMIT=0 时视为停用,不能命中 capped。 over_time = ( @@ -102,19 +112,18 @@ def grant_ad_reward( trans_id=trans_id, user_id=user_id, coin=0, status="capped", reward_date=today, reward_name=reward_name, raw=raw, reward_scene=reward_scene, ad_session_id=ad_session_id, ecpm_raw=ecpm, + app_env=src_app_env, our_code_id=src_code_id, ) return _commit_record(db, rec, trans_id) - ecpm_raw = ecpm - if not ecpm_raw and ad_session_id: - ecpm_rec = crud_ecpm.find_by_session(db, user_id=user_id, ad_session_id=ad_session_id) - ecpm_raw = ecpm_rec.ecpm_raw if ecpm_rec is not None else None + ecpm_raw = ecpm or (ecpm_rec.ecpm_raw if ecpm_rec is not None else None) if not ecpm_raw: rec = AdRewardRecord( trans_id=trans_id, user_id=user_id, coin=0, status="ecpm_missing", reward_date=today, reward_name=reward_name, raw=raw, reward_scene=reward_scene, ad_session_id=ad_session_id, ecpm_raw=None, + app_env=src_app_env, our_code_id=src_code_id, ) return _commit_record(db, rec, trans_id) @@ -131,6 +140,55 @@ def grant_ad_reward( trans_id=trans_id, user_id=user_id, coin=coin, status="granted", reward_date=today, reward_name=reward_name, raw=raw, reward_scene=reward_scene, ad_session_id=ad_session_id, ecpm_raw=ecpm_raw, + app_env=src_app_env, our_code_id=src_code_id, + ) + return _commit_record(db, rec, trans_id) + + +def record_reward_noshow( + db: Session, + user_id: int, + *, + ad_session_id: str, + ecpm: str | None = None, + adn: str | None = None, + slot_id: str | None = None, + app_env: str | None = None, + our_code_id: str | None = None, + reward_scene: str = "reward_video", +) -> AdRewardRecord: + """客户端上报「激励视频展示了但用户提前关/跳过、未触发发奖」,落一条 coin=0 status='closed_early' + 记录,供广告收益报表把「不发金币的原因」也呈现出来(只留痕,不发币)。 + + 幂等键 trans_id = 'noreward:{ad_session_id}'(每次展示唯一)。若同一 ad_session_id 已有 granted + 记录(S2S 已发奖,正常路径),说明用户其实看完了 → 跳过不写、原样返回那条,避免与正常发奖重复。 + app_env/our_code_id 由客户端直接带上(它本就持有);查不到 user 抛 UnknownUserError。 + """ + if db.get(User, user_id) is None: + raise UnknownUserError + # 同一次展示已正常发奖 → 不再记 closed_early(防与 S2S granted 重复) + granted = db.execute( + select(AdRewardRecord) + .where( + AdRewardRecord.user_id == user_id, + AdRewardRecord.ad_session_id == ad_session_id, + AdRewardRecord.status == "granted", + ) + .limit(1) + ).scalar_one_or_none() + if granted is not None: + return granted + + trans_id = f"noreward:{ad_session_id}" + existing = _find_by_trans(db, trans_id) + if existing is not None: + return existing + + rec = AdRewardRecord( + trans_id=trans_id, user_id=user_id, coin=0, status="closed_early", + reward_date=cn_today().isoformat(), reward_name=None, raw=None, + reward_scene=reward_scene, ad_session_id=ad_session_id, ecpm_raw=ecpm, + app_env=app_env, our_code_id=our_code_id, ) return _commit_record(db, rec, trans_id) diff --git a/app/schemas/ad.py b/app/schemas/ad.py index a1cb7e0..ae56aab 100644 --- a/app/schemas/ad.py +++ b/app/schemas/ad.py @@ -57,6 +57,12 @@ class EcpmReportIn(BaseModel): ) adn: str | None = Field(None, description="实际投放 ADN(getSdkName),如 pangle") slot_id: str | None = Field(None, description="实际展示代码位(底层 mediation rit)") + app_env: str | None = Field( + None, max_length=16, description="我们的穿山甲应用环境:prod(傻瓜比价正式) / test(测试应用)" + ) + our_code_id: str | None = Field( + None, max_length=64, description="我们后台配置的代码位 ID(AdConfig 的 104xxx,非底层 rit)" + ) class EcpmReportOut(BaseModel): @@ -115,24 +121,62 @@ class TestGrantOut(BaseModel): class FeedRewardIn(BaseModel): - """信息流广告完成后结算奖励。 + """比价/领券一整场信息流(轮播多条)结束后结算奖励。 - 每展示满 10 秒累计一份奖励,视频完成后一次性入账。client_event_id 用于客户端超时重试幂等。 + 规则:全程不关广告才发,金额按整场**总观看时长**折份(每 10 秒 1 份)。client_event_id 用于 + 客户端超时重试幂等。中途被用户关闭时传 aborted=True,整场不发(只记 closed_early)。 """ client_event_id: str = Field(..., min_length=8, max_length=64, description="客户端生成的幂等事件 id") ad_session_id: str | None = Field( None, min_length=8, max_length=64, description="客户端生成的一次信息流广告会话 id" ) - ecpm: str = Field(..., description="本条信息流广告 eCPM,按分/千次展示处理(SDK getEcpm 原值,非元)") - duration_seconds: int = Field(..., ge=0, description="本条广告实际展示/播放秒数") + ecpm: str = Field(..., description="本场信息流 eCPM(代表值,按分/千次展示处理;SDK getEcpm 原值,非元)") + duration_seconds: int = Field(..., ge=0, description="整场累计观看秒数(轮播各条相加)") adn: str | None = Field(None, description="实际投放 ADN") slot_id: str | None = Field(None, description="实际展示代码位") + app_env: str | None = Field( + None, max_length=16, description="我们的穿山甲应用环境:prod(傻瓜比价正式) / test(测试应用)" + ) + our_code_id: str | None = Field( + None, max_length=64, description="我们后台配置的代码位 ID(AdConfig 的 104xxx,非底层 rit)" + ) + aborted: bool = Field( + False, description="用户中途 ✕ 关闭广告(未走完比价):整场不发,记 closed_early" + ) class FeedRewardOut(BaseModel): granted: bool = Field(..., description="本次是否入账。达上限时 false") - status: str = Field(..., description="granted / capped") + status: str = Field(..., description="granted / capped / too_short / closed_early") coin: int = Field(..., description="本次发放金币") unit_count: int = Field(..., description="按 10 秒折算出的奖励份数") daily_limit: int = Field(..., description="每日信息流展示次数上限") + + +class RewardNoShowIn(BaseModel): + """激励视频展示了但用户提前关闭/跳过、未触发发奖——上报留痕(不发金币)。 + + 供广告收益报表把「有展示、没发金币」的原因也记录下来。user_id 由 JWT 取,不在 body。 + """ + + ad_session_id: str = Field( + ..., min_length=8, max_length=64, description="本次展示会话 id(与 ecpm 上报、S2S extra 一致)" + ) + watched_seconds: int = Field(0, ge=0, description="关闭前已观看秒数(仅留痕参考,不入库)") + ecpm: str | None = Field(None, description="本次展示 eCPM(分/千次,SDK getEcpm 原值);可空") + adn: str | None = Field(None, description="实际投放 ADN") + slot_id: str | None = Field(None, description="实际展示代码位(底层 mediation rit)") + app_env: str | None = Field( + None, max_length=16, description="我们的穿山甲应用环境:prod / test" + ) + our_code_id: str | None = Field( + None, max_length=64, description="我们后台配置的代码位 ID(104xxx)" + ) + + +class RewardNoShowOut(BaseModel): + ok: bool = Field(..., description="是否处理成功(落库或幂等命中)") + status: str = Field( + ..., description="closed_early(已留痕) / granted(该次其实已发奖,跳过未写)" + ) diff --git a/docs/api/README.md b/docs/api/README.md index 79bea57..25e8ec6 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -63,6 +63,7 @@ | 34 | `POST /api/v1/ad/test-grant` | Bearer | [详情](./ad-test-grant.md) | | 35 | `POST /api/v1/ad/ecpm-report` | Bearer | [详情](./ad-ecpm-report.md) | | 35a | `POST /api/v1/ad/feed-reward` | Bearer | [详情](./ad-feed-reward.md) | +| 35b | `POST /api/v1/ad/reward-noshow` | Bearer | [详情](./ad-reward-noshow.md)(激励视频提前关闭/未发奖留痕,只记原因不发币) | | **用户资料**(前缀 `/api/v1/user`) ||| | 35 | `PATCH /api/v1/user/profile` | Bearer | [详情](./user-profile.md) | | 36 | `POST /api/v1/user/avatar` | Bearer | [详情](./user-avatar.md) | @@ -104,6 +105,7 @@ | A26 | `POST /admin/api/marquee-seeds/bulk` | operator | [详情](./admin-marquee-seeds.md) | | A27 | `GET /admin/api/marquee-seeds/preview` | admin | [详情](./admin-marquee-seeds.md) | | A28 | `GET /admin/api/ad-coin-audit` | admin | [详情](./admin-ad-coin-audit.md)(看广告金币公式复算对账,只读) | +| A29 | `GET /admin/api/ad-revenue-report` | admin | [详情](./admin-ad-revenue-report.md)(广告收益报表:按用户/日期/类型/应用/代码位 聚合 条数/收益/金币,只读) | | - | `GET /admin/api/health` | 无 | admin 健康检查(无单独文档) | > ⚠️ 美团三个接口当前**无鉴权**,且 `referral-link` 的 `sid` 允许客户端传值覆盖默认渠道——见各接口"备注"。 diff --git a/docs/api/ad-ecpm-report.md b/docs/api/ad-ecpm-report.md index b29fefb..6359b00 100644 --- a/docs/api/ad-ecpm-report.md +++ b/docs/api/ad-ecpm-report.md @@ -7,11 +7,13 @@ | 字段 | 类型 | 必填 | 说明 | |---|---|---|---| -| `ad_type` | str | 是 | 广告类型:`reward_video`(激励视频) / `draw`(Draw 信息流) 等 | -| `ad_session_id` | str\|null | 否 | 客户端生成的广告会话 ID;需和穿山甲 `extra.ad_session_id` 一致,用于 S2S 缺 eCPM 时匹配 | +| `ad_type` | str | 是 | 广告类型:`reward_video`(激励视频) / `feed`(信息流) / `draw`(历史 Draw 信息流) 等 | +| `ad_session_id` | str\|null | 否 | 客户端生成的广告会话 ID;激励视频与穿山甲 `extra.ad_session_id` 一致,用于 S2S 缺 eCPM 时匹配。**信息流轮播每条展示用各自独立 id**(不复用比价会话 id,否则 `uq_ad_ecpm_record_session` 去重只留一条) | | `ecpm` | str | 是 | 穿山甲 `getShowEcpm().getEcpm()` 原始字符串,单位是**分/千次展示**(非元),后端 ÷100 转元参与金币公式 | -| `adn` | str\|null | 否 | 实际投放 ADN(`getSdkName`),如 `pangle` | +| `adn` | str\|null | 否 | 实际投放 ADN(`getSdkName`),如 `pangle`(聚合后实际填充的子渠道) | | `slot_id` | str\|null | 否 | 实际展示代码位(底层 mediation rit,非客户端配置位) | +| `app_env` | str\|null | 否 | **我们的**穿山甲应用环境:`prod`(傻瓜比价正式) / `test`(测试应用) | +| `our_code_id` | str\|null | 否 | **我们后台配置的**代码位 ID(`AdConfig` 的 104xxx,**非** `slot_id` 的底层 rit) | `user_id` 不在 body 里——由 JWT 取(Bearer),防伪造。 @@ -23,7 +25,7 @@ | `ok` | bool | 落库即 `true` | ## 说明 -客户端在广告**展示后**(`onAdShow` 读 `getShowEcpm()`)调用,把本次展示的 eCPM 落库做**内部收益统计/对账**。 +客户端在广告**展示后**(`onAdShow` 读 `getShowEcpm()`)调用,把本次展示的 eCPM 落库做**内部收益统计/对账**。激励视频每次展示上报一条;**信息流轮播每条展示各上报一条**(每条独立 `ad_session_id`),作为「广告收益报表」展示条数/收益的数据源(见 [admin-ad-revenue-report](./admin-ad-revenue-report.md))。`app_env` + `our_code_id` 供报表按「我们的应用 / 我们配置的代码位」聚合,与 `adn`/`slot_id`(底层填充渠道/rit)是两组不同口径。 - 普通激励视频发奖会先用 S2S 回调自带 `ecpm`;若缺失,再按 `ad_session_id` 读取本接口上报的 eCPM;两边都没有则不发并记录异常。 - **best-effort**:客户端 fire-and-forget,但普通激励视频若 S2S 缺 eCPM,这条上报会成为发奖依据。 diff --git a/docs/api/ad-feed-reward.md b/docs/api/ad-feed-reward.md index 0510a0f..23516a7 100644 --- a/docs/api/ad-feed-reward.md +++ b/docs/api/ad-feed-reward.md @@ -1,6 +1,6 @@ # POST /api/v1/ad/feed-reward — 信息流广告完成后结算金币 -点位 2:比价等待 / 领券信息流广告。每展示满 10 秒累计一份奖励,视频完成后一次性入账。 +点位 2:比价等待 / 领券信息流广告(轮播多条)。**整场比价全程不关广告才发**,金额按整场**总观看时长**折份(每 10 秒 1 份),结束时一次性入账。用户中途 ✕ 关闭则整场不发。 ## 鉴权 @@ -12,24 +12,27 @@ |---|---|---:|---| | `client_event_id` | string | 是 | 客户端生成的幂等事件 id,8-64 字符 | | `ad_session_id` | string\|null | 否 | 客户端生成的一次信息流广告会话 id,用于对账/排查 | -| `ecpm` | string | 是 | 本条信息流广告 eCPM(穿山甲 getEcpm 原值),按“分/千次展示”处理(非元) | -| `duration_seconds` | int | 是 | 实际展示/播放秒数 | -| `adn` | string\|null | 否 | 实际投放 ADN | -| `slot_id` | string\|null | 否 | 实际展示代码位 | +| `ecpm` | string | 是 | 本场信息流 eCPM 代表值(穿山甲 getEcpm 原值),按“分/千次展示”处理(非元) | +| `duration_seconds` | int | 是 | **整场累计观看秒数**(轮播各条相加) | +| `adn` | string\|null | 否 | 实际投放 ADN(聚合后实际填充的子渠道) | +| `slot_id` | string\|null | 否 | 实际展示代码位(底层 mediation rit) | +| `app_env` | string\|null | 否 | **我们的**应用环境:`prod`(傻瓜比价正式) / `test`(测试应用) | +| `our_code_id` | string\|null | 否 | **我们后台配置的**代码位 ID(104xxx,非底层 rit);供广告收益报表按代码位聚合金币 | +| `aborted` | bool | 否 | 用户中途 ✕ 关闭广告(未走完比价):整场不发,仅记 `closed_early`。默认 `false` | ## 响应 | 字段 | 类型 | 说明 | |---|---|---| -| `granted` | bool | 本次是否入账;达上限时为 `false` | -| `status` | string | `granted` / `capped` | -| `coin` | int | 本次发放金币 | +| `granted` | bool | 本次是否入账;未发(任一非 granted 状态)时为 `false` | +| `status` | string | `granted`(已发) / `capped`(当日次数超限) / `too_short`(整场总时长<10s 凑不满一份) / `closed_early`(用户中途关闭) | +| `coin` | int | 本次发放金币;非 granted 为 0 | | `unit_count` | int | 按 10 秒折算出的奖励份数 | | `daily_limit` | int | 每日信息流展示次数上限,默认 500 | ## 计算口径 -- 奖励份数:`duration_seconds // 10`。 +- 奖励份数:`整场总时长 // 10`。 - 单份奖励:`eCPM / 1000 × 因子1(eCPM 档) × 因子2(当天累计份序号) × 10000`,四舍五入为整数金币。 - eCPM 档:`0-100=0.1`,`101-200=0.3`,`201-400=0.4`,`>400=0.6`。 - LT 档:第 1 份 `2.0`,第 2 份 `1.5`,第 3 份 `1.3`,第 4-10 份 `1.1`,第 11 份及以后 `1.0`。 diff --git a/docs/api/ad-reward-noshow.md b/docs/api/ad-reward-noshow.md new file mode 100644 index 0000000..a36b69b --- /dev/null +++ b/docs/api/ad-reward-noshow.md @@ -0,0 +1,41 @@ +# POST /api/v1/ad/reward-noshow — 激励视频提前关闭/未发奖留痕 + +激励视频**展示了但用户提前关闭/跳过、未触发 S2S 发奖**时,客户端 best-effort 上报一条留痕记录, +让运营后台「广告数据」能呈现「有展示、没发金币」的原因。**不发金币**。 + +## 鉴权 + +需要 Bearer token。`user_id` 由 JWT 取,不在 body。 + +## 请求体 + +| 字段 | 类型 | 必填 | 说明 | +|---|---|---:|---| +| `ad_session_id` | string | 是 | 本次展示会话 id(与 eCPM 上报、S2S `extra.ad_session_id` 一致),8-64 字符 | +| `watched_seconds` | int | 否 | 关闭前已观看秒数(仅留痕参考,**不入库**)。默认 0 | +| `ecpm` | string\|null | 否 | 本次展示 eCPM(穿山甲 getEcpm 原值,分/千次展示) | +| `adn` | string\|null | 否 | 实际投放 ADN(聚合后实际填充子渠道) | +| `slot_id` | string\|null | 否 | 实际展示代码位(底层 mediation rit) | +| `app_env` | string\|null | 否 | **我们的**应用环境:`prod`(傻瓜比价正式) / `test`(测试应用) | +| `our_code_id` | string\|null | 否 | **我们后台配置的**代码位 ID(104xxx,非底层 rit) | + +## 响应 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `ok` | bool | 是否处理成功(落库或幂等命中) | +| `status` | string | `closed_early`(已留痕) / `granted`(该次其实已发奖,跳过未写) | + +## 说明 + +- 在 `onAdClose` 时若本次**未触发发奖**(无 `onRewardVerify`/`onRewardArrived`)调用。 +- **幂等**:写入 `ad_reward_record`,`trans_id = noreward:{ad_session_id}`(每次展示唯一),重试不重复。 +- **防与正常发奖重复**:若同一 `ad_session_id` 已有 `status='granted'` 记录(S2S 已发,说明用户其实看完了), + 则跳过不写,返回 `status='granted'`。 +- best-effort:客户端 fire-and-forget,失败只 log,不影响主流程。 +- 信息流的「用户中途关闭」走 [ad-feed-reward](./ad-feed-reward.md) 的 `aborted=true`(记 `closed_early`),不走本接口。 + +## 数据写入 + +- `ad_reward_record` 新增一行(`coin=0`、`status='closed_early'`、`reward_scene='reward_video'`)。 +- **不**写 `coin_account` / `coin_transaction`(不发币)。 diff --git a/docs/api/admin-ad-revenue-report.md b/docs/api/admin-ad-revenue-report.md new file mode 100644 index 0000000..bccfd8f --- /dev/null +++ b/docs/api/admin-ad-revenue-report.md @@ -0,0 +1,108 @@ +# Admin 广告收益报表 + +> 所属:Admin 组(前缀 `/admin/api/ad-revenue-report`) | 鉴权:Admin Bearer(任意已登录 admin,只读) | [← 返回 API 索引](./README.md) + +按 **用户 × 日期 × 广告类型 × 我们的应用 × 我们的代码位** 聚合,回答「每个用户某天、每类广告(激励视频 / 信息流 / 历史 Draw)分别**看了多少条**、**收益多少**、按现算法**发了多少金币**、广告来自**哪个应用的哪个代码位**」。**纯只读**,不发币、不改数据,也**不改发奖逻辑**。 + +相关表:[ad_ecpm_record](../database/ad_ecpm_record.md)、[ad_reward_record](../database/ad_reward_record.md)、[ad_feed_reward_record](../database/ad_feed_reward_record.md)。 + +## 数据来源(三流合并,聚合键 = user × ad_type × app_env × our_code_id) + +| 指标 | 来源表 | 口径 | +|---|---|---| +| 展示条数 `impressions` | `ad_ecpm_record` | 每行 = 客户端一次广告展示。激励视频每次展示上报一条;**信息流轮播每条展示各上报一条**(每条独立 `ad_session_id`) | +| 收益 `revenue_yuan` | `ad_ecpm_record` | `Σ(eCPM元 ÷ 1000)`,即每条展示预估收益累加(eCPM 原值是分,÷100 转元;÷1000 是每千次→单次)。**预估口径,非结算;测试应用多为 0** | +| 应发/实发金币 `expected_coin`/`actual_coin` | `ad_reward_record`(reward_video)+ `ad_feed_reward_record`(feed) | **复用金币审计逐条复算**(`ad_audit.audit_rows`,与正式发奖同一公式口径,不另写公式),按同维度求和;`matched = 应发==实发`。**只读复算,不改发奖** | +| 来源应用/代码位 `app_env`/`our_code_id` | 上述各表回填 | `prod`(傻瓜比价)/`test`(测试);代码位是**我们后台配的 104xxx**,非底层 rit | +| 底层渠道 `adns` | `ad_ecpm_record` | 实际填充的 ADN 子渠道集合(pangle/gdt/...),附加参考 | + +展示与金币来自不同表,做**并集**:有展示无金币(用户中途关、未达发奖)、有金币无展示(未上报 eCPM)各自成行。 + +## GET /admin/api/ad-revenue-report — 聚合报表 + +- 入参(均 query,可选): + | 参数 | 类型 | 默认 | 说明 | + |---|---|---|---| + | `date_from` | string | 今天 | 起始日 北京时间 `YYYY-MM-DD` | + | `date_to` | string | =`date_from` | 结束日 北京时间 `YYYY-MM-DD`,**闭区间**;单日时与 `date_from` 相同 | + | `user_id` | int | 全部 | 只看某用户;不传=所有用户 | + | `ad_type` | string | 全部 | `reward_video` / `feed` / `draw`;不传=全部类型 | + | `granularity` | string | `day` | `day`=按天 / `hour`=按小时(聚合键再加北京时间小时 0–23);**区间>1 天建议用 day** | + | `limit` | int(1~1000) | 500 | **展示**明细组数(截断;`total`/`total_*`/`daily` 按全量统计不受影响) | + + 约束:`date_to` 不早于 `date_from`、区间最长 **92 天**、日期须 `YYYY-MM-DD`,否则 `422`。 + +- 出参 `200`:`AdRevenueReportOut` + | 字段 | 类型 | 说明 | + |---|---|---| + | `date_from` / `date_to` | string | 报表起止日期(闭区间) | + | `daily` | `AdRevenueDaily[]` | 按日期汇总序列(全量,供按天趋势图;不受 `limit` 影响) | + | `total` | int | 聚合组**总数**(全量,不受 `limit` 影响) | + | `truncated` | bool | 明细是否被 `limit` 截断 | + | `total_impressions` | int | 全量展示条数合计 | + | `total_revenue_yuan` | float | 全量收益合计(元) | + | `total_expected_coin` | int | 全量应发金币合计 | + | `total_actual_coin` | int | 全量实发金币合计 | + | `mismatch_count` | int | 应发≠实发的组数(=0 说明全部按公式发放) | + | `items` | `AdRevenueRow[]` | 聚合明细(按 日期→用户→类型→代码位 排序) | + +### AdRevenueDaily(`daily[]` — 按天趋势) +| 字段 | 类型 | 说明 | +|---|---|---| +| `date` | string | 北京时间 `YYYY-MM-DD` | +| `impressions` | int | 当天展示条数合计 | +| `revenue_yuan` | float | 当天预估收益合计(元) | +| `expected_coin` | int | 当天应发金币合计 | +| `actual_coin` | int | 当天实发金币合计 | + +### AdRevenueRow(`items[]`) +| 字段 | 类型 | 说明 | +|---|---|---| +| `report_date` | string | 该组所属日期 北京时间 `YYYY-MM-DD` | +| `user_id` | int | | +| `ad_type` | string | `reward_video` / `feed` / `draw` | +| `app_env` | string \| null | 我们的应用:`prod`(傻瓜比价)/`test`(测试);旧数据为空 | +| `our_code_id` | string \| null | 我们配置的代码位 104xxx;旧数据为空 | +| `hour` | int \| null | 北京时间小时 0–23(`granularity=hour` 时有值;按天为 null) | +| `impressions` | int | 展示条数 | +| `revenue_yuan` | float | 收益(元),预估口径 | +| `expected_coin` | int | 应发金币(公式复算,与金币审计同源) | +| `actual_coin` | int | 实发金币(实际入账) | +| `matched` | bool | 该组应发==实发(组内任一条不符则 false) | +| `adns` | string[] | 底层填充 ADN 子渠道集合 | +| `impression_records` | `AdRevenueImpression[]` | 该组**逐条展示明细**(前端展开下钻);只要有展示就非空 | +| `records` | `AdRevenueRecord[]` | 该组**逐条发奖复算明细**(前端展开下钻);纯展示无发奖的组为空 | + +### AdRevenueImpression(`items[].impression_records[]` — 展开「展示明细」) +| 字段 | 类型 | 说明 | +|---|---|---| +| `id` | int | ad_ecpm_record 主键 | +| `created_at` | datetime | | +| `ecpm` | string | 本次展示 eCPM 原始值(分/千次展示) | +| `revenue_yuan` | float | 本次展示预估收益(元)= eCPM元 ÷ 1000 | +| `adn` | string \| null | 实际填充 ADN 子渠道 | +| `slot_id` | string \| null | 底层 mediation rit(非我们配置的广告位 ID) | + +### AdRevenueRecord(`items[].records[]` — 展开「发奖明细」) +还原金币审计的逐条列,与发奖同一复算口径。 +| 字段 | 类型 | 说明 | +|---|---|---| +| `record_id` | int | 发奖记录主键 | +| `created_at` | datetime | | +| `status` | string | `granted` / `capped` / `ecpm_missing` | +| `ecpm` | string \| null | 本次采用的 eCPM 原始值 | +| `ecpm_factor` | float \| null | 因子1(eCPM 档);非 granted 为空 | +| `units` | int | 折算份数:激励视频恒 1;信息流 = 满 10 秒份数 | +| `lt_index_start` / `lt_index_end` | int \| null | 占用「账号累计第几份」的起止 | +| `lt_factor_start` / `lt_factor_end` | float \| null | 因子2(LT)起止值 | +| `expected_coin` | int | 应发金币 | +| `actual_coin` | int | 实发金币 | +| `matched` | bool | 该条复算与实发是否一致 | + +## 说明与局限 + +- **展示 vs 发奖分离**:信息流轮播一会话可展示多条(都计入 `impressions`),但发奖仍按现规则(一会话发一次),`coin` 不因展示条数变化——这是有意设计(用户中途关只记展示不发奖)。 +- **历史 Draw 不可拆**:迁移(Draw→普通信息流)前,Draw 发奖混在 `ad_feed_reward_record` 且无类型标记,金币侧统一记 `feed`;迁移后 Draw 不再产生新数据。展示侧 `ad_type` 由客户端上报区分,故 `draw` 桶基本为空。 +- **来源字段从上线起齐全**:`app_env`/`our_code_id` 是本期新增列,历史记录为 NULL(报表来源列留空)。 +- **收益是预估**:基于客户端上报的 eCPM,非穿山甲后台结算值;以后台报表为结算权威。 +- **对账聚合级 + 逐条下钻**:行级 `matched` 给出该组(用户×类型×应用×代码位)应发是否==实发;**展开 `records` 即可看该组逐条明细**(eCPM/因子1/份数/LT/因子2/应发/实发/一致)定位到具体记录。独立逐条审计接口 [admin-ad-coin-audit](./admin-ad-coin-audit.md) 仍保留(同一复算口径,可全局按场景/只看不符筛选)。 diff --git a/docs/api/feedback.md b/docs/api/feedback.md index 259df11..e59e157 100644 --- a/docs/api/feedback.md +++ b/docs/api/feedback.md @@ -6,7 +6,7 @@ **multipart/form-data**: | 字段 | 类型 | 必填 | 说明 | |---|---|---|---| -| `content` | string | ✓ | 反馈正文,**1-2000 字**(strip 后) | +| `content` | string | ✓ | 反馈正文,**1-200 字**(strip 后) | | `contact` | string | ✗ | 联系方式(微信/QQ/手机号),**≤128 字**。原型改版后客户端已不再采集、不传该字段(后端默认空串);保留字段兼容旧端 | | `images` | file[] | ✗ | 截图,**最多 6 张**,每张走头像同款校验(JPEG/PNG/WebP,≤ 5 MB,魔数嗅探) | @@ -29,7 +29,7 @@ > 不返回上传的 image URL——这是给运营后台看的,客户端通常不需要。 ## 错误码 -- `400` 内容为空 / 内容超 2000 字 / 联系方式超 128 字 / 图片超 6 张 / 单图非法(空/过大/格式不对) +- `400` 内容为空 / 内容超 200 字 / 联系方式超 128 字 / 图片超 6 张 / 单图非法(空/过大/格式不对) - `401` 未带 token / token 无效或过期 / 用户被禁用 - `422` 缺 `content` 字段 diff --git a/docs/database/ad_ecpm_record.md b/docs/database/ad_ecpm_record.md index 127636f..61def90 100644 --- a/docs/database/ad_ecpm_record.md +++ b/docs/database/ad_ecpm_record.md @@ -7,17 +7,19 @@ ## 用在哪 / 增删改查 - **C(插入)**:`POST /ad/ecpm-report`(`create_ecpm_record`)。每次广告展示上报一条;best-effort,丢一两条不影响业务。鉴权接口已确保 user 存在。 - **U / D**:无。 -- **R**:内部收益统计/对账(按 `(user_id, report_date)` 聚合);`count_today` 排查辅助。当前无面向 C 端用户的读接口。 +- **R**:内部收益统计/对账(按 `(user_id, report_date)` 聚合);`count_today` 排查辅助;广告收益报表 [admin-ad-revenue-report](../api/admin-ad-revenue-report.md) 的展示条数/收益数据源(按 `app_env`/`our_code_id` 聚合)。当前无面向 C 端用户的读接口。 ## 字段 | 列 | 类型 | 约束 / 默认 | 说明(取值 / join) | |---|---|---|---| | `id` | Integer | PK, autoincrement | | | `user_id` | Integer | FK→user.id, index, NOT NULL | 归属用户 | -| `ad_type` | String(32) | NOT NULL | 广告类型,取值如 `reward_video`(激励视频)/ `draw`(Draw 信息流);各类型各自上报,不强行统一代码位 | -| `ad_session_id` | String(64) | UNIQUE, index, nullable | 客户端广告会话 ID;用于普通激励视频在 S2S 缺 `ecpm` 时匹配发奖 | -| `adn` | String(32) | nullable | 实际投放 ADN(`getShowEcpm().getSdkName()`,如 `pangle`/`gdt`) | +| `ad_type` | String(32) | NOT NULL | 广告类型,取值如 `reward_video`(激励视频)/ `feed`(信息流)/ `draw`(历史 Draw 信息流);各类型各自上报,不强行统一代码位 | +| `ad_session_id` | String(64) | UNIQUE, index, nullable | 客户端广告会话 ID;激励视频用于 S2S 缺 `ecpm` 时匹配发奖。**信息流轮播每条展示用各自独立 id**(不复用比价会话 id,否则 UNIQUE 去重只留一条) | +| `adn` | String(32) | nullable | 实际投放 ADN(`getShowEcpm().getSdkName()`,如 `pangle`/`gdt`;聚合后实际填充的子渠道) | | `slot_id` | String(64) | nullable | 实际展示用代码位(底层 mediation rit,非客户端配置位) | +| `app_env` | String(16) | nullable | **我们的**穿山甲应用环境:`prod`(傻瓜比价正式)/`test`(测试应用);旧数据 NULL。广告收益报表按它聚合「来源应用」 | +| `our_code_id` | String(64) | nullable | **我们后台配置的**代码位 ID(`AdConfig` 的 104xxx,**非** `slot_id` 底层 rit);旧数据 NULL | | `ecpm_raw` | String(32) | NOT NULL | 客户端上报的 eCPM **原始串**(穿山甲 getEcpm 原值,单位**分/千次展示**);后端 ÷100 转元参与金币公式 | | `report_date` | String(10) | index, NOT NULL | 北京时间日期串 `YYYY-MM-DD`,按它做按天聚合 | | `created_at` | DateTime(tz) | server_default now(), index | 时间 | diff --git a/docs/database/ad_feed_reward_record.md b/docs/database/ad_feed_reward_record.md index 5ab9e94..99cccb4 100644 --- a/docs/database/ad_feed_reward_record.md +++ b/docs/database/ad_feed_reward_record.md @@ -11,13 +11,15 @@ | `ad_session_id` | String(64) | index, nullable | 客户端生成的一次信息流广告会话 id | | `user_id` | Integer | FK → `user.id`, index, NOT NULL | 用户 | | `reward_date` | String(10) | index, NOT NULL | 北京时间日期 `YYYY-MM-DD` | -| `duration_seconds` | Integer | NOT NULL | 实际展示/播放秒数 | +| `duration_seconds` | Integer | NOT NULL | 整场比价累计观看秒数(轮播各条相加) | | `unit_count` | Integer | NOT NULL | `duration_seconds // 10` 得到的奖励份数 | | `ecpm_raw` | String(32) | NOT NULL | 客户端上报 eCPM 原始值 | -| `adn` | String(32) | nullable | 实际投放 ADN | -| `slot_id` | String(64) | nullable | 实际展示代码位 | -| `coin` | Integer | NOT NULL | 实发金币,`capped` 时为 0 | -| `status` | String(16) | NOT NULL | `granted` / `capped` | +| `adn` | String(32) | nullable | 实际投放 ADN(聚合后实际填充子渠道) | +| `slot_id` | String(64) | nullable | 实际展示代码位(底层 mediation rit) | +| `app_env` | String(16) | nullable | 来源应用 `prod`(傻瓜比价)/`test`(测试),客户端 feed-reward 上报;旧数据 NULL。广告收益报表金币侧按它聚合 | +| `our_code_id` | String(64) | nullable | 我们配置的代码位 104xxx,客户端上报;旧数据 NULL | +| `coin` | Integer | NOT NULL | 实发金币,非 `granted` 时为 0 | +| `status` | String(16) | NOT NULL | `granted`(已发) / `capped`(当日次数超限) / `too_short`(整场总时长<10s 凑不满一份) / `closed_early`(用户中途 ✕ 关闭,全程未看完) | | `created_at` | DateTime | index, NOT NULL | 创建时间 | ## 约束 diff --git a/docs/database/ad_reward_record.md b/docs/database/ad_reward_record.md index 380967c..07c08be 100644 --- a/docs/database/ad_reward_record.md +++ b/docs/database/ad_reward_record.md @@ -5,7 +5,7 @@ 每条 = 穿山甲一次**服务端激励回调**。`trans_id` 唯一做幂等键(穿山甲会重试,同号只处理一次)。`reward_scene` 区分普通激励视频、签到膨胀等场景;`reward_date`(北京时间日期串)给普通激励视频"每日上限"计数用。 ## 用在哪 / 增删改查 -- **C(插入)**:`POST /ad/pangle-callback`(穿山甲 S2S,经 SHA256 验签;`grant_ad_reward` 或场景业务处理)或 `POST /ad/test-grant`(本地联调)。普通激励视频三道闸:① 验签不过 → API 层 403,不进库;② `trans_id` 已存在 → 原样返回不重复发;③ **当日发奖次数(`DAILY_AD_REWARD_LIMIT`,默认 500)到顶** → 记一行 `status='capped'`、`coin=0`、不发币。否则按 eCPM 公式发币。 +- **C(插入)**:`POST /ad/pangle-callback`(穿山甲 S2S,经 SHA256 验签;`grant_ad_reward` 或场景业务处理)或 `POST /ad/test-grant`(本地联调)。普通激励视频三道闸:① 验签不过 → API 层 403,不进库;② `trans_id` 已存在 → 原样返回不重复发;③ **当日发奖次数(`DAILY_AD_REWARD_LIMIT`,默认 500)到顶** → 记一行 `status='capped'`、`coin=0`、不发币。否则按 eCPM 公式发币。另:`POST /ad/reward-noshow`(`record_reward_noshow`,Bearer)在用户提前关/未发奖时记一行 `status='closed_early'`、`coin=0` 留痕(同 session 已 granted 则跳过)。 - **U / D**:无。 - **R**:`GET /ad/reward-status`(看广告页:今日已发次数/上限、单次金币、本轮已看/冷却结束、今日已看时长/上限);审计/对账整表回溯。 @@ -13,13 +13,15 @@ | 列 | 类型 | 约束 / 默认 | 说明(取值 / join) | |---|---|---|---| | `id` | Integer | PK, autoincrement | | -| `trans_id` | String(64) | UNIQUE, index, NOT NULL | 穿山甲交易号(幂等键)。**被 `coin_transaction.ref_id` 引用**(biz_type=reward_video/signin_boost 等) | +| `trans_id` | String(64) | UNIQUE, index, NOT NULL | 穿山甲交易号(幂等键)。**被 `coin_transaction.ref_id` 引用**(biz_type=reward_video/signin_boost 等)。`closed_early` 留痕记录无 S2S 交易号,用合成键 `noreward:{ad_session_id}` | | `user_id` | Integer | FK→user.id, index, NOT NULL | 归属用户(回调 media_extra 带回;不存在抛 UnknownUserError) | | `reward_scene` | String(32) | NOT NULL, default `reward_video` | 奖励场景:`reward_video` 普通激励视频;`signin_boost` 签到膨胀 | | `ad_session_id` | String(64) | index, nullable | 客户端广告会话 ID,来自 `extra.ad_session_id`;用于匹配 `ad_ecpm_record` | | `ecpm_raw` | String(32) | nullable | 本次发奖采用的 eCPM 原始值;可来自 S2S `ecpm` 或客户端上报 | -| `coin` | Integer | NOT NULL, default 0 | 实发金币;`capped`/`ecpm_missing`/业务不满足时为 0 | -| `status` | String(16) | NOT NULL, default `granted` | 取值:`granted`(已发)/ `capped`(当日次数超限)/ `ecpm_missing`(缺 eCPM)/ `not_signed`/`already_boosted`/`last_day` | +| `app_env` | String(16) | nullable | 来源应用 `prod`(傻瓜比价)/`test`(测试);S2S 不带,发奖时按 `ad_session_id` 匹配 `ad_ecpm_record` 回填,查不到 NULL。广告收益报表金币侧按它聚合 | +| `our_code_id` | String(64) | nullable | 我们配置的代码位 104xxx(同上回填) | +| `coin` | Integer | NOT NULL, default 0 | 实发金币;`capped`/`ecpm_missing`/`closed_early`/业务不满足时为 0 | +| `status` | String(16) | NOT NULL, default `granted` | 取值:`granted`(已发)/ `capped`(当日次数超限)/ `ecpm_missing`(缺 eCPM)/ `closed_early`(展示了但用户提前关/跳过,未发奖,客户端 reward-noshow 留痕)/ `not_signed`/`already_boosted`/`last_day` | | `reward_date` | String(10) | index, NOT NULL | 北京时间日期串 `YYYY-MM-DD`,按它等值统计当日发奖次数 | | `reward_name` | String(64) | nullable | 穿山甲上报奖励名(参考,不作发奖依据) | | `raw` | String(1024) | nullable | 回调原始参数(审计排查) | From f15ca74a220aa78afe5b2e375edd2c1dac02f4e5 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 16 Jun 2026 01:25:57 +0800 Subject: [PATCH 2/7] =?UTF-8?q?feat(store-mapping):=20JD=20=E7=BC=93?= =?UTF-8?q?=E5=AD=98=20deeplink=20=E5=A4=B1=E6=95=88=E6=A0=87=E8=AE=B0(?= =?UTF-8?q?=E5=AF=B9=E6=A0=87=E6=B7=98=E5=AE=9D)=20(#56)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-on: https://gitea.shaguabijia.com/WonderableAI/shaguabijia-app-server/pulls/56 --- .../versions/store_mapping_jd_dl_invalid.py | 32 ++++++++++++++ app/api/internal/store.py | 9 ++-- app/api/v1/coupon.py | 15 ++++++- app/models/store_mapping.py | 2 + app/repositories/coupon_state.py | 28 +++++++++++- app/repositories/store_mapping.py | 20 +++++++++ app/schemas/coupon_state.py | 10 +++++ tests/test_store_mapping_invalidate.py | 44 +++++++++++++++++-- 8 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 alembic/versions/store_mapping_jd_dl_invalid.py diff --git a/alembic/versions/store_mapping_jd_dl_invalid.py b/alembic/versions/store_mapping_jd_dl_invalid.py new file mode 100644 index 0000000..09a1af2 --- /dev/null +++ b/alembic/versions/store_mapping_jd_dl_invalid.py @@ -0,0 +1,32 @@ +"""store_mapping 加京东 deeplink 失效标记列 jd_deeplink_invalid_at + +Revision ID: store_mapping_jd_dl_invalid +Revises: store_mapping_tb_dl_invalid +Create Date: 2026-06-15 00:00:00.000000 + +京东同淘宝: 缓存的店内搜索 deeplink 会失效(打开是"当前门店超出配送范围"页)。pricebot 比价撞到 +失效页时回退正常搜店, 并 server→server 通知把该 storeId 的 deeplink 标记失效。本列记失效时刻 +(NULL=有效); lookup 反查时过滤掉已失效的京东候选, 不再返回坏 deeplink。 +与淘宝 taobao_deeplink_invalid_at 对称; 用时间戳而非布尔: 留痕可审计、可统计失效率, 不销毁原 deeplink。 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'store_mapping_jd_dl_invalid' +down_revision: Union[str, Sequence[str], None] = 'store_mapping_tb_dl_invalid' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('store_mapping', schema=None) as batch_op: + batch_op.add_column(sa.Column('jd_deeplink_invalid_at', sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table('store_mapping', schema=None) as batch_op: + batch_op.drop_column('jd_deeplink_invalid_at') diff --git a/app/api/internal/store.py b/app/api/internal/store.py index b7959e7..104d379 100644 --- a/app/api/internal/store.py +++ b/app/api/internal/store.py @@ -95,11 +95,14 @@ def invalidate_store_mapping( x_internal_secret: Annotated[str | None, Header()] = None, ) -> StoreMappingInvalidateOut: _check_secret(x_internal_secret) - if payload.platform != "taobao": - # 当前只接淘宝; 其它平台先 no-op(affected=0), 不报错 — 向后兼容 pricebot 将来扩展。 + if payload.platform == "taobao": + affected = repo.mark_taobao_deeplink_invalid(db, payload.shop_id) + elif payload.platform == "jd": + affected = repo.mark_jd_deeplink_invalid(db, payload.shop_id) + else: + # 当前只接淘宝/京东; 其它平台先 no-op(affected=0), 不报错 — 向后兼容 pricebot 将来扩展。 logger.info("store_mapping invalidate 跳过: platform=%s 暂不支持", payload.platform) return StoreMappingInvalidateOut(ok=True, affected=0) - affected = repo.mark_taobao_deeplink_invalid(db, payload.shop_id) logger.info( "store_mapping invalidate platform=%s shop_id=%s → 标记失效 %d 行", payload.platform, payload.shop_id, affected, diff --git a/app/api/v1/coupon.py b/app/api/v1/coupon.py index 6e07120..07c19af 100644 --- a/app/api/v1/coupon.py +++ b/app/api/v1/coupon.py @@ -18,7 +18,7 @@ import httpx from fastapi import APIRouter, HTTPException, Request, status from fastapi.concurrency import run_in_threadpool -from app.api.deps import DbSession +from app.api.deps import CurrentUser, DbSession from app.core.config import settings from app.core.pricebot_router import pick_pricebot from app.db.session import SessionLocal @@ -28,6 +28,7 @@ from app.schemas.coupon_state import ( CouponPromptDismissIn, CouponPromptShouldShowOut, CouponPromptShownIn, + CouponStatsOut, ) logger = logging.getLogger("shagua.coupon") @@ -268,3 +269,15 @@ def coupon_completed_today_reset( 与 /prompt/reset 配套:开发设置「重置今日领券弹窗状态」一键把今日状态全清。MVP 不鉴权。""" coupon_repo.reset_today_completion(db, payload.device_id) return {"ok": True} + + +@router.get( + "/stats", + response_model=CouponStatsOut, + summary="累计领券数(「我的」页战绩卡「领取优惠券 X 张」)", +) +def coupon_stats(user: CurrentUser, db: DbSession) -> CouponStatsOut: + """该登录用户累计领到的券数(SUM(claimed_count),口径见 coupon_repo.sum_claimed_count)。 + **鉴权(CurrentUser)**——区别于同文件不鉴权的 /step 透传与 /prompt 频控(那些按 device_id): + 个人战绩按 user_id 聚合,必须有登录态。""" + return CouponStatsOut(coupon_count=coupon_repo.sum_claimed_count(db, user.id)) diff --git a/app/models/store_mapping.py b/app/models/store_mapping.py index 35272d0..78850a2 100644 --- a/app/models/store_mapping.py +++ b/app/models/store_mapping.py @@ -105,6 +105,8 @@ class StoreMapping(Base): jd_share_url: Mapped[str | None] = mapped_column(String(256), nullable=True) # 3.cn 短链 jd_resolved_url: Mapped[str | None] = mapped_column(Text, nullable=True) # 反查出的目标 openapp.jdmobile:// deeplink jd_deeplink: Mapped[str | None] = mapped_column(Text, nullable=True) # 拼好的 pages/search 店内搜索 deeplink + # 京东 deeplink 失效标记(同 taobao_deeplink_invalid_at):撞"当前门店超出配送范围"页时被置,NULL=有效。 + jd_deeplink_invalid_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) # 灵活字段兜底(免得加字段就迁移) attrs: Mapped[dict | None] = mapped_column(_JSON, nullable=True) diff --git a/app/repositories/coupon_state.py b/app/repositories/coupon_state.py index 33f4781..e3e2765 100644 --- a/app/repositories/coupon_state.py +++ b/app/repositories/coupon_state.py @@ -9,7 +9,7 @@ import logging from datetime import date, datetime from zoneinfo import ZoneInfo -from sqlalchemy import delete, select +from sqlalchemy import delete, func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -206,3 +206,29 @@ def record_claims( ) return 0 return written + + +# ===== 累计领券数(「我的」页战绩卡「领取优惠券 X 张」)===== + +def sum_claimed_count(db: Session, user_id: int) -> int: + """该用户累计领到的优惠券张数。口径(2026-06-15 用户定):SUM(claimed_count) —— + 各成功领券记录的 pricebot 展示张数(claimed_count 列,存的是 display_count)之和, + 与领券完成时给用户看的「本次领了 N 张」同源。 + + - 只算 status ∈ {success, already_claimed}:already_claimed=今日已领过,协议里算「已领到」; + failed / skipped 不计。 + - claimed_count 为 0 的保持 0:那是同 count_group 合并去重项(pricebot 只让一条出数),不重复计; + 为 NULL 的兜底成 1(成功领到至少 1 张;实际 pricebot to_dict 恒下发 display_count,NULL 基本不出现)。 + - 维度 user_id:登录态领的券才归入。登录前匿名领的(user_id 为空)不算(产品可接受)。 + """ + total = db.execute( + select( + func.coalesce( + func.sum(func.coalesce(CouponClaimRecord.claimed_count, 1)), 0 + ) + ).where( + CouponClaimRecord.user_id == user_id, + CouponClaimRecord.status.in_(("success", "already_claimed")), + ) + ).scalar_one() + return int(total or 0) diff --git a/app/repositories/store_mapping.py b/app/repositories/store_mapping.py index 358aa7d..ed59cfc 100644 --- a/app/repositories/store_mapping.py +++ b/app/repositories/store_mapping.py @@ -124,6 +124,23 @@ def mark_taobao_deeplink_invalid(db: Session, shop_id: str) -> int: return result.rowcount or 0 +def mark_jd_deeplink_invalid(db: Session, store_id: str) -> int: + """把所有 id_jd=store_id 的行标记京东 deeplink 失效(置 invalid_at=now)。返回本次新标记的行数。 + + 同 mark_taobao_deeplink_invalid:按 storeId 标记**所有**行(撞京东"当前门店超出配送范围"页), + 全标掉才能让后续 lookup 不再返回它。幂等:已标记的行(invalid_at 非 NULL)跳过。""" + result = db.execute( + update(StoreMapping) + .where( + StoreMapping.id_jd == store_id, + StoreMapping.jd_deeplink_invalid_at.is_(None), + ) + .values(jd_deeplink_invalid_at=func.now()) + ) + db.commit() + return result.rowcount or 0 + + # ============================================================ # 缓存查询: 比价前按"源平台店名"反查已沉淀的各目标平台店铺 id, 命中就让 pricebot 直接 # deeplink 跳店内搜索, 省掉"开平台→进店→分享反查"整段。 @@ -181,6 +198,9 @@ def lookup_nearest( if tgt == "taobao": # 失效的淘宝 deeplink(撞过错误页被 invalidate)整条排除 = 当没缓存, pricebot 走正常搜店。 cands = [r for r in cands if r.taobao_deeplink_invalid_at is None] + elif tgt == "jd": + # 同上:失效的京东 deeplink(撞"当前门店超出配送范围"被 invalidate)整条排除。 + cands = [r for r in cands if r.jd_deeplink_invalid_at is None] if not cands: continue best = _pick_best(cands, lat, lng) diff --git a/app/schemas/coupon_state.py b/app/schemas/coupon_state.py index 15e9c16..236d311 100644 --- a/app/schemas/coupon_state.py +++ b/app/schemas/coupon_state.py @@ -39,3 +39,13 @@ class CouponCompletedTodayOut(BaseModel): """这台设备今天是否已跑完整轮领券(到 done 帧)。完成 → 首页「去领取」卡置灰。""" completed: bool + + +class CouponStatsOut(BaseModel): + """「我的」页战绩卡「领取优惠券 X 张」数据源:该登录用户累计领到的券数。 + + 口径(2026-06-15 用户定):SUM(claimed_count) —— 各成功领券记录的 pricebot 展示张数之和, + 与领券完成时给用户看的「本次领了 N 张」同源。详见 repositories.coupon_state.sum_claimed_count。 + """ + + coupon_count: int diff --git a/tests/test_store_mapping_invalidate.py b/tests/test_store_mapping_invalidate.py index e1930f9..2d3188f 100644 --- a/tests/test_store_mapping_invalidate.py +++ b/tests/test_store_mapping_invalidate.py @@ -16,11 +16,13 @@ _SECRET = "test-internal-secret-only-for-pytest" def _mk_row(db, *, trace_id, name_meituan=None, name_taobao=None, - id_taobao=None, deeplink=None, lat=None, lng=None): + id_taobao=None, deeplink=None, lat=None, lng=None, + name_jd=None, id_jd=None, jd_deeplink=None): row = StoreMapping( trace_id=trace_id, business_type="food", name_meituan=name_meituan, name_taobao=name_taobao, id_taobao=id_taobao, taobao_deeplink=deeplink, lat=lat, lng=lng, + name_jd=name_jd, id_jd=id_jd, jd_deeplink=jd_deeplink, ) db.add(row) db.commit() @@ -110,9 +112,45 @@ def test_invalidate_endpoint_marks_and_lookup_miss(client, db, monkeypatch): assert "taobao" not in lk.json() -def test_invalidate_endpoint_non_taobao_noop(client, monkeypatch): +def test_invalidate_endpoint_unsupported_platform_noop(client, monkeypatch): + # 当前支持 taobao/jd;其它平台(如 meituan)no-op 返 affected=0 monkeypatch.setattr(settings, "INTERNAL_API_SECRET", _SECRET) r = client.post("/internal/store-mapping/invalidate", - json={"platform": "jd", "shop_id": "X"}, + json={"platform": "meituan", "shop_id": "X"}, headers={"X-Internal-Secret": _SECRET}) assert r.status_code == 200 and r.json()["affected"] == 0 + + +# ---------- 京东(对称淘宝)---------- + +def test_mark_jd_invalid_marks_all_rows_with_store_id(db): + _mk_row(db, trace_id="j1", name_meituan="兰州拉面", name_jd="兰州拉面", id_jd="JBAD", jd_deeplink="dl_a") + _mk_row(db, trace_id="j2", name_meituan="兰州拉面", name_jd="兰州拉面", id_jd="JBAD", jd_deeplink="dl_b") + _mk_row(db, trace_id="j3", name_meituan="兰州拉面", name_jd="兰州拉面", id_jd="JGOOD", jd_deeplink="dl_g") + assert repo.mark_jd_deeplink_invalid(db, "JBAD") == 2 + assert repo.mark_jd_deeplink_invalid(db, "JBAD") == 0 # 幂等 + rows = {r.trace_id: r for r in db.query(StoreMapping).all()} + assert rows["j1"].jd_deeplink_invalid_at is not None + assert rows["j2"].jd_deeplink_invalid_at is not None + assert rows["j3"].jd_deeplink_invalid_at is None # 不同 storeId 不动 + + +def test_lookup_filters_invalid_jd(db): + _mk_row(db, trace_id="j1", name_meituan="兰州拉面", name_jd="兰州拉面", id_jd="JBAD", jd_deeplink="dl") + assert repo.lookup_nearest(db, "meituan", "兰州拉面")["jd"]["store_id"] == "JBAD" + repo.mark_jd_deeplink_invalid(db, "JBAD") + assert "jd" not in repo.lookup_nearest(db, "meituan", "兰州拉面") + + +def test_invalidate_endpoint_jd_marks_and_lookup_miss(client, db, monkeypatch): + monkeypatch.setattr(settings, "INTERNAL_API_SECRET", _SECRET) + _mk_row(db, trace_id="j1", name_meituan="店J", name_jd="店J", id_jd="JBADX", jd_deeplink="dl") + _mk_row(db, trace_id="j2", name_meituan="店J", name_jd="店J", id_jd="JBADX", jd_deeplink="dl2") + r = client.post("/internal/store-mapping/invalidate", + json={"platform": "jd", "shop_id": "JBADX"}, + headers={"X-Internal-Secret": _SECRET}) + assert r.status_code == 200 and r.json()["affected"] == 2 + lk = client.get("/internal/store-mapping/lookup", + params={"source_platform": "meituan", "name": "店J"}, + headers={"X-Internal-Secret": _SECRET}) + assert "jd" not in lk.json() From f97048ff56f353cd4dc95adbd1183e7fe665b17e Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 16 Jun 2026 01:39:05 +0800 Subject: [PATCH 3/7] =?UTF-8?q?fix(alembic):=20=E5=90=88=E5=B9=B6=E5=A4=9A?= =?UTF-8?q?=20head=20(store=5Fmapping=5Fjd=5Fdl=5Finvalid=20+=20ad=5Freven?= =?UTF-8?q?ue)=20=E4=BF=AE=E5=A4=8D=200.1.2=20=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #56 的 jd_deeplink_invalid 迁移与 ad_revenue 报表迁移从同一 branchpoint 分叉成双 head, 部署时 alembic upgrade head 报 "Multiple head revisions"。 加一个 no-op merge 节点统一 DAG, 两支迁移列不冲突。 Co-Authored-By: Claude Opus 4.8 --- ..._merge_store_mapping_jd_dl_invalid_and_.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 alembic/versions/45047b5a884c_merge_store_mapping_jd_dl_invalid_and_.py diff --git a/alembic/versions/45047b5a884c_merge_store_mapping_jd_dl_invalid_and_.py b/alembic/versions/45047b5a884c_merge_store_mapping_jd_dl_invalid_and_.py new file mode 100644 index 0000000..58c3271 --- /dev/null +++ b/alembic/versions/45047b5a884c_merge_store_mapping_jd_dl_invalid_and_.py @@ -0,0 +1,26 @@ +"""merge store_mapping_jd_dl_invalid and ad_revenue heads (0.1.2) + +Revision ID: 45047b5a884c +Revises: d4e68464761d, store_mapping_jd_dl_invalid +Create Date: 2026-06-16 01:38:26.273226 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '45047b5a884c' +down_revision: Union[str, Sequence[str], None] = ('d4e68464761d', 'store_mapping_jd_dl_invalid') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass From 783dfd059d26964c6ad686c22fddda7119e7b885 Mon Sep 17 00:00:00 2001 From: chenshuobo Date: Tue, 16 Jun 2026 15:40:46 +0800 Subject: [PATCH 4/7] =?UTF-8?q?feat(meituan-etl):=20=E7=A6=BB=E7=BA=BF?= =?UTF-8?q?=E5=BA=93=E6=89=A9=E5=88=B0=E5=85=A8=E5=9B=BD=20359=20=E5=9F=8E?= =?UTF-8?q?(=E5=A4=9A=E5=9F=8E=E5=B9=B6=E5=8F=91=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E5=85=A5=E5=BA=93)=20(#57)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 智能推荐 / 销量最高两 tab 的离线库此前只有北京(ETL 写死 cityId),改为遍历 美团官方城市字典 359 个地级市全量抓取。实测一个地级市 cityId 已覆盖其下辖县级市 (徐州→邳州/新沂/睢宁等),按地级市抓即可,无需区县层级。 - 城市字典:tools/gen_meituan_cities.py 从美团 Excel 生成随仓库 JSON (app/integrations/data/meituan_cities.json,359 城),app/integrations/cities.py 读取; - ETL:city_id 参数化 + 城市级并发(默认 12)+ worker 启动错峰削平瞬时峰值 + 主线程逐城串行入库(Session 不跨线程); - 配速实测:单城全量 ~110s/~2300 条;15 并发抓完一轮 ~50-60min,402 占 3% 退避全消化; 每 3h 一轮全量(--interval 10800),窗口充裕; - prune 双护栏:本轮 0 入库 或 失败城占比 >5% 时跳过,防上游故障/大面积限流误删全表; - 仅写入侧;读取侧(rec/top-sales 按城过滤)待后续(依赖用户定位→cityId 映射,字典无经纬度)。 --------- Co-authored-by: chenshuobo <1119780489@qq.com> Reviewed-on: https://gitea.shaguabijia.com/WonderableAI/shaguabijia-app-server/pulls/57 Co-authored-by: chenshuobo Co-committed-by: chenshuobo --- app/integrations/cities.py | 34 + app/integrations/data/meituan_cities.json | 1797 +++++++++++++++++++++ scripts/pull_meituan_coupons.py | 215 ++- tools/gen_meituan_cities.py | 57 + 4 files changed, 2053 insertions(+), 50 deletions(-) create mode 100644 app/integrations/cities.py create mode 100644 app/integrations/data/meituan_cities.json create mode 100644 tools/gen_meituan_cities.py diff --git a/app/integrations/cities.py b/app/integrations/cities.py new file mode 100644 index 0000000..a91fa55 --- /dev/null +++ b/app/integrations/cities.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +"""美团城市字典(地级市)读取。 + +数据源:app/integrations/data/meituan_cities.json,由 tools/gen_meituan_cities.py 从 +美团官方「城市字典」Excel 生成(359 个地级市:city_id / 城市名 / 省份)。 +实测一个地级市 cityId 已覆盖其下辖县级市供给(徐州 → 邳州 / 新沂 / 睢宁 …),故无区县层级。 + +- ETL(scripts/pull_meituan_coupons.py)遍历 all_cities() 全量抓取入库。 +- 读取侧将来「按城市过滤」时,也从这里取 city_id ↔ 城市名映射。 +""" +from __future__ import annotations + +import functools +import json +from pathlib import Path + +_JSON = Path(__file__).resolve().parent / "data" / "meituan_cities.json" + + +@functools.lru_cache(maxsize=1) +def all_cities() -> list[dict]: + """全部城市 [{city_id, name, province}, ...](359 个地级市)。""" + return json.loads(_JSON.read_text(encoding="utf-8")) + + +@functools.lru_cache(maxsize=1) +def _by_id() -> dict[str, dict]: + return {c["city_id"]: c for c in all_cities()} + + +def city_name(city_id: str) -> str | None: + """city_id → 城市名(查不到返回 None)。""" + c = _by_id().get(city_id) + return c["name"] if c else None diff --git a/app/integrations/data/meituan_cities.json b/app/integrations/data/meituan_cities.json new file mode 100644 index 0000000..8431142 --- /dev/null +++ b/app/integrations/data/meituan_cities.json @@ -0,0 +1,1797 @@ +[ + { + "city_id": "3NUYJKKJXPHVNZUHFK3HWUDHNM", + "name": "宣城市", + "province": "安徽省" + }, + { + "city_id": "LXXSHOY7LNK74ZK2SKVUXFY72Q", + "name": "阜阳市", + "province": "安徽省" + }, + { + "city_id": "ZEBF2LBJOEHGM4XGFPNW4IHBIA", + "name": "合肥市", + "province": "安徽省" + }, + { + "city_id": "WADWEY3GR6IARJLSNGQWG2KI4E", + "name": "滁州市", + "province": "安徽省" + }, + { + "city_id": "Z62QL3X66AOT6MSY7LB6LVO4CI", + "name": "芜湖市", + "province": "安徽省" + }, + { + "city_id": "M6QCWRCECRT6ZEJ6RERR6IWHFI", + "name": "淮南市", + "province": "安徽省" + }, + { + "city_id": "UP2NBJACSAO7FUH4XQQDMBOUE4", + "name": "马鞍山市", + "province": "安徽省" + }, + { + "city_id": "WLL7BSHUBPTLTMITFITJX2GKWY", + "name": "蚌埠市", + "province": "安徽省" + }, + { + "city_id": "ECTBNJ4KNNHPGXEVI4TBJNU7BY", + "name": "亳州市", + "province": "安徽省" + }, + { + "city_id": "KNEGDNOSRGNQPAMOMH54ATLIUU", + "name": "六安市", + "province": "安徽省" + }, + { + "city_id": "YH53FRR55H6VA4KXW36OJ7RNSE", + "name": "宿州市", + "province": "安徽省" + }, + { + "city_id": "LH6LX5DUVFPOFCAQ6NEGPWCLBI", + "name": "淮北市", + "province": "安徽省" + }, + { + "city_id": "RIFXWJ46SEJXE7EP6HZXEUIALU", + "name": "铜陵市", + "province": "安徽省" + }, + { + "city_id": "FK6F7KMO4WNARZKUXMMSBWGATM", + "name": "安庆市", + "province": "安徽省" + }, + { + "city_id": "M3CHGNNUSFSPS5HRV3W7MCBWFI", + "name": "黄山市", + "province": "安徽省" + }, + { + "city_id": "XFI6PM7SRO6NHBGYQUZEEA2WT4", + "name": "池州市", + "province": "安徽省" + }, + { + "city_id": "D2ZILSASTTGEFQWZIDTHA7PWU4", + "name": "澳门", + "province": "澳门特别行政区" + }, + { + "city_id": "WKV2HMXUEK634WP64CUCUQGM64", + "name": "北京市", + "province": "北京市" + }, + { + "city_id": "HPMKHLM3QR6EZGMY7GEI4H3QYQ", + "name": "泉州市", + "province": "福建省" + }, + { + "city_id": "6V523YRU54Y3PEAM2XPADNJM2U", + "name": "福州市", + "province": "福建省" + }, + { + "city_id": "HH3ZZCERPVQIYUZPW4A2U4JKZI", + "name": "莆田市", + "province": "福建省" + }, + { + "city_id": "BQ5RWEJS4O7W27SQMLPMRIRJDU", + "name": "宁德市", + "province": "福建省" + }, + { + "city_id": "TDN7XDQMZEP6ZCK6UO3VVMCSMM", + "name": "三明市", + "province": "福建省" + }, + { + "city_id": "ZKNTORN6YTZL2BXRYUSRGV3CHU", + "name": "厦门市", + "province": "福建省" + }, + { + "city_id": "Y2DI2QLZN5NNACMD3KI2DBR4IA", + "name": "龙岩市", + "province": "福建省" + }, + { + "city_id": "C4RS32I6QAHWLP55UQWI3N5LLY", + "name": "南平市", + "province": "福建省" + }, + { + "city_id": "5T23WDEOAP7RYNU4JNL2ZG7PDQ", + "name": "漳州市", + "province": "福建省" + }, + { + "city_id": "UHZBROATFB2KWNMNLS23DPRJBY", + "name": "定西市", + "province": "甘肃省" + }, + { + "city_id": "WLPAHIUOIVKS644QSN4V5ZY5XQ", + "name": "金昌市", + "province": "甘肃省" + }, + { + "city_id": "J7TO3UHZ57ABNUKIQUBCIJZQFA", + "name": "白银市", + "province": "甘肃省" + }, + { + "city_id": "GX277SS75375VEFYVCEHBL6ISA", + "name": "临夏回族自治州", + "province": "甘肃省" + }, + { + "city_id": "ORA3R7F2LSJHUOCIDTZCP54G7Q", + "name": "张掖市", + "province": "甘肃省" + }, + { + "city_id": "L6VQYYOJNTHSLW5JCR5SXBITDM", + "name": "武威市", + "province": "甘肃省" + }, + { + "city_id": "65OXQPQVFXNOYHXTMCE2RNFRL4", + "name": "兰州市", + "province": "甘肃省" + }, + { + "city_id": "IUIZYQ7E2SPMEAIGOUUNZWBQOA", + "name": "天水市", + "province": "甘肃省" + }, + { + "city_id": "BYBRGRDRV4NKAWCU6GWBIVLX3Q", + "name": "酒泉市", + "province": "甘肃省" + }, + { + "city_id": "KDST2VRETG6WK5SMJO2G2FN2RU", + "name": "嘉峪关市", + "province": "甘肃省" + }, + { + "city_id": "R3Q2XWVFVF4T2ADZZZMPZJRWBA", + "name": "庆阳市", + "province": "甘肃省" + }, + { + "city_id": "7FWNT2TP66SU4QEP6IBBMZWFNE", + "name": "陇南市", + "province": "甘肃省" + }, + { + "city_id": "T33S2GYGPVHAL5SR2FLSPTJSBE", + "name": "平凉市", + "province": "甘肃省" + }, + { + "city_id": "RR6KAWBLOKD4H2UINKYPFCPXO4", + "name": "甘南藏族自治州", + "province": "甘肃省" + }, + { + "city_id": "QKX4DS3CTJJG7SFW5SBHPLD42I", + "name": "茂名市", + "province": "广东省" + }, + { + "city_id": "JJZ75A32XCQNZU4IN2ZCEUGN3M", + "name": "梅州市", + "province": "广东省" + }, + { + "city_id": "SJSOXOSJASLUT6LBH4E32SUKKQ", + "name": "清远市", + "province": "广东省" + }, + { + "city_id": "SQQWAN5BQOVSX55S7EPF7QHMAU", + "name": "珠海市", + "province": "广东省" + }, + { + "city_id": "NGRJMW6JMRS6U2KUJEONSORAEY", + "name": "韶关市", + "province": "广东省" + }, + { + "city_id": "FAUMIGOSOET4E5WR5BL6P3OZHA", + "name": "佛山市", + "province": "广东省" + }, + { + "city_id": "JSBIH55ICFZQ2D3LEV47YMZ2NI", + "name": "河源市", + "province": "广东省" + }, + { + "city_id": "KOYAYPD2DBLDCF2ZI5GCW4LF6Y", + "name": "中山市", + "province": "广东省" + }, + { + "city_id": "647JGFPUYM4VWVLZCPSHT63XKQ", + "name": "汕头市", + "province": "广东省" + }, + { + "city_id": "AMOIPZW3Q2NMTDSEREFVM4SV74", + "name": "深圳市", + "province": "广东省" + }, + { + "city_id": "XIHEJY4H2CDZJCLIXDIN36BKXQ", + "name": "广州市", + "province": "广东省" + }, + { + "city_id": "UZ6OT4CYUR42KCTTED2KJW6EGA", + "name": "东莞市", + "province": "广东省" + }, + { + "city_id": "RM3HLOIUEYKTQ5O2JSVEKDMFRY", + "name": "阳江市", + "province": "广东省" + }, + { + "city_id": "C6XNJAZA6N3NNUDBUU3JIZPTDI", + "name": "潮州市", + "province": "广东省" + }, + { + "city_id": "AUPF3G2ULSV4TDT4L3NMHRTY6Y", + "name": "揭阳市", + "province": "广东省" + }, + { + "city_id": "7HIITKBPRXTVBA2FBBN443XITQ", + "name": "云浮市", + "province": "广东省" + }, + { + "city_id": "TO6ILZ7MPJMJN3S7W2SXMIFQQY", + "name": "江门市", + "province": "广东省" + }, + { + "city_id": "DTTBMGCIOMPCZY5NETUEKWJ6PY", + "name": "汕尾市", + "province": "广东省" + }, + { + "city_id": "HJY7JYWBA6FQY42RYSKX3RZTRI", + "name": "湛江市", + "province": "广东省" + }, + { + "city_id": "SLICHB4FBDVDLI53MR74WVUUNI", + "name": "肇庆市", + "province": "广东省" + }, + { + "city_id": "ZPX4JXJVBBYSSD2KTWHAPXO6NE", + "name": "惠州市", + "province": "广东省" + }, + { + "city_id": "JH4Q44RQA4EZ3Q6MHQVEVE7KZQ", + "name": "百色市", + "province": "广西壮族自治区" + }, + { + "city_id": "H2JXFEJFIL4PPFMYOS4MHZ5IBM", + "name": "崇左市", + "province": "广西壮族自治区" + }, + { + "city_id": "SXIRRISUOEGBU335AWT2ZFL6A4", + "name": "贵港市", + "province": "广西壮族自治区" + }, + { + "city_id": "HEQHKC4KP7YGGYVBZM5JEUI5AQ", + "name": "北海市", + "province": "广西壮族自治区" + }, + { + "city_id": "SKGG7KMFKVDIDKRVQEPTS7SIE4", + "name": "贺州市", + "province": "广西壮族自治区" + }, + { + "city_id": "N4WR7CWCULNA5Z35OTDJSZYDCU", + "name": "钦州市", + "province": "广西壮族自治区" + }, + { + "city_id": "T4RXX2WY6WPQYZEUXVJNZBXQZU", + "name": "梧州市", + "province": "广西壮族自治区" + }, + { + "city_id": "3R23AS3EIY7EYE2D5MWWORZODI", + "name": "河池市", + "province": "广西壮族自治区" + }, + { + "city_id": "57SMWWCV7X44E256P4I23OQ3AA", + "name": "防城港市", + "province": "广西壮族自治区" + }, + { + "city_id": "YHGHVIQ37UCTNQ4JKPQEAUWIQA", + "name": "桂林市", + "province": "广西壮族自治区" + }, + { + "city_id": "MQJZTM455OKZLAN5WQYUTA5TDE", + "name": "柳州市", + "province": "广西壮族自治区" + }, + { + "city_id": "C6FZPLB4NJQ6VUPSKDJH3EWQDM", + "name": "玉林市", + "province": "广西壮族自治区" + }, + { + "city_id": "BFSU5W6E5XBIDFPQLVPGRDSATY", + "name": "南宁市", + "province": "广西壮族自治区" + }, + { + "city_id": "CKXOQUZDNOVNME3PEBOY2CULQQ", + "name": "来宾市", + "province": "广西壮族自治区" + }, + { + "city_id": "LV32FV6IQTKFR7JIBEQMHRUVCA", + "name": "贵阳市", + "province": "贵州省" + }, + { + "city_id": "F26RCNKMFTZONJCSJ5C6FHVY74", + "name": "毕节市", + "province": "贵州省" + }, + { + "city_id": "G2LMYRWVRCK7BTD2WM4NWX4SYM", + "name": "黔南布依族苗族自治州", + "province": "贵州省" + }, + { + "city_id": "MFSOO3NBMB2PVLIVSI5EJK7MWY", + "name": "黔西南布依族苗族自治州", + "province": "贵州省" + }, + { + "city_id": "KWUL44L7SEMJGIMXCWSSEB3OOA", + "name": "遵义市", + "province": "贵州省" + }, + { + "city_id": "T2P3OFGQZUGR7D6TGRCMDF22GI", + "name": "铜仁市", + "province": "贵州省" + }, + { + "city_id": "AGUFUANSZNGC4TMOPZO65IRSPI", + "name": "六盘水市", + "province": "贵州省" + }, + { + "city_id": "2XOCOSNUAK3J5QDGTKIBWKK7KU", + "name": "安顺市", + "province": "贵州省" + }, + { + "city_id": "6XRTSAEYJTA2UBKXO4XEPQE5ZY", + "name": "黔东南苗族侗族自治州", + "province": "贵州省" + }, + { + "city_id": "YRMKRP2GOE2VMRS73N4YRIZUHY", + "name": "三亚市", + "province": "海南省" + }, + { + "city_id": "CJRGVLBNLJAVBJ4ZKKIZ3FZ2LY", + "name": "白沙黎族自治县", + "province": "海南省" + }, + { + "city_id": "5XOUAJ5Z4J4K7SVIQGXL2OM2JQ", + "name": "保亭黎族苗族自治县", + "province": "海南省" + }, + { + "city_id": "2UFQ6A2QRJPXPH3VOYEAQHMVSQ", + "name": "海口市", + "province": "海南省" + }, + { + "city_id": "TGCVXVS4M7NDQM4ROUDCVI6I3A", + "name": "承德市", + "province": "河北省" + }, + { + "city_id": "JEUP6QWCOXPSM3SQTINQCJKIGM", + "name": "衡水市", + "province": "河北省" + }, + { + "city_id": "Z442MNCW6BO2BBHIRUPRBACXPI", + "name": "唐山市", + "province": "河北省" + }, + { + "city_id": "RFE6R34GD4FY3LUKC2ICSF6AFY", + "name": "张家口市", + "province": "河北省" + }, + { + "city_id": "CWJN55M73VZDCYJEQ7AHDBWGGY", + "name": "沧州市", + "province": "河北省" + }, + { + "city_id": "DFL4ES776ECRGBYNOPLWKB247I", + "name": "雄安新区", + "province": "河北省" + }, + { + "city_id": "ZLSXYY34IHBHIC2NOVPQQBFTBE", + "name": "保定市", + "province": "河北省" + }, + { + "city_id": "3DO6Z2QRJQFMPLLDS55PG7DSBU", + "name": "石家庄市", + "province": "河北省" + }, + { + "city_id": "PR57XT25LI3246VGASEPSHP63E", + "name": "邢台市", + "province": "河北省" + }, + { + "city_id": "SKYLNH737BS56TD452FOKYL36U", + "name": "邯郸市", + "province": "河北省" + }, + { + "city_id": "5PWPERL7GQKJD6QPLR2TWUUM7E", + "name": "秦皇岛市", + "province": "河北省" + }, + { + "city_id": "5T2TGV6SJFVL3MO7HIMN2KTQTA", + "name": "廊坊市", + "province": "河北省" + }, + { + "city_id": "ECSTLZ7GP7IX3MB5EVNKS47MLE", + "name": "焦作市", + "province": "河南省" + }, + { + "city_id": "VTWW34QB2F5Q4LW7ISNUMWX7GY", + "name": "开封市", + "province": "河南省" + }, + { + "city_id": "D2NUN47NY4Q55X3UED4JMSI6CM", + "name": "周口市", + "province": "河南省" + }, + { + "city_id": "TR3XJFQR4EFYRRIX7TUQF3B26Y", + "name": "郑州市", + "province": "河南省" + }, + { + "city_id": "CKJGF5S6XMHW5ZJEBU7MJC47QA", + "name": "新乡市", + "province": "河南省" + }, + { + "city_id": "IFASZ625MCFJQKPLJ7EA2SMJUU", + "name": "商丘市", + "province": "河南省" + }, + { + "city_id": "SKXPYKTTRG4YAUHE2HZXWRWXGM", + "name": "鹤壁市", + "province": "河南省" + }, + { + "city_id": "G5LXE74CUHO2K6BBRN7Q5DSRJY", + "name": "漯河市", + "province": "河南省" + }, + { + "city_id": "SLNOAFJV2LTBSH7SJCRXJA36K4", + "name": "驻马店市", + "province": "河南省" + }, + { + "city_id": "65WO7LH7CFDUGKYMXQLRAYKKWM", + "name": "安阳市", + "province": "河南省" + }, + { + "city_id": "RIX2X7FAVTZCAQ5RT2C2CWK22Q", + "name": "南阳市", + "province": "河南省" + }, + { + "city_id": "SZOW5OY3U54SSY4WRC65VJUNTI", + "name": "平顶山市", + "province": "河南省" + }, + { + "city_id": "7VPIDDUS4P2LSZ6Q5S57MAQDEM", + "name": "信阳市", + "province": "河南省" + }, + { + "city_id": "4VYCRORUOZ4DC2U6S3CT6H6KWE", + "name": "洛阳市", + "province": "河南省" + }, + { + "city_id": "M5WNO2BQ3UGLLHBCG4NCEFPP5U", + "name": "濮阳市", + "province": "河南省" + }, + { + "city_id": "LY3O6PBPIWETMA3ZOL6ETY5UB4", + "name": "三门峡市", + "province": "河南省" + }, + { + "city_id": "FXXJLIRE72LS2W4OWWQVJMRJHA", + "name": "许昌市", + "province": "河南省" + }, + { + "city_id": "5XH353QTY3VWF2KYOCZCL3TOXY", + "name": "牡丹江市", + "province": "黑龙江省" + }, + { + "city_id": "2KGRZKF6IECV2W7K5J64Y2LY4M", + "name": "齐齐哈尔市", + "province": "黑龙江省" + }, + { + "city_id": "FASGWS5ADVSTFGJG6TGBZPZP6Q", + "name": "鹤岗市", + "province": "黑龙江省" + }, + { + "city_id": "2O6CDIXSWIKBXILZEEPKCS7MVI", + "name": "双鸭山市", + "province": "黑龙江省" + }, + { + "city_id": "OZ2PTOBYTBG57XJZMIC23QFJKM", + "name": "佳木斯市", + "province": "黑龙江省" + }, + { + "city_id": "FO24MQMULT3J5JW64APNXSQEPU", + "name": "伊春市", + "province": "黑龙江省" + }, + { + "city_id": "ETZ2HYWVU6U7SKU6G4JAO64RUQ", + "name": "黑河市", + "province": "黑龙江省" + }, + { + "city_id": "PR7EJNBY2VZBEUT36JAWE3TM7I", + "name": "七台河市", + "province": "黑龙江省" + }, + { + "city_id": "HADAAVLERKIW4SQGCTQYGX4AL4", + "name": "哈尔滨市", + "province": "黑龙江省" + }, + { + "city_id": "CGTU45YC5C3JYLHMA47USDPA7Y", + "name": "大庆市", + "province": "黑龙江省" + }, + { + "city_id": "T4W7SQIPOM4EYMEFFRAB5BSTII", + "name": "鸡西市", + "province": "黑龙江省" + }, + { + "city_id": "TYGZHNQL6YT7CX6EEG5DJQQHMA", + "name": "绥化市", + "province": "黑龙江省" + }, + { + "city_id": "OOSJTSN2CVUUCKD6XAB7EYYIPY", + "name": "大兴安岭地区", + "province": "黑龙江省" + }, + { + "city_id": "I3YF3EKZHIZTN6TZOTYTGZ2UXQ", + "name": "随州市", + "province": "湖北省" + }, + { + "city_id": "ESGVBOSTHW7JWEVCGYJUTEHEBQ", + "name": "宜昌市", + "province": "湖北省" + }, + { + "city_id": "MTJRWJ53XBW5SBTWHKNNZDLM7U", + "name": "十堰市", + "province": "湖北省" + }, + { + "city_id": "PXZLF2ISKQL5ACM67ZCBNOGDT4", + "name": "黄石市", + "province": "湖北省" + }, + { + "city_id": "44RMTOEHPUFXBHZXX4IQ4IRZVQ", + "name": "荆州市", + "province": "湖北省" + }, + { + "city_id": "OTKZGG743NFC474ADMMRX4ZOOA", + "name": "鄂州市", + "province": "湖北省" + }, + { + "city_id": "EXOUAZAQ73OEFAK72CHQ32GQHQ", + "name": "恩施土家族苗族自治州", + "province": "湖北省" + }, + { + "city_id": "SUCY7I72QJDZD7EBFXREIQ67SI", + "name": "咸宁市", + "province": "湖北省" + }, + { + "city_id": "QENSGB5R7HGYDXCG2LQZQTO3TU", + "name": "荆门市", + "province": "湖北省" + }, + { + "city_id": "OHIWL6SAE2PR4EJR4BOMLAE6FU", + "name": "武汉市", + "province": "湖北省" + }, + { + "city_id": "ZXCE4WV2CDVPQTA4HAOVELQMNE", + "name": "襄阳市", + "province": "湖北省" + }, + { + "city_id": "KEFN5OPSS4ZZF6NU2TTL72S6HE", + "name": "孝感市", + "province": "湖北省" + }, + { + "city_id": "ROAHLMQ67H6M5NDFVXROJG723E", + "name": "黄冈市", + "province": "湖北省" + }, + { + "city_id": "YBEBX2YYN4WPBNH6Z6C73DNE7I", + "name": "张家界市", + "province": "湖南省" + }, + { + "city_id": "45XGRKYGSCPE5VNRYF4FVJGFMM", + "name": "株洲市", + "province": "湖南省" + }, + { + "city_id": "LK3SEIBRU7GTDT4J2EPTLIO33U", + "name": "永州市", + "province": "湖南省" + }, + { + "city_id": "R4YXFIK53W5E556BSGSBJWS4DM", + "name": "郴州市", + "province": "湖南省" + }, + { + "city_id": "PQDO3RNADWXX75OWZW2GSXJ4SE", + "name": "怀化市", + "province": "湖南省" + }, + { + "city_id": "RRRT6QOJYEJ432L3F76ZN5NHCA", + "name": "长沙市", + "province": "湖南省" + }, + { + "city_id": "B6WPNMCZ3ENQSV4NFY5MSTPDAM", + "name": "岳阳市", + "province": "湖南省" + }, + { + "city_id": "KNDZW5EHDPKP2DX7HBLKP4DYLM", + "name": "益阳市", + "province": "湖南省" + }, + { + "city_id": "PA2GHG3XZ7I47HTKZ4YAFH3OYY", + "name": "湘西土家族苗族自治州", + "province": "湖南省" + }, + { + "city_id": "I7CNIUA5PYV2EHDEW3RGYT2R4U", + "name": "邵阳市", + "province": "湖南省" + }, + { + "city_id": "SRI2SU4FN66FMJJCKQOCZD72ZY", + "name": "常德市", + "province": "湖南省" + }, + { + "city_id": "H7UHHJAMQUL7UA5QEUTGKNSL3A", + "name": "湘潭市", + "province": "湖南省" + }, + { + "city_id": "EFB255OBTB2BUDZENR5UVIC7ZQ", + "name": "衡阳市", + "province": "湖南省" + }, + { + "city_id": "RDMANB4KCM3OJSNVGZWVYVME6E", + "name": "娄底市", + "province": "湖南省" + }, + { + "city_id": "LD37PDU5OB4UAV5QDOBMKG5YTY", + "name": "吉林市", + "province": "吉林省" + }, + { + "city_id": "EO3GF4XNF5RXWRPUVAT3KTQO4U", + "name": "四平市", + "province": "吉林省" + }, + { + "city_id": "4GD7OS4CAQABH5YIWVK5SKGHMY", + "name": "通化市", + "province": "吉林省" + }, + { + "city_id": "6DRI2R5VAWMYJHJPCJKUNMDYEQ", + "name": "延边朝鲜族自治州", + "province": "吉林省" + }, + { + "city_id": "TVBCNVGUND4MOUOOUXFGX7DIUA", + "name": "松原市", + "province": "吉林省" + }, + { + "city_id": "JYY62HSKBUVK5OU7KGJDKQ4RTA", + "name": "白城市", + "province": "吉林省" + }, + { + "city_id": "QEDUHKMZ36CHJTKRD6O2ZPLNBU", + "name": "长春市", + "province": "吉林省" + }, + { + "city_id": "4EADVCBJMZ5UBH2FVRT6QCLS2U", + "name": "辽源市", + "province": "吉林省" + }, + { + "city_id": "EUQD5EGS2LR5KJSFNG6PPSIHHI", + "name": "白山市", + "province": "吉林省" + }, + { + "city_id": "YLTIISPCLBEGTZZX3WUWAD7WDE", + "name": "淮安市", + "province": "江苏省" + }, + { + "city_id": "L6U5DZP6MESXPMHOHCDMJS55O4", + "name": "宿迁市", + "province": "江苏省" + }, + { + "city_id": "HQMLYA7TDGMYQAXFCDUXBZPYHI", + "name": "镇江市", + "province": "江苏省" + }, + { + "city_id": "K6XJ4UN65ZD6XQKYEG5YN7HCRI", + "name": "盐城市", + "province": "江苏省" + }, + { + "city_id": "36I4X3EZZU4EHOSCLQI5OAKKBE", + "name": "南通市", + "province": "江苏省" + }, + { + "city_id": "S3GWFQU6QAVRDKLJT77LD6OFLE", + "name": "泰州市", + "province": "江苏省" + }, + { + "city_id": "IO6F4AFGAVIFRGYTZEC4TXM7W4", + "name": "无锡市", + "province": "江苏省" + }, + { + "city_id": "NUXNK2VOFSD2JFTEO2AMWX6NSU", + "name": "扬州市", + "province": "江苏省" + }, + { + "city_id": "TEVZU6CU6SK57HFW7DFNGMQ44A", + "name": "南京市", + "province": "江苏省" + }, + { + "city_id": "UTYSRBQ4FSB7XLWCF3Z2HTKNUA", + "name": "常州市", + "province": "江苏省" + }, + { + "city_id": "OCZOBCJDEXKE7KBN3BD7AYQG2Q", + "name": "徐州市", + "province": "江苏省" + }, + { + "city_id": "6LIBPJGZROLXE3CLZGJRYMYBOU", + "name": "连云港市", + "province": "江苏省" + }, + { + "city_id": "FS4PIU74F7QKYARDWR5ZMOLICI", + "name": "苏州市", + "province": "江苏省" + }, + { + "city_id": "R2F4OWUO65HYZW2IQIKINORZ7Y", + "name": "赣州市", + "province": "江西省" + }, + { + "city_id": "YW346BTN3VFYNRC3744UR5MZXY", + "name": "抚州市", + "province": "江西省" + }, + { + "city_id": "QR3FDR26U2EJIXOMBHL7IJLQSA", + "name": "南昌市", + "province": "江西省" + }, + { + "city_id": "OAJHJL7L7VNW2Q5UXRE7F4CUJQ", + "name": "九江市", + "province": "江西省" + }, + { + "city_id": "OMH7D45R4DX2KNHLV3G2UP56OY", + "name": "景德镇市", + "province": "江西省" + }, + { + "city_id": "YSB2PAEROB2IZSJZFVFH7KJPEI", + "name": "鹰潭市", + "province": "江西省" + }, + { + "city_id": "5OYAMNORCXKYA6UF7DW6KFFBIU", + "name": "上饶市", + "province": "江西省" + }, + { + "city_id": "SMHZOYKE7BXQJ2NT6Q24TFMLEQ", + "name": "吉安市", + "province": "江西省" + }, + { + "city_id": "2RZV26OUPKUHUJ5ZPB673VDGZU", + "name": "萍乡市", + "province": "江西省" + }, + { + "city_id": "232VHZEEZ6SXACE4AC5HQ4ZTFQ", + "name": "新余市", + "province": "江西省" + }, + { + "city_id": "QRLM74YXNDW2QDBWLTFGEMXK2I", + "name": "宜春市", + "province": "江西省" + }, + { + "city_id": "S6OHUVUKIIWPVMQD44RREUMNT4", + "name": "葫芦岛市", + "province": "辽宁省" + }, + { + "city_id": "DQQ4OIFUGFYJY3XZRK5VDWMLCA", + "name": "辽阳市", + "province": "辽宁省" + }, + { + "city_id": "S4YXGFEYXEUG6ISZ6O337OPVSI", + "name": "阜新市", + "province": "辽宁省" + }, + { + "city_id": "D3JHM7A4CG6RJMBD7YRDS5JOYU", + "name": "盘锦市", + "province": "辽宁省" + }, + { + "city_id": "VTRWMOSS6PCUYUAIPG6VPBKUUQ", + "name": "营口市", + "province": "辽宁省" + }, + { + "city_id": "ZPHFGWBIEVLKP5CVZNZUB3CRT4", + "name": "朝阳市", + "province": "辽宁省" + }, + { + "city_id": "NGYYULZ4UAGD3Q2PG726FFXSHU", + "name": "抚顺市", + "province": "辽宁省" + }, + { + "city_id": "Q5BRTSW752VSHIAKLLL7KL5TNA", + "name": "锦州市", + "province": "辽宁省" + }, + { + "city_id": "ZGV3WNPOSS7J4ZWBP6ZQG46BNM", + "name": "沈阳市", + "province": "辽宁省" + }, + { + "city_id": "XEX676YYMTYIV5QPIUZB4TA7IY", + "name": "本溪市", + "province": "辽宁省" + }, + { + "city_id": "PRTEQZMLNLQNZXJHRCYYLWZB4E", + "name": "丹东市", + "province": "辽宁省" + }, + { + "city_id": "4GN4WF6UQRFU64T4FVZPRDXRWQ", + "name": "鞍山市", + "province": "辽宁省" + }, + { + "city_id": "3QTZDFLJFSLLOOVCZ65PSDAVOU", + "name": "铁岭市", + "province": "辽宁省" + }, + { + "city_id": "CC4ZTMKKXI73ZEVT5QQTJN5SMM", + "name": "大连市", + "province": "辽宁省" + }, + { + "city_id": "3MBJEFDLAVOMQZ7L7CM5MNSYKA", + "name": "鄂尔多斯市", + "province": "内蒙古自治区" + }, + { + "city_id": "5NOS4YC5WO2IZCQPVB6MCBYDJ4", + "name": "呼和浩特市", + "province": "内蒙古自治区" + }, + { + "city_id": "OQNIP675H7L5R64652BH7KHUOQ", + "name": "通辽市", + "province": "内蒙古自治区" + }, + { + "city_id": "4WA6I63MGVINV5DNLNWRRHCDDM", + "name": "阿拉善盟", + "province": "内蒙古自治区" + }, + { + "city_id": "ELI6BDJBAN6RCYTETMK2EX2UKU", + "name": "乌兰察布市", + "province": "内蒙古自治区" + }, + { + "city_id": "PV5ZAAXFW2DZCVZKCF4I4KK7BQ", + "name": "巴彦淖尔市", + "province": "内蒙古自治区" + }, + { + "city_id": "YU6UUT6G6T6AMWTJFECDIUQFEQ", + "name": "乌海市", + "province": "内蒙古自治区" + }, + { + "city_id": "V4MYANW5QFZCXG3FIDPXA3HOTE", + "name": "呼伦贝尔市", + "province": "内蒙古自治区" + }, + { + "city_id": "S5DCFJWJ7J3MJSLY2PHKWLNPOQ", + "name": "包头市", + "province": "内蒙古自治区" + }, + { + "city_id": "LY7SAZRFSJJMRU3JEO5SKNKIVM", + "name": "兴安盟", + "province": "内蒙古自治区" + }, + { + "city_id": "NM2XP54CNQCFOILKACYEQWUSGM", + "name": "锡林郭勒盟", + "province": "内蒙古自治区" + }, + { + "city_id": "S5G3IO75IDEJPZQA6VFM3OYPDI", + "name": "赤峰市", + "province": "内蒙古自治区" + }, + { + "city_id": "UUFUUPM5RT6ZU5UKILQC5YQV54", + "name": "吴忠市", + "province": "宁夏回族自治区" + }, + { + "city_id": "VMSRLIATK44WQXQEWAL63AXJ3M", + "name": "固原市", + "province": "宁夏回族自治区" + }, + { + "city_id": "4GWWCAAKGNJV2SMQPSWWZNCGYY", + "name": "中卫市", + "province": "宁夏回族自治区" + }, + { + "city_id": "6IE7GEETBQEF7GUSGU2FLIUEEM", + "name": "石嘴山市", + "province": "宁夏回族自治区" + }, + { + "city_id": "VI4YIH3URSON4Q4MWOEESXJ56Q", + "name": "银川市", + "province": "宁夏回族自治区" + }, + { + "city_id": "SIE4ED6QWVRT727GEHWBFH3DAA", + "name": "海东市", + "province": "青海省" + }, + { + "city_id": "JNJH6OJZIOQKXDXWW5ZGEHG5MA", + "name": "海西蒙古族藏族自治州", + "province": "青海省" + }, + { + "city_id": "MJADYNCKQNDJU2TXACTDP5I52M", + "name": "海北藏族自治州", + "province": "青海省" + }, + { + "city_id": "LRGFXIVB6RJWQWYAFH7EIUHCPE", + "name": "黄南藏族自治州", + "province": "青海省" + }, + { + "city_id": "J4TG3PCK2ZEMNEUMIPZF32UNQY", + "name": "果洛藏族自治州", + "province": "青海省" + }, + { + "city_id": "2YS5POGG53LKZGFBIUMDWP57SM", + "name": "玉树藏族自治州", + "province": "青海省" + }, + { + "city_id": "GRZMJEZCA2DNCZK3O6ZSUHMRPM", + "name": "西宁市", + "province": "青海省" + }, + { + "city_id": "NBFQIACRBBCAH5AZWJ5LVT7AU4", + "name": "海南藏族自治州", + "province": "青海省" + }, + { + "city_id": "MQUKCLQ76P4FRRECDBA3HBKT7Q", + "name": "滨州市", + "province": "山东省" + }, + { + "city_id": "633FVSBDDBM5WSMXSKOCX6QC5I", + "name": "潍坊市", + "province": "山东省" + }, + { + "city_id": "4434FVT3PXLMV6UAWLEW6O3M5A", + "name": "菏泽市", + "province": "山东省" + }, + { + "city_id": "P7PK4UBVCOHW3PI6IPEIA54DLY", + "name": "济南市", + "province": "山东省" + }, + { + "city_id": "I5M6JGTGSQEWX6HL7E5I6GRBAY", + "name": "德州市", + "province": "山东省" + }, + { + "city_id": "V562AOMBVU5NG5GB3EPK6U42XY", + "name": "烟台市", + "province": "山东省" + }, + { + "city_id": "4OSPHTE5TD24J6DYGR6DXMEDKY", + "name": "淄博市", + "province": "山东省" + }, + { + "city_id": "227TLAVTUJABWPJD4S4ZECJ3FY", + "name": "临沂市", + "province": "山东省" + }, + { + "city_id": "DAEZKZU32ZAPJGUTA6LLGO3WTY", + "name": "聊城市", + "province": "山东省" + }, + { + "city_id": "LBRRK2EOYJN5MLYJWT4R3QBSXM", + "name": "东营市", + "province": "山东省" + }, + { + "city_id": "AB6PBGCDBTNTG4KUQROY2FJ4GY", + "name": "枣庄市", + "province": "山东省" + }, + { + "city_id": "EVANGU7WZCVRAAM6NWTDJVP7SU", + "name": "济宁市", + "province": "山东省" + }, + { + "city_id": "GNUEGWZ3OKRWAKKVJ5THHHX6YY", + "name": "泰安市", + "province": "山东省" + }, + { + "city_id": "F3VWSF4ART2FYYBBOZYWKRXTUI", + "name": "青岛市", + "province": "山东省" + }, + { + "city_id": "KD6MNWWLVKB4E655XMV6MMA3KE", + "name": "日照市", + "province": "山东省" + }, + { + "city_id": "LHYVF4LBCOZ34G3WNZYUVEIQGA", + "name": "威海市", + "province": "山东省" + }, + { + "city_id": "ENVYDMYGDO3BMDXSAVQYZXLX74", + "name": "阳泉市", + "province": "山西省" + }, + { + "city_id": "UXOUG4UIF7ZRJYCNMQJ3LDN5FY", + "name": "临汾市", + "province": "山西省" + }, + { + "city_id": "4NZPT6Z35BMYACJ2HZGHUWRJ6E", + "name": "吕梁市", + "province": "山西省" + }, + { + "city_id": "KSNXQME2A3VFHCE3DM3SFZKIJQ", + "name": "晋城市", + "province": "山西省" + }, + { + "city_id": "HDOX7WKYSHJKEHET6TUYMVCTMQ", + "name": "太原市", + "province": "山西省" + }, + { + "city_id": "DFJIZVXJGBGBIABPSL3DGMIDIE", + "name": "长治市", + "province": "山西省" + }, + { + "city_id": "5KQYYTJR2EMP653QIALMA6LXXI", + "name": "忻州市", + "province": "山西省" + }, + { + "city_id": "T76EOJA332RIHML7B6LYS5LF4U", + "name": "朔州市", + "province": "山西省" + }, + { + "city_id": "HVX67CKT5TS6GPRDFDYOOLK4PE", + "name": "大同市", + "province": "山西省" + }, + { + "city_id": "S4NGXQJDOH7E4IHDWOH3EK6IIE", + "name": "晋中市", + "province": "山西省" + }, + { + "city_id": "GWDLZXLAWU54FKQ6G3HRQRR7E4", + "name": "运城市", + "province": "山西省" + }, + { + "city_id": "3FFTTN5PPV7MBCE5AGY2NGYOOI", + "name": "安康市", + "province": "陕西省" + }, + { + "city_id": "GACVPL3SWO3ZKH73JMJV6YI4NY", + "name": "延安市", + "province": "陕西省" + }, + { + "city_id": "6KPS7VRMW57P2DAC6OPR4ISHQQ", + "name": "商洛市", + "province": "陕西省" + }, + { + "city_id": "OMMF6XLNDNYWG5TBSNTWO2ZJZ4", + "name": "渭南市", + "province": "陕西省" + }, + { + "city_id": "EALFXGMWYRS6E6TWXQ2K3YHV4M", + "name": "咸阳市", + "province": "陕西省" + }, + { + "city_id": "WFG7U6JNUWDIS5ZZYM4FSM5C64", + "name": "榆林市", + "province": "陕西省" + }, + { + "city_id": "R3VBMYTCF5LVHO35X3MYJQFOOE", + "name": "宝鸡市", + "province": "陕西省" + }, + { + "city_id": "RQOWP7C234IS4RKSHB26IYZ5IU", + "name": "西安市", + "province": "陕西省" + }, + { + "city_id": "K4YU6B4T5GZLRWVHGVCR3576HI", + "name": "铜川市", + "province": "陕西省" + }, + { + "city_id": "VVFVAPLKSCN5KN4Q6RK2GPGUUA", + "name": "汉中市", + "province": "陕西省" + }, + { + "city_id": "2QSF6IG3KMDXWO5VP7FXHMMKXA", + "name": "上海市", + "province": "上海市" + }, + { + "city_id": "X3JCRNIPTCUU6DGOFFJ4MUK37M", + "name": "眉山市", + "province": "四川省" + }, + { + "city_id": "GQ24IZNTZJ3PUB5FDAMDA4W7UI", + "name": "攀枝花市", + "province": "四川省" + }, + { + "city_id": "6B6WT62WHBZRHPQUT7BAD2N6ZI", + "name": "泸州市", + "province": "四川省" + }, + { + "city_id": "HJ35P4KXL442MLIII7PFFWUNAE", + "name": "雅安市", + "province": "四川省" + }, + { + "city_id": "K4A6VSJH2AJYT46LMUSVQZPCCU", + "name": "资阳市", + "province": "四川省" + }, + { + "city_id": "646ZNPATOOM3MHI3LDU6HI4KFI", + "name": "阿坝藏族羌族自治州", + "province": "四川省" + }, + { + "city_id": "MU735ZDBFPXRQDUZ3I35JK3XEU", + "name": "内江市", + "province": "四川省" + }, + { + "city_id": "4WPGGJ63USY77GSRN2PPFCYKPQ", + "name": "广安市", + "province": "四川省" + }, + { + "city_id": "O4FFS4DALDAAKIFAUH4F5V5VS4", + "name": "宜宾市", + "province": "四川省" + }, + { + "city_id": "IRFJVK2KXBE6BZ7CSN4UFCI624", + "name": "绵阳市", + "province": "四川省" + }, + { + "city_id": "NELFD7FEKKUNDJ46VLD55SMDCE", + "name": "甘孜藏族自治州", + "province": "四川省" + }, + { + "city_id": "TKMVEUPZSQCXNRZPBEIK3F45AI", + "name": "遂宁市", + "province": "四川省" + }, + { + "city_id": "VQW7DPB4KTUI65COJBO3NODU24", + "name": "巴中市", + "province": "四川省" + }, + { + "city_id": "STP4ELXTVGQSB572LFRFJRIUUY", + "name": "南充市", + "province": "四川省" + }, + { + "city_id": "6ST5EX2JVXUCLR5GP5VEFSKN5M", + "name": "成都市", + "province": "四川省" + }, + { + "city_id": "UWNFCMW3HYJRALQI2MJH6EM2O4", + "name": "德阳市", + "province": "四川省" + }, + { + "city_id": "J5ZYU7XRV6CHSJAGOPQKS5YXNA", + "name": "达州市", + "province": "四川省" + }, + { + "city_id": "KYJTF5S746T35RFBMR65BLGM6U", + "name": "凉山彝族自治州", + "province": "四川省" + }, + { + "city_id": "RDUXR23XROLB4NGRVDDLXFXDSE", + "name": "乐山市", + "province": "四川省" + }, + { + "city_id": "4TUBIBHMVESJUCGMTUSLJPHXWI", + "name": "广元市", + "province": "四川省" + }, + { + "city_id": "AXQL57AO27NCHYMEOLRHAAKMTA", + "name": "自贡市", + "province": "四川省" + }, + { + "city_id": "4RXX566RZORCXS6HLEX3DIICSM", + "name": "花莲县", + "province": "台湾" + }, + { + "city_id": "BD5Y7SISWSSQGP3HTVPU6TXAH4", + "name": "台东县", + "province": "台湾" + }, + { + "city_id": "I4DNWLECRYOJAZLGYQB7PBBJXQ", + "name": "台中市", + "province": "台湾" + }, + { + "city_id": "GIZQIESFOMAEQSDQKOEQ5RTTPA", + "name": "南投县", + "province": "台湾" + }, + { + "city_id": "MPM6M2C634FAW7KYG3KIHERDTU", + "name": "彰化县", + "province": "台湾" + }, + { + "city_id": "NH2NK6JVOBBYYTK2G53ADWLX4Y", + "name": "苗栗县", + "province": "台湾" + }, + { + "city_id": "DW2Q2R2UEEDNQA7IHBGYV423K4", + "name": "新竹市", + "province": "台湾" + }, + { + "city_id": "FX5AOPFRPHGHHYZB4XNIPXLNNM", + "name": "新北市", + "province": "台湾" + }, + { + "city_id": "UVNZNB6G4M35RV3IGRUN6OXMXE", + "name": "屏东县", + "province": "台湾" + }, + { + "city_id": "MPB3M2YK24ZORO3EDCJ6UDWIGQ", + "name": "基隆市", + "province": "台湾" + }, + { + "city_id": "EBHVITJPDHJEEMMZTM4TD4UVRU", + "name": "台北市", + "province": "台湾" + }, + { + "city_id": "FQS4PZNCTQEX6I5F34Z2AGJUZM", + "name": "高雄市", + "province": "台湾" + }, + { + "city_id": "WVOJ636Q7MGT6RMN6QYQ4SZWIE", + "name": "嘉义市", + "province": "台湾" + }, + { + "city_id": "QRLER4EEMMKYGQLER2RWEQA74E", + "name": "台南市", + "province": "台湾" + }, + { + "city_id": "K2LHF64R2P4OJ7MDHJ6J2NSTPE", + "name": "桃园市", + "province": "台湾" + }, + { + "city_id": "BILG6LJIWUCTXZ6CDPXYAVM6XI", + "name": "澎湖县", + "province": "台湾" + }, + { + "city_id": "3FYRA3O2HUMLIQAAJQCPX2TETE", + "name": "宜兰县", + "province": "台湾" + }, + { + "city_id": "4MW6X22PAPVMHB6SBGF3RYS324", + "name": "天津市", + "province": "天津市" + }, + { + "city_id": "YEYPP4SQOBXU5UCNDN7ORSR6DI", + "name": "拉萨市", + "province": "西藏自治区" + }, + { + "city_id": "UNE6UPENGWQDOWGABDJEAQ2FEY", + "name": "山南市", + "province": "西藏自治区" + }, + { + "city_id": "VDXKN2YCIUPOHBZKKQHPN6HJWE", + "name": "林芝市", + "province": "西藏自治区" + }, + { + "city_id": "EAWIMNI77H72EYSOAYV3M76CB4", + "name": "阿里地区", + "province": "西藏自治区" + }, + { + "city_id": "HCF6UHTXOOKIOA43AIJH3ARKXA", + "name": "昌都市", + "province": "西藏自治区" + }, + { + "city_id": "Y47QI3KJY352QV3VOPXHM2IDWU", + "name": "日喀则市", + "province": "西藏自治区" + }, + { + "city_id": "R4UWJX44GVAA54NFKHT4Y4ZC5A", + "name": "那曲市", + "province": "西藏自治区" + }, + { + "city_id": "2D37GB5XUALJDXONWJIGXV3QXU", + "name": "香港", + "province": "香港特别行政区" + }, + { + "city_id": "PA5W7Z255K3EGYA4LTA7BGLHRA", + "name": "巴音郭楞蒙古自治州", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "RJZOCU5ECQCOLCOCHJH2UNLQJM", + "name": "哈密市", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "RYK6AR3VDQJFXZLX3MHYFV5VAI", + "name": "塔城地区", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "MXUVGU5NPVTNINAKKPNLWUQ54Q", + "name": "博尔塔拉蒙古自治州", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "T5RKMSH5VIGRTHDZOKV2EIKPIM", + "name": "伊犁哈萨克自治州", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "NABMKFZPOUCZMS4TUVJSZ24DNA", + "name": "克孜勒苏柯尔克孜自治州", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "2YQBXWNFYVJX4NU6WXB5II6X34", + "name": "昌吉回族自治州", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "DBKCQCCQU2URQG2EP5ZNPKBETY", + "name": "乌鲁木齐市", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "DON6KYBCR2XJQOJQGOZTYV4RMM", + "name": "阿克苏地区", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "WKG47NEVYII5JISZJN7QSI2BDQ", + "name": "克拉玛依市", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "DWLK6D3OUOXOVQHSEJVTRIDRAI", + "name": "喀什地区", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "SAPKF2PQGJD4UMVJZTC3IZKI64", + "name": "阿勒泰地区", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "ARWSLGG54LGUGN3XMIWW76NW34", + "name": "吐鲁番市", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "VQCWAKL6ADHYFTSVPBGPDFSB2I", + "name": "北屯市", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "ES5A6MOROAG6F2XAQ25TYYPWUE", + "name": "铁门关市", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "36IUY52AESIPF4QEQAR2RTNQYA", + "name": "和田地区", + "province": "新疆维吾尔自治区" + }, + { + "city_id": "Z26KSUL6ULS65ITZNUCRRWBYJM", + "name": "文山壮族苗族自治州", + "province": "云南省" + }, + { + "city_id": "XBBUUATPBD2IUJR47FCKVUXB2I", + "name": "昭通市", + "province": "云南省" + }, + { + "city_id": "TK2LP3JCYYMPTV4OA3WJCH7UQ4", + "name": "怒江傈僳族自治州", + "province": "云南省" + }, + { + "city_id": "ISJ4FESOYKCQ5LXOQO6NL7TQHA", + "name": "曲靖市", + "province": "云南省" + }, + { + "city_id": "ELBUBVI5UMUIEQGIOAHPMESXFA", + "name": "西双版纳傣族自治州", + "province": "云南省" + }, + { + "city_id": "QHOYHEGZM4WSJZPFEU6FCBYIWY", + "name": "玉溪市", + "province": "云南省" + }, + { + "city_id": "4G4SPJ7MVHMYZAWMQ4642PMLVI", + "name": "保山市", + "province": "云南省" + }, + { + "city_id": "HCHRV2LGJ2TJ4X6BWNWI2IMID4", + "name": "普洱市", + "province": "云南省" + }, + { + "city_id": "IS4Q6NASBWHO3RFO7UCIWOXVI4", + "name": "昆明市", + "province": "云南省" + }, + { + "city_id": "TDJZOAZFQUQPYRR5BHOJTG6RWM", + "name": "红河哈尼族彝族自治州", + "province": "云南省" + }, + { + "city_id": "EIYC62RNU4SHQW3RLAMDPTQYTI", + "name": "大理白族自治州", + "province": "云南省" + }, + { + "city_id": "BA3XFPITAYKBUWDKU3QONRHBC4", + "name": "德宏傣族景颇族自治州", + "province": "云南省" + }, + { + "city_id": "6P6FFFO6C5MLNCVRJAJUICSVQI", + "name": "临沧市", + "province": "云南省" + }, + { + "city_id": "OXHMWH2TSIDI7BQ43EHAMXJ6N4", + "name": "丽江市", + "province": "云南省" + }, + { + "city_id": "QQPDT4LBI2K2KMBNXZ6YH7X2FI", + "name": "楚雄彝族自治州", + "province": "云南省" + }, + { + "city_id": "XBG4EJAWCRJL2TDPNJ23PTFYHQ", + "name": "迪庆藏族自治州", + "province": "云南省" + }, + { + "city_id": "DINNCH54AP74TJ62MICEYAZP74", + "name": "宁波市", + "province": "浙江省" + }, + { + "city_id": "XYTSLYGB2ETU6HG7GIXA7X5SOE", + "name": "嘉兴市", + "province": "浙江省" + }, + { + "city_id": "NNAALJZXGAWALR3LGE2V4UZT6U", + "name": "丽水市", + "province": "浙江省" + }, + { + "city_id": "H5UOJ5MQYJ737GS3TXYN2OJHUU", + "name": "杭州市", + "province": "浙江省" + }, + { + "city_id": "HG5VQGOMSCEGNXJXKO6XCNCHMY", + "name": "湖州市", + "province": "浙江省" + }, + { + "city_id": "LJ2SWEPRINTYDH5A2QHRMI5US4", + "name": "衢州市", + "province": "浙江省" + }, + { + "city_id": "TW4RRM62TDA7WWU77FDSLSAXGY", + "name": "台州市", + "province": "浙江省" + }, + { + "city_id": "HBBN247QZ6YUW5ZYSS6D7RVJCA", + "name": "绍兴市", + "province": "浙江省" + }, + { + "city_id": "GVFB23SJZGRPRXXTIKDCAOCDPI", + "name": "金华市", + "province": "浙江省" + }, + { + "city_id": "UEW4ENX7N7IFGFM7FD5SZ5GI7Y", + "name": "舟山市", + "province": "浙江省" + }, + { + "city_id": "HCCXS5DGRJQMYZMRLGVAMUIQEA", + "name": "温州市", + "province": "浙江省" + }, + { + "city_id": "FDGY55I6IHKY76E3MWDBOT2R6Y", + "name": "重庆市", + "province": "重庆市" + } +] \ No newline at end of file diff --git a/scripts/pull_meituan_coupons.py b/scripts/pull_meituan_coupons.py index aaefdc2..3bc2977 100644 --- a/scripts/pull_meituan_coupons.py +++ b/scripts/pull_meituan_coupons.py @@ -1,45 +1,74 @@ -"""美团 CPS 券定时抓取入库(北京试点)。 +"""美团 CPS 券定时抓取入库(全国 359 个地级市)。 -把 3 路券抓进 meituan_coupon 表,供「销量/佣金排序」从库里捞、本地排序,不再实时打美团: +把每个城市的 3 路券抓进 meituan_coupon 表,供「智能推荐 / 销量最高」从库里捞、本地排序, +不再实时打美团: 1. search_waimai : 到家/外卖, 搜「外卖」 翻到尽头 2. search_meishi : 到家/外卖, 搜「美食」 翻到尽头 3. store_supply : 到店, 多业务线供给(到餐+到综+酒店+门票) 翻到尽头 +城市来自 app/integrations/data/meituan_cities.json(美团官方城市字典,359 地级市)。 +实测一个地级市 cityId 已覆盖其下辖县级市(徐州 → 邳州/新沂…),故按地级市抓即可。 + +并发:城市级并发(每城内部仍顺序跑 3 路),主线程串行入库(Session 不跨线程)。 +实测单城全量 ~110s/~2300 条;15 并发抓完全国一轮 ~50–60min,402 占比 ~3% 且退避全消化。 +mentor 要求每 3h 全量一轮,窗口充裕,故默认 12 并发 + 启动错峰,把瞬时峰值与 402 压更低。 + 按 (source, product_view_sign) upsert 存最新态;last_seen 每轮刷新。带文件锁,防止 -上一轮没跑完下一轮又起(本地 5~10min、跨进程 cron 都安全)。 +上一轮没跑完下一轮又起。 用法: - # 单轮(打通验证 / 给 cron 用,线上每 1h 一次) + # 单轮全量(给 cron 用,线上每 3h 一次) python -m scripts.pull_meituan_coupons --once - # 本地循环(默认每 10min 一轮) - python -m scripts.pull_meituan_coupons --loop --interval 600 + # 常驻循环(每 3h 一轮) + python -m scripts.pull_meituan_coupons --loop --interval 10800 + + # 本地测试:只抓前 N 个城市 / 指定城市 + python -m scripts.pull_meituan_coupons --once --limit 3 + python -m scripts.pull_meituan_coupons --once --city-ids OCZOBCJDEXKE7KBN3BD7AYQG2Q + +部署(服务器):推荐 cron 跑 --once(每 3h),避免长驻进程孤儿: + 0 */3 * * * cd /path/to/app-server && .venv/bin/python -m scripts.pull_meituan_coupons --once >> data/etl.log 2>&1 """ from __future__ import annotations import argparse import hashlib +import logging import os import re import sys import tempfile +import threading import time +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timedelta, timezone -# Windows 控制台按 UTF-8 输出中文/¥ +# Windows 控制台按 UTF-8 输出中文/¥;line_buffering=True 让 print 每行即时 flush—— +# 否则 stdout 重定向到 cron/后台日志文件时是块缓冲,要攒到 ~4KB 或进程退出才落盘, +# 常驻(--loop)时几乎看不到每轮进度。 try: - sys.stdout.reconfigure(encoding="utf-8") # type: ignore[attr-defined] + sys.stdout.reconfigure(encoding="utf-8", line_buffering=True) # type: ignore[attr-defined] except Exception: # noqa: BLE001 pass -from sqlalchemy import delete, func, select -from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy import delete, func, select # noqa: E402 +from sqlalchemy.dialects.postgresql import insert as pg_insert # noqa: E402 -from app.db.session import SessionLocal -from app.integrations.meituan import MeituanCpsError, _call -from app.models.meituan_coupon import MeituanCoupon +from app.db.session import SessionLocal, engine # noqa: E402 +from app.integrations.cities import all_cities # noqa: E402 +from app.integrations.meituan import MeituanCpsError, _call # noqa: E402 +from app.models.meituan_coupon import MeituanCoupon # noqa: E402 + +# dev 默认 create_engine(echo=True) 会逐条打 SQL,本脚本逐城 upsert 会把日志刷爆; +# ETL 不需要 SQL 日志,显式关掉(入库不受影响;线上 prod 本就 echo=False)。 +engine.echo = False + +# 美团调用偶发错误(本机走代理高并发时的 SSL EOF / 上游 code=5 等)会被 meituan._call 的 +# logger.exception 打完整 traceback,并发抓取下单轮可刷数十 KB 日志。ETL 自己用 _STATS 统计 +# 「放弃页数」,无需逐条 traceback,故把美团 logger 压到 CRITICAL(线上直连少见此类错误)。 +logging.getLogger("shagua.meituan").setLevel(logging.CRITICAL) -CITY_BEIJING = "WKV2HMXUEK634WP64CUCUQGM64" QUERY_PATH = "/cps_open/common/api/v1/query_coupon" PAGE_SIZE = 20 MAX_PAGES = 80 # 单路安全上限(搜索 ~52 页、供给 ~70 页) @@ -49,7 +78,12 @@ RETRY = 7 # 无 sudo 部署时 data/ 常属 root,cps 写不了会导致每轮 PermissionError、cron 抓不进数据。 # 需要指定位置时用环境变量 MEITUAN_ETL_LOCK 覆盖。 LOCK_FILE = os.environ.get("MEITUAN_ETL_LOCK") or os.path.join(tempfile.gettempdir(), "meituan_etl.lock") -LOCK_STALE_SEC = 30 * 60 # 锁超过 30min 视为陈旧(进程异常退出残留),自动接管 +# 多城全量一轮 ~50–60min,锁陈旧阈值放宽到 90min,避免把「正在跑的轮次」误判为残留而接管。 +LOCK_STALE_SEC = 90 * 60 + +DEFAULT_CONCURRENCY = 12 # 并发城市数(实测 15 并发 402 占 3% 可退避消化;12 更稳,3h 窗口充裕) +STARTUP_STAGGER = 0.3 # 首批城市启动错峰间隔秒(削平瞬时峰值,实测能压低 402) +PRUNE_FAIL_RATIO_MAX = 0.05 # 失败城占比超此值则本轮跳过 prune(避免大面积抓取失败误删库) SOURCES = [ {"code": "search_waimai", "label": "外卖·搜外卖", "kind": "search", "platform": 1, "keyword": "外卖"}, @@ -58,33 +92,44 @@ SOURCES = [ "platform": 2, "biz_lines": [1, 2, 3, 4]}, ] +# 跨线程统计(并发抓取下汇总请求量 / 402 / 放弃页数,供观测限流) +_STATS_LOCK = threading.Lock() +_STATS = {"req": 0, "r402": 0, "err": 0} + + +def _bump(key: str) -> None: + with _STATS_LOCK: + _STATS[key] += 1 + # ───────────────────────── 美团调用 ───────────────────────── def _call_retry(body: dict) -> dict | None: - """打美团,402/频繁退避重试;其它错误打印并放弃本页。""" + """打美团,402/频繁退避重试;其它错误放弃本页。并发下不逐条 print(避免刷屏),计入 _STATS。""" for a in range(RETRY): + _bump("req") try: return _call(QUERY_PATH, body) except MeituanCpsError as e: msg = str(e) if "402" in msg or "频繁" in msg: + _bump("r402") time.sleep(2.5 * (a + 1)) continue - print(f" [warn] meituan: {msg[:80]}") + _bump("err") return None - except Exception as e: # noqa: BLE001 - print(f" [warn] {type(e).__name__}: {str(e)[:60]}") + except Exception: # noqa: BLE001 time.sleep(2.0 * (a + 1)) + _bump("err") return None -def _pull_search(platform: int, keyword: str) -> list[dict]: +def _pull_search(city_id: str, platform: int, keyword: str) -> list[dict]: rows: list[dict] = [] sid = None pg = 1 while pg <= MAX_PAGES: - body = {"platform": platform, "searchText": keyword, "cityId": CITY_BEIJING, "pageSize": PAGE_SIZE} + body = {"platform": platform, "searchText": keyword, "cityId": city_id, "pageSize": PAGE_SIZE} if sid: body["searchId"] = sid else: @@ -102,14 +147,14 @@ def _pull_search(platform: int, keyword: str) -> list[dict]: return rows -def _pull_supply(platform: int, biz_lines: list[int]) -> list[dict]: +def _pull_supply(city_id: str, platform: int, biz_lines: list[int]) -> list[dict]: rows: list[dict] = [] sid = None biz_param = [{"bizLine": b} for b in biz_lines] for _ in range(MAX_PAGES): body = { "multipleSupplyList": [{"platform": platform, "bizLineParamList": biz_param}], - "cityId": CITY_BEIJING, + "cityId": city_id, "sortField": 2, # 供给查询 sortField 必填;我们入库后本地再排,这里给个默认 "pageSize": PAGE_SIZE, } @@ -157,7 +202,7 @@ def _to_cents(yuan) -> int | None: return None -def _parse_item(item: dict, source: dict) -> dict | None: +def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: cpd = item.get("couponPackDetail") or {} br = item.get("brandInfo") or {} ci = item.get("commissionInfo") or {} @@ -190,7 +235,7 @@ def _parse_item(item: dict, source: dict) -> dict | None: "source": source["code"], "platform": source["platform"], "biz_line": item.get("bizLine") or cpd.get("bizLine"), - "city_id": CITY_BEIJING, + "city_id": city_id, "product_view_sign": str(sign)[:128], "sku_view_id": cpd.get("skuViewId"), "name": (name[:256] or None), @@ -210,6 +255,27 @@ def _parse_item(item: dict, source: dict) -> dict | None: } +def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> list[dict]: + """抓单个城市的 3 路券并解析。worker 线程内执行(只抓取+解析,不碰 DB)。 + + 启动错峰:首批并发的 `concurrency` 个城市按 index 错开首请求,削平瞬时峰值(压低 402)。 + """ + if stagger: + time.sleep((index % concurrency) * stagger) + cid = city["city_id"] + parsed: list[dict] = [] + for src in SOURCES: + if src["kind"] == "search": + items = _pull_search(cid, src["platform"], src["keyword"]) + else: + items = _pull_supply(cid, src["platform"], src["biz_lines"]) + for it in items: + p = _parse_item(it, src, cid) + if p: + parsed.append(p) + return parsed + + # ───────────────────────── 入库(upsert) ───────────────────────── def _upsert(db, rows: list[dict], now: datetime) -> tuple[int, int]: @@ -272,30 +338,53 @@ def _release_lock() -> None: # ───────────────────────── 主流程 ───────────────────────── -def run_once(prune_hours: int = 24) -> None: +def run_once( + prune_hours: int = 24, + concurrency: int = DEFAULT_CONCURRENCY, + stagger: float = STARTUP_STAGGER, + cities: list[dict] | None = None, +) -> None: if not _acquire_lock(): print(f"[{datetime.now():%H:%M:%S}] 上一轮还在跑(锁占用),跳过本轮") return t0 = time.time() now = datetime.now(timezone.utc) + with _STATS_LOCK: + _STATS.update(req=0, r402=0, err=0) + cities = cities if cities is not None else all_cities() + n_city = len(cities) db = SessionLocal() try: - total = 0 - for src in SOURCES: - ts = time.time() - if src["kind"] == "search": - items = _pull_search(src["platform"], src["keyword"]) - else: - items = _pull_supply(src["platform"], src["biz_lines"]) - parsed = [p for p in (_parse_item(it, src) for it in items) if p] - up, dup = _upsert(db, parsed, now) - total += up - print(f" {src['label']:18} 抓{len(items):5} 解析{len(parsed):5} " - f"入库{up:5} (本源去重{dup:3}) {time.time() - ts:4.0f}s") - # 清理长期未再出现的陈旧券(美团 sign 轮换 / 券下架后的残留),默认 24h 宽限。 - # 护栏:仅在本轮确有入库(total>0)时才清理。美团整体故障时本轮可能 0 入库 - # (脚本不抛异常、只是抓回空),若仍照常 prune,连续故障会按 last_seen 把全表删空。 - if prune_hours and prune_hours > 0 and total > 0: + total_up = 0 + done = 0 + fails: list[str] = [] + # 城市级并发抓取(worker 只抓取+解析),主线程逐城串行入库(Session 不跨线程)。 + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futs = { + pool.submit(_pull_one_city, city, i, concurrency, stagger): city + for i, city in enumerate(cities) + } + for fut in as_completed(futs): + city = futs[fut] + try: + parsed = fut.result() + except Exception as e: # noqa: BLE001 + fails.append(city["name"]) + print(f" [fail] {city['name']}: {type(e).__name__}: {str(e)[:60]}") + continue + up, _dup = _upsert(db, parsed, now) + total_up += up + done += 1 + if done % 20 == 0 or done == n_city: + print(f" [{done}/{n_city}] {city['name']:10} 抓{len(parsed):5} 入库{up:5} " + f"(累计入库 {total_up}, 用时 {time.time() - t0:.0f}s)") + + # 清理陈旧券(美团 sign 轮换 / 券下架残留),默认 24h 宽限。两道护栏防误删: + # ① total_up>0:本轮 0 入库(疑似上游整体故障,脚本不抛异常只抓回空)时跳过,否则 + # 会按 last_seen 把全表删空; + # ② 失败城占比 ≤5%:大面积城市抓取失败(限流/网络)时跳过,避免误删还在架上的券。 + fail_ratio = len(fails) / max(1, n_city) + if prune_hours and prune_hours > 0 and total_up > 0 and fail_ratio <= PRUNE_FAIL_RATIO_MAX: cutoff = now - timedelta(hours=prune_hours) pruned = db.execute( delete(MeituanCoupon).where(MeituanCoupon.last_seen < cutoff) @@ -303,36 +392,62 @@ def run_once(prune_hours: int = 24) -> None: db.commit() if pruned: print(f" 清理陈旧券(>{prune_hours}h 未再出现): {pruned} 条") - elif prune_hours and prune_hours > 0 and total == 0: - print(" 本轮 0 入库(疑似上游故障),跳过清理以防误删全表") + elif prune_hours and prune_hours > 0: + reason = "本轮 0 入库" if total_up == 0 else f"失败城占比 {fail_ratio:.0%}>{PRUNE_FAIL_RATIO_MAX:.0%}" + print(f" 跳过 prune({reason},防误删)") + cnt = db.execute(select(func.count()).select_from(MeituanCoupon)).scalar() - print(f"[{datetime.now():%H:%M:%S}] 本轮完成: 入库 {total} 条, 表总计 {cnt} 行, " - f"用时 {time.time() - t0:.0f}s") + with _STATS_LOCK: + req, r402, err = _STATS["req"], _STATS["r402"], _STATS["err"] + fail_tail = (": " + ",".join(fails[:10])) if fails else "" + print(f"[{datetime.now():%H:%M:%S}] 本轮完成: {done}/{n_city} 城, 入库 {total_up} 条, " + f"表总计 {cnt} 行, 请求 {req}(402 {r402} / 放弃 {err}), " + f"失败城 {len(fails)}{fail_tail}, 用时 {time.time() - t0:.0f}s") finally: db.close() _release_lock() +def _select_cities(limit: int, city_ids: str) -> list[dict]: + cities = all_cities() + if city_ids.strip(): + want = {c.strip() for c in city_ids.split(",") if c.strip()} + return [c for c in cities if c["city_id"] in want] + if limit and limit > 0: + return cities[:limit] + return cities + + def main() -> None: - ap = argparse.ArgumentParser(description="美团 CPS 券定时抓取入库") + ap = argparse.ArgumentParser(description="美团 CPS 券定时抓取入库(全国 359 地级市)") ap.add_argument("--once", action="store_true", help="只跑一轮(默认)") ap.add_argument("--loop", action="store_true", help="循环跑") - ap.add_argument("--interval", type=int, default=600, help="循环间隔秒(默认 600=10min)") + ap.add_argument("--interval", type=int, default=10800, + help="循环间隔秒(默认 10800=3h,对齐每 3h 全量一轮)") ap.add_argument("--prune-hours", type=int, default=24, help="清理超过 N 小时未再出现的陈旧券(默认 24;0=不清理)") + ap.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY, + help=f"并发城市数(默认 {DEFAULT_CONCURRENCY};实测 15 并发 402 占 3%% 可退避消化)") + ap.add_argument("--stagger", type=float, default=STARTUP_STAGGER, + help=f"首批城市启动错峰间隔秒(默认 {STARTUP_STAGGER};削平瞬时峰值压低 402)") + ap.add_argument("--limit", type=int, default=0, help="只抓前 N 个城市(测试用,0=全部)") + ap.add_argument("--city-ids", default="", help="只抓这些 cityId(逗号分隔,测试用,优先于 --limit)") args = ap.parse_args() + cities = _select_cities(args.limit, args.city_ids) + print(f"城市数: {len(cities)} / 并发: {args.concurrency} / 错峰: {args.stagger}s / 间隔: {args.interval}s") + if args.loop: print(f"循环模式: 每 {args.interval}s 一轮 (Ctrl-C 退出)") while True: try: - run_once(args.prune_hours) + run_once(args.prune_hours, args.concurrency, args.stagger, cities) except Exception as e: # noqa: BLE001 print(f"[{datetime.now():%H:%M:%S}] 本轮异常: {type(e).__name__}: {e}") _release_lock() time.sleep(args.interval) else: - run_once(args.prune_hours) + run_once(args.prune_hours, args.concurrency, args.stagger, cities) if __name__ == "__main__": diff --git a/tools/gen_meituan_cities.py b/tools/gen_meituan_cities.py new file mode 100644 index 0000000..231fb7e --- /dev/null +++ b/tools/gen_meituan_cities.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +"""从美团「城市字典」Excel 生成随仓库的 JSON(app/integrations/data/meituan_cities.json)。 + +本地一次性工具:美团每年更新城市字典时,把新 Excel 放进来重跑即可。 +Excel 不入库、不进仓库(二进制、需 openpyxl);生成的 JSON 随仓库提交,部署到服务器后 +ETL(scripts/pull_meituan_coupons.py)直接读它遍历全部城市。 + +字典口径:359 个地级市,经实测一个地级市 cityId 已覆盖其下辖县级市(如徐州→邳州/新沂), +故无需区县层级。 + +用法: + python tools/gen_meituan_cities.py ["城市字典xxx.xlsx" 路径] + (不传则用默认 e:\\codes\\城市字典2025 (1).xlsx) +""" +from __future__ import annotations + +import json +import os +import sys + +import openpyxl + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +DEFAULT_EXCEL = r"e:\codes\城市字典2025 (1).xlsx" +OUT = os.path.join(ROOT, "app", "integrations", "data", "meituan_cities.json") + + +def main() -> None: + excel = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_EXCEL + wb = openpyxl.load_workbook(excel, data_only=True) + ws = wb.worksheets[0] + rows = list(ws.iter_rows(values_only=True)) + header = rows[0] + + cities = [] + seen = set() + for r in rows[1:]: + cid = str(r[0]).strip() if r[0] else "" + name = str(r[1]).strip() if len(r) > 1 and r[1] else "" + prov = str(r[2]).strip() if len(r) > 2 and r[2] else "" + if not cid or cid in seen: + continue + seen.add(cid) + cities.append({"city_id": cid, "name": name, "province": prov}) + + os.makedirs(os.path.dirname(OUT), exist_ok=True) + with open(OUT, "w", encoding="utf-8") as f: + json.dump(cities, f, ensure_ascii=False, indent=1) + + print(f"表头: {header}") + print(f"写出 {len(cities)} 城 → {OUT}") + print("样例:", cities[:3]) + + +if __name__ == "__main__": + main() From f853938095bf6263d480ab21b1c0017f7437a76c Mon Sep 17 00:00:00 2001 From: chenshuobo Date: Tue, 16 Jun 2026 21:19:47 +0800 Subject: [PATCH 5/7] =?UTF-8?q?fix(meituan-etl):=20=E6=B8=85=E6=B4=97?= =?UTF-8?q?=E6=96=87=E6=9C=AC=20NUL=20=E5=AD=97=E8=8A=82=20+=20=E9=80=90?= =?UTF-8?q?=E5=9F=8E=E5=85=A5=E5=BA=93=E5=AE=B9=E9=94=99(=E4=BF=AE?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E9=A6=96=E7=81=8C=E5=B4=A9=E5=9C=A8=E8=84=8F?= =?UTF-8?q?=E6=95=B0=E6=8D=AE)=20(#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 全国 359 城全量首灌崩在厦门:美团某券文本字段含 NUL(0x00),PostgreSQL text/jsonb 拒绝该字节,整批 upsert 抛 DataError → --once 进程崩、后续 300+ 城 全不跑(本地 20 城没撞上、跑全国才暴露)。两处修复: - _strip_nul 递归清洗入库 dict 所有字符串(含 raw JSON)的 NUL; - 逐城 _upsert 包 try/except + rollback,单城入库失败记 fails 跳过,不再让 一条脏数据 / 一次抖动拖垮整轮 359 城。 --------- Co-authored-by: chenshuobo <1119780489@qq.com> Reviewed-on: https://gitea.shaguabijia.com/WonderableAI/shaguabijia-app-server/pulls/58 Co-authored-by: chenshuobo Co-committed-by: chenshuobo --- scripts/pull_meituan_coupons.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/pull_meituan_coupons.py b/scripts/pull_meituan_coupons.py index 3bc2977..3431cc8 100644 --- a/scripts/pull_meituan_coupons.py +++ b/scripts/pull_meituan_coupons.py @@ -202,6 +202,18 @@ def _to_cents(yuan) -> int | None: return None +def _strip_nul(v): + """递归去掉字符串里的 NUL(0x00):PostgreSQL 的 text / jsonb 字段都不接受该字节, + 美团偶有脏数据(某券文本含 NUL)会让整批 upsert 抛 DataError。入库前统一清洗。""" + if isinstance(v, str): + return v.replace(chr(0), "") + if isinstance(v, dict): + return {k: _strip_nul(x) for k, x in v.items()} + if isinstance(v, list): + return [_strip_nul(x) for x in v] + return v + + def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: cpd = item.get("couponPackDetail") or {} br = item.get("brandInfo") or {} @@ -231,7 +243,8 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: dist = None dedup_raw = f"{brand}|{name}|{price_cents}" - return { + # 入库前递归清洗 NUL(美团脏数据偶含 0x00,PostgreSQL text/jsonb 拒绝整批 → DataError) + return _strip_nul({ "source": source["code"], "platform": source["platform"], "biz_line": item.get("bizLine") or cpd.get("bizLine"), @@ -252,7 +265,7 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: "delivery_distance_m": dist, "dedup_key": hashlib.md5(dedup_raw.encode("utf-8")).hexdigest(), "raw": item, - } + }) def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> list[dict]: @@ -372,7 +385,13 @@ def run_once( fails.append(city["name"]) print(f" [fail] {city['name']}: {type(e).__name__}: {str(e)[:60]}") continue - up, _dup = _upsert(db, parsed, now) + try: + up, _dup = _upsert(db, parsed, now) + except Exception as e: # noqa: BLE001 + db.rollback() # 事务已 abort,必须 rollback 才能继续给下一城复用 Session + fails.append(city["name"]) + print(f" [入库失败] {city['name']}: {type(e).__name__}: {str(e)[:80]}") + continue total_up += up done += 1 if done % 20 == 0 or done == n_city: From 9d93b70b9ba23d3823e2eb0335442e5e642f0bb1 Mon Sep 17 00:00:00 2001 From: marco Date: Wed, 17 Jun 2026 10:00:29 +0800 Subject: [PATCH 6/7] =?UTF-8?q?feat(user):=20username=20=E5=AF=B9=E5=A4=96?= =?UTF-8?q?=E5=B1=95=E7=A4=BA=20+=20=E9=BB=98=E8=AE=A4=E6=98=B5=E7=A7=B0?= =?UTF-8?q?=20+=20=E5=AD=98=E9=87=8F=E5=9B=9E=E5=A1=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 代提工作区既有的他人在制品(非本次 CPS);CPS 迁移链 cps_tables 依赖其 b3f1a2c4d5e6 迁移,需一并提交。 Co-Authored-By: Claude Opus 4.8 --- ...d5e6_user_username_and_default_nickname.py | 79 +++++++++++++++++++ app/models/user.py | 4 + app/repositories/user.py | 48 +++++++++++ app/schemas/auth.py | 2 + tests/test_auth.py | 37 +++++++++ tests/test_invite.py | 18 +++-- 6 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 alembic/versions/b3f1a2c4d5e6_user_username_and_default_nickname.py diff --git a/alembic/versions/b3f1a2c4d5e6_user_username_and_default_nickname.py b/alembic/versions/b3f1a2c4d5e6_user_username_and_default_nickname.py new file mode 100644 index 0000000..ba76a60 --- /dev/null +++ b/alembic/versions/b3f1a2c4d5e6_user_username_and_default_nickname.py @@ -0,0 +1,79 @@ +"""user.username(对外展示账号 ID)+ 默认昵称,并回填存量用户 + +Revision ID: b3f1a2c4d5e6 +Revises: 45047b5a884c +Create Date: 2026-06-16 12:00:00.000000 + +加 user.username(11 位纯数字、首位非 1 与手机号天然区分、全局唯一)。存量用户: +生成唯一 username,昵称为空的补 9 位字母数字随机昵称。 +迁移自包含(不 import app 业务代码,逻辑冻结为当时快照)。 +""" +import secrets +import string +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b3f1a2c4d5e6' +down_revision: Union[str, Sequence[str], None] = '45047b5a884c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_USERNAME_FIRST = "23456789" # 首位避前导 0、避 1(手机号都以 1 开头) +_USERNAME_DIGITS = "0123456789" +_NICKNAME_ALPHABET = string.ascii_letters + string.digits + + +def _gen_username() -> str: + return secrets.choice(_USERNAME_FIRST) + "".join( + secrets.choice(_USERNAME_DIGITS) for _ in range(10) + ) + + +def _gen_nickname() -> str: + return "".join(secrets.choice(_NICKNAME_ALPHABET) for _ in range(9)) + + +def upgrade() -> None: + # 1. 先加可空列(存量行此刻还没值) + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.add_column(sa.Column('username', sa.String(length=11), nullable=True)) + + # 2. 回填存量:每人一个唯一 username;昵称为空的补随机昵称。 + # 表里 username 原本全空,本批用 used 防互撞即可(无既有非空值需规避)。 + # "user" 是 PostgreSQL 保留字,必须加双引号(SQLite 也接受双引号标识符)。 + bind = op.get_bind() + rows = bind.execute(sa.text('SELECT id, nickname FROM "user"')).fetchall() + used = set() + for row in rows: + uid, nick = row[0], row[1] + while True: + uname = _gen_username() + if uname not in used: + used.add(uname) + break + if nick is None or not str(nick).strip(): + bind.execute( + sa.text('UPDATE "user" SET username = :u, nickname = :n WHERE id = :i'), + {"u": uname, "n": _gen_nickname(), "i": uid}, + ) + else: + bind.execute( + sa.text('UPDATE "user" SET username = :u WHERE id = :i'), + {"u": uname, "i": uid}, + ) + + # 3. 收紧:NOT NULL + 唯一索引(对齐模型 unique=True, index=True) + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.alter_column('username', existing_type=sa.String(length=11), nullable=False) + batch_op.create_index('ix_user_username', ['username'], unique=True) + + +def downgrade() -> None: + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.drop_index('ix_user_username') + batch_op.drop_column('username') diff --git a/app/models/user.py b/app/models/user.py index 67d6600..1161ba1 100644 --- a/app/models/user.py +++ b/app/models/user.py @@ -27,6 +27,10 @@ class User(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) phone: Mapped[str] = mapped_column(String(20), unique=True, index=True, nullable=False) + # 对外展示的账号 ID:11 位纯数字、首位非 1(与手机号天然区分——手机号都以 1 开头)、全局唯一、 + # 创建时随机生成、不可变、不参与登录(登录仍走 phone)。生成见 repositories/user._gen_username。 + username: Mapped[str] = mapped_column(String(11), unique=True, index=True, nullable=False) + # 注册渠道:jverify / sms。后续加 wechat / apple 时扩 register_channel: Mapped[str] = mapped_column(String(20), nullable=False, default="jverify") diff --git a/app/repositories/user.py b/app/repositories/user.py index 697d7e8..33c53ae 100644 --- a/app/repositories/user.py +++ b/app/repositories/user.py @@ -4,6 +4,8 @@ """ from __future__ import annotations +import secrets +import string from datetime import datetime, timezone from sqlalchemy import select @@ -12,6 +14,50 @@ from sqlalchemy.orm import Session from app.models.user import User +# ===== 创建时分配的标识:用户名(对外展示账号 ID)+ 默认昵称 ===== + +# 用户名 = 11 位纯数字,首位 2-9(避前导 0、避 1 与手机号区分),后 10 位 0-9。 +# 空间 8×10^10,创建时随机生成 + 查重去重,全局唯一、不可变、不参与登录。 +_USERNAME_FIRST = "23456789" +_USERNAME_DIGITS = "0123456789" +_USERNAME_LEN = 11 + +# 默认昵称 = 9 位大小写字母 + 数字随机串(创建时给,用户可后续改;不要求唯一)。 +_NICKNAME_ALPHABET = string.ascii_letters + string.digits +_NICKNAME_LEN = 9 + + +def _gen_username() -> str: + """随机一个 11 位纯数字、首位非 0/1 的用户名(不保证唯一,唯一性由 _gen_unique_username 兜)。""" + return secrets.choice(_USERNAME_FIRST) + "".join( + secrets.choice(_USERNAME_DIGITS) for _ in range(_USERNAME_LEN - 1) + ) + + +def _gen_nickname() -> str: + """随机一个 9 位字母+数字的默认昵称。""" + return "".join(secrets.choice(_NICKNAME_ALPHABET) for _ in range(_NICKNAME_LEN)) + + +def get_user_by_username(db: Session, username: str) -> User | None: + return db.execute( + select(User).where(User.username == username) + ).scalar_one_or_none() + + +def _gen_unique_username(db: Session) -> str: + """生成一个库里尚不存在的用户名(仿 invite.ensure_code 的碰撞重试)。 + + 8×10^10 空间下碰撞极罕见,查重 + 重试足够;并发下的残留碰撞由 username 唯一约束 + 兜底(commit 抛 IntegrityError,与现有 phone 并发同号同级别,概率 ~ 用户数/8e10)。 + """ + for _ in range(8): + uname = _gen_username() + if get_user_by_username(db, uname) is None: + return uname + raise RuntimeError("生成用户名连续碰撞,请重试") + + def get_user_by_id(db: Session, user_id: int) -> User | None: return db.get(User, user_id) @@ -36,6 +82,8 @@ def upsert_user_for_login( if user is None: user = User( phone=phone, + username=_gen_unique_username(db), + nickname=_gen_nickname(), register_channel=register_channel, last_login_at=now, ) diff --git a/app/schemas/auth.py b/app/schemas/auth.py index a5eb077..600baaf 100644 --- a/app/schemas/auth.py +++ b/app/schemas/auth.py @@ -18,6 +18,8 @@ class UserOut(BaseModel): model_config = ConfigDict(from_attributes=True) # 允许直接 UserOut.model_validate(orm_user) id: int + # 对外展示的账号 ID:11 位纯数字、首位非 1、全局唯一、创建时分配、不可变。区别于登录用的 phone。 + username: str phone: str nickname: str | None = None avatar_url: str | None = None diff --git a/tests/test_auth.py b/tests/test_auth.py index 3aca97e..f9a6be1 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -206,3 +206,40 @@ def test_sms_gc_purges_stale_only(monkeypatch) -> None: assert "stale" not in sms._codes and "fresh" in sms._codes assert "old" not in sms._last_sent and "recent" in sms._last_sent assert "yesterday" not in sms._daily_count and "today" in sms._daily_count + + +# ============================ 用户名 / 默认昵称 ============================ + +def test_login_assigns_username_and_nickname(client) -> None: + """新用户创建即分配:11 位纯数字 username(首位非 0/1,与手机号天然区分)+ 9 位字母数字默认昵称。""" + phone = "13600136000" + client.post("/api/v1/auth/sms/send", json={"phone": phone}) + r = client.post("/api/v1/auth/sms/login", json={"phone": phone, "code": "123456"}) + assert r.status_code == 200, r.text + user = r.json()["user"] + + uname = user["username"] + assert uname.isdigit() and len(uname) == 11 # 11 位纯数字 + assert uname[0] not in ("0", "1") # 无前导 0、与手机号(均以 1 开头)区分 + + nick = user["nickname"] + assert nick and len(nick) == 9 and nick.isalnum() # 9 位字母+数字 + + +def test_username_stable_and_unique_on_relogin() -> None: + """同号重登 username 不变(不可变标识);不同号 username 不同(唯一)。 + 直连 repository 绕开 HTTP 短信冷却。""" + from app.db.session import SessionLocal + from app.repositories import user as user_repo + + db = SessionLocal() + try: + a1 = user_repo.upsert_user_for_login(db, phone="13522220001", register_channel="sms") + uname_a, id_a = a1.username, a1.id + a2 = user_repo.upsert_user_for_login(db, phone="13522220001", register_channel="sms") + assert a2.id == id_a and a2.username == uname_a # 重登:同一行、username 稳定不变 + + b = user_repo.upsert_user_for_login(db, phone="13522220002", register_channel="sms") + assert b.username != uname_a # 不同用户、username 唯一 + finally: + db.close() diff --git a/tests/test_invite.py b/tests/test_invite.py index 574adad..a578c6a 100644 --- a/tests/test_invite.py +++ b/tests/test_invite.py @@ -300,7 +300,7 @@ def test_bind_no_code_no_fingerprint(client) -> None: # ===================================================================== def test_invitees_basic(client) -> None: - """A 邀 B、C → 列表返 2 条、total=2、没设资料的名字=脱敏手机号、头像 null、金币对。""" + """A 邀 B、C → 列表返 2 条、total=2、名字=被邀请人默认昵称(创建即有)、头像 null、金币对。""" a = _login(client, "13800002050") a_code = _my_code(client, a) for phone in ("13800002051", "13800002052"): @@ -313,9 +313,14 @@ def test_invitees_basic(client) -> None: assert body["total"] == 2 assert body["has_more"] is False assert len(body["items"]) == 2 - # 没设昵称头像 → 名字脱敏手机号、头像 null(不依赖顺序用 set) + # 创建即分配默认昵称 → 名字=被邀请人昵称(取实际值对比)、头像 null(不依赖顺序用 set) + with SessionLocal() as db: + expected_names = { + get_user_by_phone(db, "13800002051").nickname, + get_user_by_phone(db, "13800002052").nickname, + } names = {it["display_name"] for it in body["items"]} - assert names == {"138****2051", "138****2052"} + assert names == expected_names assert all(it["avatar_url"] is None for it in body["items"]) assert all(it["coins"] == INVITE_INVITER_COINS for it in body["items"]) @@ -336,9 +341,12 @@ def test_invitees_order_desc(client) -> None: ).scalar_one() rel_b.created_at = datetime.now(timezone.utc) - timedelta(hours=2) db.commit() + nick_b = ub.nickname + nick_c = get_user_by_phone(db, "13800002062").nickname items = client.get("/api/v1/invite/invitees", headers=_auth(a)).json()["items"] - assert items[0]["display_name"] == "138****2062" # C 刚邀,在前 - assert items[1]["display_name"] == "138****2061" # B 2 小时前,在后 + # 创建即有默认昵称 → display_name=被邀请人昵称;本用例核心验证倒序(C 刚邀在前、B 2h 前在后) + assert items[0]["display_name"] == nick_c + assert items[1]["display_name"] == nick_b def test_invitees_nickname_priority(client) -> None: From 277f9b16a2298c92092250972ca35664eb8453ad Mon Sep 17 00:00:00 2001 From: marco Date: Wed, 17 Jun 2026 10:00:29 +0800 Subject: [PATCH 7/7] =?UTF-8?q?feat(cps):=20=E7=BE=A4=E5=8F=91=E5=88=B8?= =?UTF-8?q?=E5=88=86=E5=8F=91=E4=B8=8E=E5=AF=B9=E8=B4=A6=20+=20=E7=9F=AD?= =?UTF-8?q?=E9=93=BE=E7=82=B9=E5=87=BB=E8=BF=BD=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 群(sid)/活动池/转链(带sid)/美团 query_order 按 sid 对账/按群统计; 短链 /c/{code} 记点击(PV/UV)→302 跳美团,点击→下单漏斗。 Co-Authored-By: Claude Opus 4.8 --- alembic/versions/cps_link_tables.py | 61 ++++++ alembic/versions/cps_tables.py | 92 +++++++++ app/admin/main.py | 2 + app/admin/repositories/cps.py | 287 ++++++++++++++++++++++++++++ app/admin/routers/cps.py | 268 ++++++++++++++++++++++++++ app/admin/schemas/cps.py | 122 ++++++++++++ app/api/v1/cps_redirect.py | 39 ++++ app/core/config.py | 7 + app/integrations/meituan.py | 41 +++- app/main.py | 3 + app/models/__init__.py | 4 + app/models/cps_activity.py | 34 ++++ app/models/cps_group.py | 32 ++++ app/models/cps_link.py | 52 +++++ app/models/cps_order.py | 63 ++++++ app/repositories/cps_link.py | 82 ++++++++ 16 files changed, 1187 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/cps_link_tables.py create mode 100644 alembic/versions/cps_tables.py create mode 100644 app/admin/repositories/cps.py create mode 100644 app/admin/routers/cps.py create mode 100644 app/admin/schemas/cps.py create mode 100644 app/api/v1/cps_redirect.py create mode 100644 app/models/cps_activity.py create mode 100644 app/models/cps_group.py create mode 100644 app/models/cps_link.py create mode 100644 app/models/cps_order.py create mode 100644 app/repositories/cps_link.py diff --git a/alembic/versions/cps_link_tables.py b/alembic/versions/cps_link_tables.py new file mode 100644 index 0000000..7c42d4f --- /dev/null +++ b/alembic/versions/cps_link_tables.py @@ -0,0 +1,61 @@ +"""cps_link / cps_click tables (群发短链 + 点击统计) + +Revision ID: cps_link_tables +Revises: cps_tables +Create Date: 2026-06-16 13:20:00.000000 + +群发券链接套一层 /c/{code} 做点击统计:cps_link(短码→美团目标) + cps_click(点击事件)。 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "cps_link_tables" +down_revision: Union[str, Sequence[str], None] = "cps_tables" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_NOW = sa.text("CURRENT_TIMESTAMP") + + +def upgrade() -> None: + op.create_table( + "cps_link", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("code", sa.String(length=16), nullable=False), + sa.Column("group_id", sa.Integer(), nullable=False), + sa.Column("activity_id", sa.Integer(), nullable=False), + sa.Column("sid", sa.String(length=64), nullable=False), + sa.Column("platform", sa.String(length=20), nullable=False, server_default="meituan"), + sa.Column("target_url", sa.String(length=1024), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=_NOW, nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("code", name="uq_cps_link_code"), + ) + op.create_index("ix_cps_link_code", "cps_link", ["code"]) + op.create_index("ix_cps_link_group_id", "cps_link", ["group_id"]) + op.create_index("ix_cps_link_activity_id", "cps_link", ["activity_id"]) + op.create_index("ix_cps_link_sid", "cps_link", ["sid"]) + + op.create_table( + "cps_click", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("link_id", sa.Integer(), nullable=False), + sa.Column("group_id", sa.Integer(), nullable=False), + sa.Column("sid", sa.String(length=64), nullable=False), + sa.Column("ip", sa.String(length=64), nullable=True), + sa.Column("ua", sa.String(length=512), nullable=True), + sa.Column("clicked_at", sa.DateTime(timezone=True), server_default=_NOW, nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_cps_click_link_id", "cps_click", ["link_id"]) + op.create_index("ix_cps_click_group_id", "cps_click", ["group_id"]) + op.create_index("ix_cps_click_sid", "cps_click", ["sid"]) + op.create_index("ix_cps_click_clicked_at", "cps_click", ["clicked_at"]) + + +def downgrade() -> None: + op.drop_table("cps_click") + op.drop_table("cps_link") diff --git a/alembic/versions/cps_tables.py b/alembic/versions/cps_tables.py new file mode 100644 index 0000000..8180e35 --- /dev/null +++ b/alembic/versions/cps_tables.py @@ -0,0 +1,92 @@ +"""cps_group / cps_activity / cps_order tables (CPS 分发与对账) + +Revision ID: cps_tables +Revises: b3f1a2c4d5e6 +Create Date: 2026-06-16 12:40:00.000000 + +群发 CPS 优惠券的分发与对账:cps_group(群=sid) + cps_activity(可推广活动池) + +cps_order(美团 query_order 按 sid 拉回的订单明细)。金额统一存「分」。 +order_id 全局唯一(reconcile 按它 upsert,订单状态会变则更新)。 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "cps_tables" +down_revision: Union[str, Sequence[str], None] = "b3f1a2c4d5e6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_NOW = sa.text("CURRENT_TIMESTAMP") + + +def upgrade() -> None: + op.create_table( + "cps_group", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("sid", sa.String(length=64), nullable=False), + sa.Column("name", sa.String(length=128), nullable=False), + sa.Column("member_count", sa.Integer(), nullable=True), + sa.Column("status", sa.String(length=20), nullable=False, server_default="active"), + sa.Column("remark", sa.String(length=256), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=_NOW, nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("sid", name="uq_cps_group_sid"), + ) + op.create_index("ix_cps_group_sid", "cps_group", ["sid"]) + + op.create_table( + "cps_activity", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("platform", sa.String(length=20), nullable=False, server_default="meituan"), + sa.Column("name", sa.String(length=128), nullable=False), + sa.Column("act_id", sa.String(length=64), nullable=True), + sa.Column("product_view_sign", sa.String(length=128), nullable=True), + sa.Column("status", sa.String(length=20), nullable=False, server_default="active"), + sa.Column("remark", sa.String(length=256), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=_NOW, nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + + op.create_table( + "cps_order", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("order_id", sa.String(length=64), nullable=False), + sa.Column("sid", sa.String(length=64), nullable=True), + sa.Column("act_id", sa.String(length=64), nullable=True), + sa.Column("biz_line", sa.Integer(), nullable=True), + sa.Column("trade_type", sa.Integer(), nullable=True), + sa.Column("pay_price_cents", sa.Integer(), nullable=True), + sa.Column("commission_cents", sa.Integer(), nullable=True), + sa.Column("commission_rate", sa.String(length=16), nullable=True), + sa.Column("refund_price_cents", sa.Integer(), nullable=True), + sa.Column("refund_profit_cents", sa.Integer(), nullable=True), + sa.Column("mt_status", sa.String(length=8), nullable=True), + sa.Column("invalid_reason", sa.String(length=128), nullable=True), + sa.Column("product_name", sa.String(length=512), nullable=True), + sa.Column("pay_time", sa.DateTime(timezone=True), nullable=True), + sa.Column("mt_update_time", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "raw", + sa.JSON().with_variant(postgresql.JSONB(), "postgresql"), + nullable=False, + ), + sa.Column("first_seen", sa.DateTime(timezone=True), server_default=_NOW, nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=_NOW, nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("order_id", name="uq_cps_order_order_id"), + ) + op.create_index("ix_cps_order_order_id", "cps_order", ["order_id"]) + op.create_index("ix_cps_order_sid", "cps_order", ["sid"]) + op.create_index("ix_cps_order_act_id", "cps_order", ["act_id"]) + op.create_index("ix_cps_order_mt_status", "cps_order", ["mt_status"]) + op.create_index("ix_cps_order_pay_time", "cps_order", ["pay_time"]) + + +def downgrade() -> None: + op.drop_table("cps_order") + op.drop_table("cps_activity") + op.drop_table("cps_group") diff --git a/app/admin/main.py b/app/admin/main.py index c65aa9e..a67b403 100644 --- a/app/admin/main.py +++ b/app/admin/main.py @@ -19,6 +19,7 @@ from app.admin.routers.admins import router as admins_router from app.admin.routers.audit import router as audit_router from app.admin.routers.auth import router as auth_router from app.admin.routers.config import router as config_router +from app.admin.routers.cps import router as cps_router from app.admin.routers.dashboard import router as dashboard_router from app.admin.routers.ops_stat_config import router as ops_stat_config_router from app.admin.routers.feedback import router as feedback_router @@ -90,5 +91,6 @@ admin_app.include_router(feedback_router) admin_app.include_router(admins_router) admin_app.include_router(audit_router) admin_app.include_router(config_router) +admin_app.include_router(cps_router) admin_app.include_router(ad_audit_router) admin_app.include_router(ad_revenue_router) diff --git a/app/admin/repositories/cps.py b/app/admin/repositories/cps.py new file mode 100644 index 0000000..17516e2 --- /dev/null +++ b/app/admin/repositories/cps.py @@ -0,0 +1,287 @@ +"""admin CPS 数据访问:群/活动 CRUD + 美团订单拉单入库(对账) + 按群统计聚合。 + +美团 query_order 返回的金额是「元」字符串、时间是秒级时间戳,这里统一转「分」+ tz-aware。 +统计在 Python 侧聚合(订单量级小、admin 低频),逻辑清晰、跨 PG/SQLite 无 SQL 方言坑。 +""" +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +from uuid import uuid4 + +from sqlalchemy import desc, select +from sqlalchemy.orm import Session + +from app.admin.repositories.queries import _as_utc, offset_paginate +from app.integrations import meituan +from app.repositories import cps_link as cps_link_repo +from app.models.cps_activity import CpsActivity +from app.models.cps_group import CpsGroup +from app.models.cps_order import CpsOrder + +# 美团订单状态:取消(4)/风控(5)不计佣金;结算(6)为佣金真正到账 +_INVALID_STATUS = {"4", "5"} +_SETTLED_STATUS = "6" + + +# ───────────── 单位换算 ───────────── +def _yuan_to_cents(v: object) -> int | None: + """「元」字符串/数 → 分。None / 空 / 字面 "null" → None。""" + if v is None: + return None + s = str(v).strip() + if not s or s.lower() == "null": + return None + try: + return int((Decimal(s) * 100).to_integral_value()) + except (InvalidOperation, ValueError): + return None + + +def _ts_to_dt(ts: object) -> datetime | None: + """秒级时间戳 → tz-aware UTC datetime(绝对时刻,前端按北京展示)。""" + if not ts: + return None + try: + return datetime.fromtimestamp(int(ts), tz=timezone.utc) + except (ValueError, OSError, TypeError): + return None + + +# ───────────── 群 ───────────── +def get_group(db: Session, group_id: int) -> CpsGroup | None: + return db.get(CpsGroup, group_id) + + +def get_group_by_sid(db: Session, sid: str) -> CpsGroup | None: + return db.execute(select(CpsGroup).where(CpsGroup.sid == sid)).scalar_one_or_none() + + +def list_groups( + db: Session, *, keyword: str | None = None, status: str | None = None, + limit: int = 20, cursor: int | None = None, +) -> tuple[list[CpsGroup], int | None, int]: + stmt = select(CpsGroup) + if keyword and keyword.strip(): + kw = f"%{keyword.strip()}%" + stmt = stmt.where(CpsGroup.name.ilike(kw) | CpsGroup.sid.ilike(kw)) + if status: + stmt = stmt.where(CpsGroup.status == status) + return offset_paginate(db, stmt, (desc(CpsGroup.id),), limit=limit, cursor=cursor) + + +def create_group( + db: Session, *, name: str, sid: str | None = None, + member_count: int | None = None, remark: str | None = None, commit: bool = True, +) -> CpsGroup: + """建群。sid 留空 → 自动 g(先 flush 拿 id 再回填)。""" + group = CpsGroup( + name=name, + sid=sid or f"tmp{uuid4().hex[:20]}", # 临时唯一占位,留空时下面回填 g + member_count=member_count, + remark=remark, + ) + db.add(group) + db.flush() + if not sid: + group.sid = f"g{group.id}" + db.flush() + if commit: + db.commit() + db.refresh(group) + return group + + +def update_group( + db: Session, group: CpsGroup, *, name: str | None = None, + member_count: int | None = None, status: str | None = None, + remark: str | None = None, commit: bool = True, +) -> CpsGroup: + if name is not None: + group.name = name + if member_count is not None: + group.member_count = member_count + if status is not None: + group.status = status + if remark is not None: + group.remark = remark + if commit: + db.commit() + db.refresh(group) + else: + db.flush() + return group + + +# ───────────── 活动 ───────────── +def get_activity(db: Session, activity_id: int) -> CpsActivity | None: + return db.get(CpsActivity, activity_id) + + +def list_activities( + db: Session, *, platform: str | None = None, status: str | None = None, + limit: int = 20, cursor: int | None = None, +) -> tuple[list[CpsActivity], int | None, int]: + stmt = select(CpsActivity) + if platform: + stmt = stmt.where(CpsActivity.platform == platform) + if status: + stmt = stmt.where(CpsActivity.status == status) + return offset_paginate(db, stmt, (desc(CpsActivity.id),), limit=limit, cursor=cursor) + + +def create_activity( + db: Session, *, name: str, platform: str = "meituan", act_id: str | None = None, + product_view_sign: str | None = None, remark: str | None = None, commit: bool = True, +) -> CpsActivity: + activity = CpsActivity( + name=name, platform=platform, act_id=act_id, + product_view_sign=product_view_sign, remark=remark, + ) + db.add(activity) + if commit: + db.commit() + db.refresh(activity) + else: + db.flush() + return activity + + +# ───────────── 订单(对账) ───────────── +def _map_order_fields(r: dict) -> dict: + """美团 query_order 单条 dataList → CpsOrder 字段(金额转分、时间转 datetime)。""" + pn = r.get("productName") + if pn and len(pn) > 500: + pn = pn[:500] + return { + "sid": (r.get("sid") or None), + "act_id": str(r["actId"]) if r.get("actId") is not None else None, + "biz_line": r.get("businessLine"), + "trade_type": r.get("tradeType"), + "pay_price_cents": _yuan_to_cents(r.get("payPrice")), + "commission_cents": _yuan_to_cents(r.get("profit")), + "commission_rate": str(r["commissionRate"]) if r.get("commissionRate") is not None else None, + "refund_price_cents": _yuan_to_cents(r.get("refundPrice")), + "refund_profit_cents": _yuan_to_cents(r.get("refundProfit")), + "mt_status": str(r["status"]) if r.get("status") is not None else None, + "invalid_reason": (r.get("invalidReason") or None), + "product_name": pn or None, + "pay_time": _ts_to_dt(r.get("payTime")), + "mt_update_time": _ts_to_dt(r.get("updateTime")), + "raw": r, + } + + +def reconcile_orders( + db: Session, *, start_time: int, end_time: int, + query_time_type: int = 1, sid: str | None = None, max_pages: int = 200, +) -> dict: + """调美团 query_order 分页拉单 → 按 order_id upsert。返回 {fetched, inserted, updated, pages}。 + + 订单状态会随时间变(付款→完成→结算/退款),重复拉同一单则更新。max_pages 防异常死循环。 + """ + fetched = inserted = updated = pages = 0 + page = 1 + while page <= max_pages: + resp = meituan.query_order( + sid=sid, start_time=start_time, end_time=end_time, + query_time_type=query_time_type, page=page, limit=100, + ) + rows = ((resp.get("data") or {}).get("dataList")) or [] + if not rows: + break + pages += 1 + for r in rows: + order_id = str(r.get("orderId") or "").strip() + if not order_id: + continue + fetched += 1 + fields = _map_order_fields(r) + existing = db.execute( + select(CpsOrder).where(CpsOrder.order_id == order_id) + ).scalar_one_or_none() + if existing is None: + db.add(CpsOrder(order_id=order_id, **fields)) + inserted += 1 + else: + for k, v in fields.items(): + setattr(existing, k, v) + updated += 1 + if len(rows) < 100: + break + page += 1 + db.commit() + return {"fetched": fetched, "inserted": inserted, "updated": updated, "pages": pages} + + +def list_orders( + db: Session, *, sid: str | None = None, mt_status: str | None = None, + limit: int = 20, cursor: int | None = None, +) -> tuple[list[CpsOrder], int | None, int]: + stmt = select(CpsOrder) + if sid: + stmt = stmt.where(CpsOrder.sid == sid) + if mt_status: + stmt = stmt.where(CpsOrder.mt_status == mt_status) + return offset_paginate(db, stmt, (desc(CpsOrder.id),), limit=limit, cursor=cursor) + + +# ───────────── 统计(按群聚合) ───────────── +def group_stats( + db: Session, *, date_from: datetime | None = None, date_to: datetime | None = None, +) -> list[dict]: + """按 sid 聚合订单 + join 群信息。已建但本期无单的活跃群也列出(全 0)。 + 未归群的 sid(历史/其它来源)单独成行 group_id=None。按预估佣金降序。 + """ + stmt = select(CpsOrder) + if date_from is not None: + stmt = stmt.where(CpsOrder.pay_time >= _as_utc(date_from)) + if date_to is not None: + stmt = stmt.where(CpsOrder.pay_time <= _as_utc(date_to)) + orders = list(db.execute(stmt).scalars().all()) + + groups = {g.sid: g for g in db.execute(select(CpsGroup)).scalars().all()} + + buckets: dict[str | None, list[CpsOrder]] = {} + for o in orders: + buckets.setdefault(o.sid, []).append(o) + + rows: list[dict] = [] + for sid, items in buckets.items(): + g = groups.get(sid) if sid else None + valid = [o for o in items if o.mt_status not in _INVALID_STATUS] + settled = [o for o in items if o.mt_status == _SETTLED_STATUS] + canceled = [o for o in items if o.mt_status in _INVALID_STATUS] + rows.append({ + "group_id": g.id if g else None, + "sid": sid, + "name": g.name if g else (sid or "(无 sid 归属)"), + "member_count": g.member_count if g else None, + "order_count": len(valid), + "settled_count": len(settled), + "canceled_count": len(canceled), + "gmv_cents": sum(o.pay_price_cents or 0 for o in valid), + "est_commission_cents": sum(o.commission_cents or 0 for o in valid), + "settled_commission_cents": sum(o.commission_cents or 0 for o in settled), + }) + + # 已建的活跃群但本期无单 → 补 0 行,让运营看到全部群 + seen = set(buckets.keys()) + for sid, g in groups.items(): + if sid not in seen and g.status == "active": + rows.append({ + "group_id": g.id, "sid": sid, "name": g.name, + "member_count": g.member_count, "order_count": 0, + "settled_count": 0, "canceled_count": 0, "gmv_cents": 0, + "est_commission_cents": 0, "settled_commission_cents": 0, + }) + + # 合并点击数据(按 group_id):点击只对生成过我们短链的群有,未归群行(group_id=None)恒 0 + clicks = cps_link_repo.click_stats_by_group(db, date_from=date_from, date_to=date_to) + for r in rows: + c = clicks.get(r["group_id"]) if r["group_id"] is not None else None + r["click_pv"] = c["pv"] if c else 0 + r["click_uv"] = c["uv"] if c else 0 + + rows.sort(key=lambda x: x["est_commission_cents"], reverse=True) + return rows diff --git a/app/admin/routers/cps.py b/app/admin/routers/cps.py new file mode 100644 index 0000000..592f363 --- /dev/null +++ b/app/admin/routers/cps.py @@ -0,0 +1,268 @@ +"""admin CPS 分发与对账:群(sid)管理 + 活动池 + 生成带 sid 券链接 + 美团订单对账 + 统计。 + +当前仅接美团(get_referral_link / query_order)。淘宝/京东待各自联盟凭证。 +群/活动管理 = operator;订单对账(涉佣金) = finance;只读列表/统计 = 登录即可。 +""" +from __future__ import annotations + +import time +from datetime import datetime, timedelta, timezone +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query, Request + +from app.admin.audit import write_audit +from app.admin.deps import AdminDb, get_client_ip, get_current_admin, require_role +from app.admin.repositories import cps as cps_repo +from app.admin.schemas.common import CursorPage +from app.core.config import settings +from app.repositories import cps_link as cps_link_repo +from app.admin.schemas.cps import ( + CpsActivityCreate, + CpsActivityOut, + CpsGroupCreate, + CpsGroupOut, + CpsGroupStat, + CpsGroupUpdate, + CpsOrderOut, + CpsReconcileResult, + CpsReferralLinkOut, + CpsReferralLinkRequest, + CpsStatsOut, +) +from app.integrations import meituan +from app.integrations.meituan import MeituanCpsError +from app.models.admin import AdminUser + +router = APIRouter( + prefix="/admin/api/cps", + tags=["admin-cps"], + dependencies=[Depends(get_current_admin)], +) + + +# ───────────── 群(sid) ───────────── +@router.get("/groups", response_model=CursorPage[CpsGroupOut], summary="群(sid)列表") +def list_groups( + db: AdminDb, + keyword: Annotated[str | None, Query(max_length=100)] = None, + status: Annotated[str | None, Query(pattern="^(active|archived)$")] = None, + limit: Annotated[int, Query(ge=1, le=100)] = 20, + cursor: Annotated[int | None, Query()] = None, +) -> CursorPage[CpsGroupOut]: + items, next_cursor, total = cps_repo.list_groups( + db, keyword=keyword, status=status, limit=limit, cursor=cursor + ) + return CursorPage( + items=[CpsGroupOut.model_validate(g) for g in items], + next_cursor=next_cursor, + total=total, + ) + + +@router.post("/groups", response_model=CpsGroupOut, summary="新建群(分配 sid)") +def create_group( + body: CpsGroupCreate, + request: Request, + admin: Annotated[AdminUser, Depends(require_role("operator"))], + db: AdminDb, +) -> CpsGroupOut: + if body.sid and cps_repo.get_group_by_sid(db, body.sid) is not None: + raise HTTPException(status_code=409, detail="sid 已存在") + group = cps_repo.create_group( + db, name=body.name, sid=body.sid, member_count=body.member_count, + remark=body.remark, commit=False, + ) + write_audit( + db, admin, action="cps.group.create", target_type="cps_group", target_id=group.id, + detail={"sid": group.sid, "name": group.name}, ip=get_client_ip(request), commit=False, + ) + db.commit() + db.refresh(group) + return CpsGroupOut.model_validate(group) + + +@router.patch("/groups/{group_id}", response_model=CpsGroupOut, summary="编辑群") +def update_group( + group_id: int, + body: CpsGroupUpdate, + request: Request, + admin: Annotated[AdminUser, Depends(require_role("operator"))], + db: AdminDb, +) -> CpsGroupOut: + group = cps_repo.get_group(db, group_id) + if group is None: + raise HTTPException(status_code=404, detail="群不存在") + cps_repo.update_group( + db, group, name=body.name, member_count=body.member_count, + status=body.status, remark=body.remark, commit=False, + ) + write_audit( + db, admin, action="cps.group.update", target_type="cps_group", target_id=group_id, + detail=body.model_dump(exclude_none=True), ip=get_client_ip(request), commit=False, + ) + db.commit() + db.refresh(group) + return CpsGroupOut.model_validate(group) + + +# ───────────── 活动 ───────────── +@router.get("/activities", response_model=CursorPage[CpsActivityOut], summary="活动列表") +def list_activities( + db: AdminDb, + platform: Annotated[str | None, Query(max_length=20)] = None, + status: Annotated[str | None, Query(pattern="^(active|archived)$")] = None, + limit: Annotated[int, Query(ge=1, le=100)] = 20, + cursor: Annotated[int | None, Query()] = None, +) -> CursorPage[CpsActivityOut]: + items, next_cursor, total = cps_repo.list_activities( + db, platform=platform, status=status, limit=limit, cursor=cursor + ) + return CursorPage( + items=[CpsActivityOut.model_validate(a) for a in items], + next_cursor=next_cursor, + total=total, + ) + + +@router.post("/activities", response_model=CpsActivityOut, summary="新建活动") +def create_activity( + body: CpsActivityCreate, + request: Request, + admin: Annotated[AdminUser, Depends(require_role("operator"))], + db: AdminDb, +) -> CpsActivityOut: + if not body.act_id and not body.product_view_sign: + raise HTTPException(status_code=400, detail="act_id 或 product_view_sign 必填其一") + activity = cps_repo.create_activity( + db, name=body.name, platform=body.platform, act_id=body.act_id, + product_view_sign=body.product_view_sign, remark=body.remark, commit=False, + ) + write_audit( + db, admin, action="cps.activity.create", target_type="cps_activity", target_id=activity.id, + detail={"name": activity.name, "act_id": activity.act_id, "platform": activity.platform}, + ip=get_client_ip(request), commit=False, + ) + db.commit() + db.refresh(activity) + return CpsActivityOut.model_validate(activity) + + +# ───────────── 转链 ───────────── +@router.post("/referral-link", response_model=CpsReferralLinkOut, summary="生成群发短链(带 sid + 点击统计)") +def generate_referral_link( + body: CpsReferralLinkRequest, + request: Request, + admin: Annotated[AdminUser, Depends(require_role("operator"))], + db: AdminDb, +) -> CpsReferralLinkOut: + group = cps_repo.get_group(db, body.group_id) + if group is None: + raise HTTPException(status_code=404, detail="群不存在") + activity = cps_repo.get_activity(db, body.activity_id) + if activity is None: + raise HTTPException(status_code=404, detail="活动不存在") + if activity.platform != "meituan": + raise HTTPException(status_code=400, detail=f"当前仅支持美团转链(活动平台={activity.platform})") + try: + # 全要:短链作跳转目标(微信可打开),长链/deeplink 作参考/备用 + resp = meituan.get_referral_link( + act_id=activity.act_id, + product_view_sign=activity.product_view_sign, + sid=group.sid, + link_type_list=[1, 2, 3], + ) + except MeituanCpsError as e: + raise HTTPException(status_code=502, detail=f"美团转链失败: {e}") from e + + link_map = {str(k): v for k, v in (resp.get("referralLinkMap") or {}).items()} + if not link_map and resp.get("data"): + link_map = {"1": resp["data"]} + # 跳转目标优先短链(2,微信可打开)→ 长链(1)→ deeplink(3) + target_url = link_map.get("2") or link_map.get("1") or link_map.get("3") + if not target_url: + raise HTTPException(status_code=502, detail="美团未返回有效链接(检查 actId 是否在推广有效期)") + + # 存我们的短链:发群用 /c/{code},用户点 → 我们记点击 → 302 跳 target_url + link = cps_link_repo.create_link( + db, group_id=group.id, activity_id=activity.id, sid=group.sid, + target_url=target_url, platform=activity.platform, commit=False, + ) + write_audit( + db, admin, action="cps.referral_link.generate", target_type="cps_link", target_id=link.id, + detail={"sid": group.sid, "activity_id": activity.id, "code": link.code}, + ip=get_client_ip(request), commit=False, + ) + db.commit() + db.refresh(link) + + base = settings.CPS_REDIRECT_BASE.rstrip("/") + redirect_url = f"{base}/c/{link.code}" if base else f"/c/{link.code}" + return CpsReferralLinkOut( + redirect_url=redirect_url, + code=link.code, + sid=group.sid, + group_name=group.name, + activity_name=activity.name, + target_url=target_url, + link_map=link_map, + ) + + +# ───────────── 订单对账 ───────────── +@router.post("/orders/reconcile", response_model=CpsReconcileResult, summary="拉取美团订单对账") +def reconcile_orders( + request: Request, + admin: Annotated[AdminUser, Depends(require_role("finance"))], + db: AdminDb, + days: Annotated[int, Query(ge=1, le=90)] = 7, + sid: Annotated[str | None, Query(max_length=64)] = None, +) -> CpsReconcileResult: + now = int(time.time()) + try: + result = cps_repo.reconcile_orders( + db, start_time=now - days * 86400, end_time=now, sid=sid, + ) + except MeituanCpsError as e: + raise HTTPException(status_code=502, detail=f"美团拉单失败: {e}") from e + write_audit( + db, admin, action="cps.order.reconcile", target_type="cps_order", target_id=None, + detail={"days": days, "sid": sid, **result}, ip=get_client_ip(request), commit=True, + ) + return CpsReconcileResult(**result) + + +@router.get("/orders", response_model=CursorPage[CpsOrderOut], summary="订单明细") +def list_orders( + db: AdminDb, + sid: Annotated[str | None, Query(max_length=64)] = None, + mt_status: Annotated[str | None, Query(max_length=8)] = None, + limit: Annotated[int, Query(ge=1, le=100)] = 20, + cursor: Annotated[int | None, Query()] = None, +) -> CursorPage[CpsOrderOut]: + items, next_cursor, total = cps_repo.list_orders( + db, sid=sid, mt_status=mt_status, limit=limit, cursor=cursor + ) + return CursorPage( + items=[CpsOrderOut.model_validate(o) for o in items], + next_cursor=next_cursor, + total=total, + ) + + +# ───────────── 统计 ───────────── +@router.get("/stats", response_model=CpsStatsOut, summary="按群对账统计") +def get_stats( + db: AdminDb, + days: Annotated[int, Query(ge=1, le=90)] = 30, +) -> CpsStatsOut: + date_to = datetime.now(timezone.utc) + date_from = date_to - timedelta(days=days) + rows = cps_repo.group_stats(db, date_from=date_from, date_to=date_to) + stats = [CpsGroupStat(**r) for r in rows] + return CpsStatsOut( + groups=stats, + total_order_count=sum(s.order_count for s in stats), + total_est_commission_cents=sum(s.est_commission_cents for s in stats), + total_settled_commission_cents=sum(s.settled_commission_cents for s in stats), + ) diff --git a/app/admin/schemas/cps.py b/app/admin/schemas/cps.py new file mode 100644 index 0000000..9028fe1 --- /dev/null +++ b/app/admin/schemas/cps.py @@ -0,0 +1,122 @@ +"""admin CPS 分发与对账 schemas。金额统一「分」(cents),前端 yuan() 展示。""" +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + + +# ───────────── 群(sid) ───────────── +class CpsGroupOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + sid: str + name: str + member_count: int | None = None + status: str + remark: str | None = None + created_at: datetime + + +class CpsGroupCreate(BaseModel): + name: str = Field(min_length=1, max_length=128) + # 留空 → 后端自动生成 g;填写则必须字母+数字(美团 sid 限制)。 + sid: str | None = Field(default=None, max_length=64, pattern=r"^[A-Za-z0-9]+$") + member_count: int | None = Field(default=None, ge=0) + remark: str | None = Field(default=None, max_length=256) + + +class CpsGroupUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=128) + member_count: int | None = Field(default=None, ge=0) + status: str | None = Field(default=None, pattern=r"^(active|archived)$") + remark: str | None = Field(default=None, max_length=256) + + +# ───────────── 活动(actId) ───────────── +class CpsActivityOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + platform: str + name: str + act_id: str | None = None + product_view_sign: str | None = None + status: str + remark: str | None = None + created_at: datetime + + +class CpsActivityCreate(BaseModel): + name: str = Field(min_length=1, max_length=128) + platform: str = Field(default="meituan", pattern=r"^(meituan|taobao|jd)$") + act_id: str | None = Field(default=None, max_length=64) + product_view_sign: str | None = Field(default=None, max_length=128) + remark: str | None = Field(default=None, max_length=256) + + +# ───────────── 转链 ───────────── +class CpsReferralLinkRequest(BaseModel): + group_id: int + activity_id: int + # 链接类型:1长链 2短链 3deeplink。默认短链(微信可打开)+deeplink。 + link_types: list[int] = Field(default_factory=lambda: [2, 3]) + + +class CpsReferralLinkOut(BaseModel): + redirect_url: str # ★ 我们的群发短链(/c/{code}),发群就用这个,点击经我们统计 + code: str + sid: str + group_name: str + activity_name: str + target_url: str # 实际 302 跳转的美团短链 + link_map: dict[str, str] # 美团原始 1长链/2短链/3deeplink(参考/备用) + + +# ───────────── 对账拉单 ───────────── +class CpsReconcileResult(BaseModel): + fetched: int # 本次美团返回的订单条数 + inserted: int # 新入库 + updated: int # 已存在→更新状态/佣金 + pages: int # 翻了几页 + + +# ───────────── 订单明细 ───────────── +class CpsOrderOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + order_id: str + sid: str | None = None + act_id: str | None = None + pay_price_cents: int | None = None + commission_cents: int | None = None + commission_rate: str | None = None + mt_status: str | None = None + invalid_reason: str | None = None + product_name: str | None = None + pay_time: datetime | None = None + + +# ───────────── 统计 ───────────── +class CpsGroupStat(BaseModel): + group_id: int | None = None # None = 未归群的 sid(历史/其它来源) + sid: str | None = None + name: str + member_count: int | None = None + click_pv: int = 0 # 点击总次数 + click_uv: int = 0 # 独立点击(按 ip+ua 近似去重) + order_count: int # 有效订单数(不含取消/风控) + settled_count: int # 已结算单数 + canceled_count: int # 取消+风控单数 + gmv_cents: int # 成交额(有效单 payPrice 合计) + est_commission_cents: int # 预估佣金(有效单 profit 合计) + settled_commission_cents: int # 已结算佣金 + + +class CpsStatsOut(BaseModel): + groups: list[CpsGroupStat] + total_order_count: int + total_est_commission_cents: int + total_settled_commission_cents: int diff --git a/app/api/v1/cps_redirect.py b/app/api/v1/cps_redirect.py new file mode 100644 index 0000000..3b4120c --- /dev/null +++ b/app/api/v1/cps_redirect.py @@ -0,0 +1,39 @@ +"""CPS 群发短链跳转:用户点 /c/{code} → 记一条点击 → 302 跳美团。 + +公网、无鉴权(群里任何人点都要能跳)。记点击失败绝不影响跳转(用户体验优先)。 +""" +from __future__ import annotations + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import RedirectResponse +from sqlalchemy.orm import Session + +from app.db.session import get_db +from app.repositories import cps_link as cps_link_repo + +router = APIRouter(tags=["cps-redirect"]) + +# code 不存在/失效时的兜底落地(避免用户看到报错) +_FALLBACK_URL = "https://www.meituan.com/" + + +def _client_ip(request: Request) -> str | None: + xff = request.headers.get("x-forwarded-for") + if xff: + return xff.split(",")[0].strip() + return request.client.host if request.client else None + + +@router.get("/c/{code}", summary="群发短链跳转(记点击 + 302)") +def cps_redirect(code: str, request: Request, db: Session = Depends(get_db)): + link = cps_link_repo.get_by_code(db, code) + if link is None: + return RedirectResponse(_FALLBACK_URL, status_code=302) + try: + cps_link_repo.record_click( + db, link=link, ip=_client_ip(request), + ua=request.headers.get("user-agent"), + ) + except Exception: + pass # 记点击失败不阻断跳转 + return RedirectResponse(link.target_url, status_code=302) diff --git a/app/core/config.py b/app/core/config.py index a06ebd0..5c3a787 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -190,6 +190,13 @@ class Settings(BaseSettings): # MVP 落地页就放 app-server 的 /media 静态目录下(dl.html)。 INVITE_LANDING_URL: str = "https://app-api.shaguabijia.com/media/dl.html" + # ===== CPS 群发短链跳转 ===== + # 群发券链接套一层我们的短链 /c/{code} 做点击统计后 302 跳美团。这里是写进 admin + # 生成链接里的对外跳转域名前缀。本地填 http://localhost:8770;生产**强烈建议用独立 + # 域名**(大量群发会被微信封,别用主 API 域名连累整个 App)。留空 → 跳转端点用请求 + # 的 Host 兜底拼绝对地址。 + CPS_REDIRECT_BASE: str = "" + # ===== CORS ===== CORS_ALLOW_ORIGINS: str = "" diff --git a/app/integrations/meituan.py b/app/integrations/meituan.py index 6d1b10a..b628b90 100644 --- a/app/integrations/meituan.py +++ b/app/integrations/meituan.py @@ -126,13 +126,23 @@ def query_coupon( def get_referral_link( *, - product_view_sign: str, + product_view_sign: str | None = None, + act_id: str | None = None, platform: int = 1, biz_line: int | None = None, sid: str | None = None, link_type_list: list[int] | None = None, ) -> dict[str, Any]: - body: dict[str, Any] = {"productViewSign": product_view_sign} + """换推广链接。actId(活动物料)与 productViewSign(商品券)二选一。 + 用 actId 入参时,出参 data 为 null,链接全在 referralLinkMap(实测)。 + """ + if not act_id and not product_view_sign: + raise MeituanCpsError("act_id 或 product_view_sign 必填其一") + body: dict[str, Any] = {} + if act_id: + body["actId"] = act_id + else: + body["productViewSign"] = product_view_sign if platform == 2: body["platform"] = 2 if biz_line is not None: @@ -142,3 +152,30 @@ def get_referral_link( body["linkTypeList"] = link_type_list or [1, 3] return _call("/cps_open/common/api/v1/get_referral_link", body) + + +def query_order( + *, + sid: str | None = None, + start_time: int, + end_time: int, + query_time_type: int = 1, + page: int = 1, + limit: int = 100, +) -> dict[str, Any]: + """按时间窗(+可选 sid)拉 CPS 订单做对账。实测要点: + - 分页是 page / limit(不是 pageNo/pageSize);startTime/endTime 是 10 位秒级时间戳 + - query_time_type: 1 按支付时间 2 按更新时间 + - 响应 data.dataList[],每条含 orderId / sid / payPrice(元) / profit(元) / + commissionRate / status(2付款 3完成 4取消 5风控 6结算) / payTime(秒) / actId 等 + """ + body: dict[str, Any] = { + "queryTimeType": query_time_type, + "startTime": start_time, + "endTime": end_time, + "page": page, + "limit": limit, + } + if sid: + body["sid"] = sid + return _call("/cps_open/common/api/v1/query_order", body) diff --git a/app/main.py b/app/main.py index cea4568..140a7a5 100644 --- a/app/main.py +++ b/app/main.py @@ -20,6 +20,7 @@ from app.api.v1.compare import router as compare_router from app.api.v1.compare_milestone import router as compare_milestone_router from app.api.v1.compare_record import router as compare_record_router from app.api.v1.coupon import router as coupon_router +from app.api.v1.cps_redirect import router as cps_redirect_router from app.api.internal.price import router as internal_price_router from app.api.internal.store import router as internal_store_router from app.api.v1.feedback import router as feedback_router @@ -107,6 +108,8 @@ app.include_router(report_router) app.include_router(internal_price_router) app.include_router(internal_store_router) app.include_router(platform_router) +# CPS 群发短链跳转 /c/{code}(公网无鉴权:记点击 → 302 跳美团) +app.include_router(cps_redirect_router) # 用户上传文件(头像)静态服务。生产可改由 nginx 直接 serve MEDIA_ROOT。 _media_root = Path(settings.MEDIA_ROOT) diff --git a/app/models/__init__.py b/app/models/__init__.py index a089a8a..1955e31 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -6,6 +6,10 @@ from app.models.ad_watch_log import AdWatchLog # noqa: F401 from app.models.admin import AdminAuditLog, AdminUser # noqa: F401 from app.models.app_config import AppConfig # noqa: F401 from app.models.comparison import ComparisonRecord # noqa: F401 +from app.models.cps_activity import CpsActivity # noqa: F401 +from app.models.cps_group import CpsGroup # noqa: F401 +from app.models.cps_link import CpsClick, CpsLink # noqa: F401 +from app.models.cps_order import CpsOrder # noqa: F401 from app.models.comparison_milestone import ComparisonMilestoneClaim # noqa: F401 from app.models.coupon_state import ( # noqa: F401 CouponClaimRecord, diff --git a/app/models/cps_activity.py b/app/models/cps_activity.py new file mode 100644 index 0000000..d52823a --- /dev/null +++ b/app/models/cps_activity.py @@ -0,0 +1,34 @@ +"""CPS 可推广活动池(cps_activity)。 + +运营预存常推的活动(券),生成链接时选它,省得每次记 actId。当前只接美团: +`act_id` = 美团联盟「我要推广-活动推广」第一列的物料 ID。`platform` 预留,接 +淘宝/京东时复用本表(各自的活动标识)。 +""" +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class CpsActivity(Base): + __tablename__ = "cps_activity" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + platform: Mapped[str] = mapped_column(String(20), nullable=False, default="meituan") # meituan|taobao|jd + name: Mapped[str] = mapped_column(String(128), nullable=False) + # 美团活动物料 ID(actId);淘宝/京东接入后存各自活动标识。 + act_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + # 美团商品券 productViewSign(与 act_id 二选一转链;多数活动用 act_id)。 + product_view_sign: Mapped[str | None] = mapped_column(String(128), nullable=True) + status: Mapped[str] = mapped_column(String(20), nullable=False, default="active") # active | archived + remark: Mapped[str | None] = mapped_column(String(256), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return f"" diff --git a/app/models/cps_group.py b/app/models/cps_group.py new file mode 100644 index 0000000..fcd40fd --- /dev/null +++ b/app/models/cps_group.py @@ -0,0 +1,32 @@ +"""CPS 推广群(cps_group)。 + +每个微信群一条记录,`sid` 是美团联盟二级渠道追踪位(get_referral_link / query_order +共用):发券时塞进推广链接,订单回来按 sid 归群对账。美团限制 sid 仅字母+数字 ≤64。 +""" +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class CpsGroup(Base): + __tablename__ = "cps_group" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + # 每群唯一的渠道标识,建群时分配(留空则自动生成 g)。字母+数字,≤64。 + sid: Mapped[str] = mapped_column(String(64), unique=True, index=True, nullable=False) + name: Mapped[str] = mapped_column(String(128), nullable=False) + # 群人数,运营填,作转化率分母(可空)。 + member_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + status: Mapped[str] = mapped_column(String(20), nullable=False, default="active") # active | archived + remark: Mapped[str | None] = mapped_column(String(256), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return f"" diff --git a/app/models/cps_link.py b/app/models/cps_link.py new file mode 100644 index 0000000..ef53859 --- /dev/null +++ b/app/models/cps_link.py @@ -0,0 +1,52 @@ +"""CPS 群发短链(cps_link)+ 点击事件(cps_click)。 + +群发券链接套一层我们自己的短链 `/c/{code}`:用户点 → 记一条 cps_click → 302 跳 +target_url(美团短链,微信可打开)。据此统计点击 PV/UV(按群/活动/时段),配合 cps_order +的下单/佣金做"点击→下单→佣金"漏斗。group_id/sid 在 cps_click 里冗余一份,统计直接按 +群聚合点击、不必 join cps_link。 +""" +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class CpsLink(Base): + __tablename__ = "cps_link" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + # 短码:发群链接 /c/{code} 的 code,全局唯一。 + code: Mapped[str] = mapped_column(String(16), unique=True, index=True, nullable=False) + group_id: Mapped[int] = mapped_column(Integer, index=True, nullable=False) + activity_id: Mapped[int] = mapped_column(Integer, index=True, nullable=False) + sid: Mapped[str] = mapped_column(String(64), index=True, nullable=False) + platform: Mapped[str] = mapped_column(String(20), nullable=False, default="meituan") + # 302 跳转目标(美团短链,微信内可打开并唤起 App)。 + target_url: Mapped[str] = mapped_column(String(1024), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return f"" + + +class CpsClick(Base): + __tablename__ = "cps_click" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + link_id: Mapped[int] = mapped_column(Integer, index=True, nullable=False) + group_id: Mapped[int] = mapped_column(Integer, index=True, nullable=False) # 冗余,按群聚合快 + sid: Mapped[str] = mapped_column(String(64), index=True, nullable=False) # 冗余 + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + ua: Mapped[str | None] = mapped_column(String(512), nullable=True) + clicked_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), index=True, nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return f"" diff --git a/app/models/cps_order.py b/app/models/cps_order.py new file mode 100644 index 0000000..306b30d --- /dev/null +++ b/app/models/cps_order.py @@ -0,0 +1,63 @@ +"""CPS 对账订单(cps_order)。 + +从美团联盟 query_order 按时间窗拉回、按 sid 归群的订单明细。字段对齐 query_order +实测返回: + - payPrice / profit 是「元」字符串 → 入库统一转「分」(与全站口径一致) + - payTime / updateTime 是秒级时间戳 → 入库转 tz-aware datetime + - status: 2付款 3完成 4取消 5风控 6结算(取消/风控不计佣金) +order_id 全局唯一,reconcile 按它 upsert(订单状态会变,重复拉则更新)。 +""" +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import JSON, DateTime, Integer, String, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class CpsOrder(Base): + __tablename__ = "cps_order" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + # 美团订单号(加密串),全局唯一,upsert 幂等键。 + order_id: Mapped[str] = mapped_column(String(64), unique=True, index=True, nullable=False) + # 渠道追踪位 = 群 sid(历史无 sid 订单为空)。按它归群聚合。 + sid: Mapped[str | None] = mapped_column(String(64), index=True, nullable=True) + act_id: Mapped[str | None] = mapped_column(String(64), index=True, nullable=True) + biz_line: Mapped[int | None] = mapped_column(Integer, nullable=True) # 1=外卖 + trade_type: Mapped[int | None] = mapped_column(Integer, nullable=True) # 1=cps 2=cpa + + pay_price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) + commission_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) # 预估佣金(profit) + commission_rate: Mapped[str | None] = mapped_column(String(16), nullable=True) # "300"=3% "10"=0.1% + refund_price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) + refund_profit_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) + + # 美团订单状态: 2付款 3完成 4取消 5风控 6结算 + mt_status: Mapped[str | None] = mapped_column(String(8), index=True, nullable=True) + invalid_reason: Mapped[str | None] = mapped_column(String(128), nullable=True) + product_name: Mapped[str | None] = mapped_column(String(512), nullable=True) + + pay_time: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), index=True, nullable=True + ) + mt_update_time: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + raw: Mapped[dict] = mapped_column( + JSON().with_variant(JSONB(), "postgresql"), nullable=False, default=dict + ) + first_seen: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return ( + f"" + ) diff --git a/app/repositories/cps_link.py b/app/repositories/cps_link.py new file mode 100644 index 0000000..9991ff8 --- /dev/null +++ b/app/repositories/cps_link.py @@ -0,0 +1,82 @@ +"""CPS 群发短链 + 点击 数据访问(用户侧跳转端点 + admin 生成/统计 共用)。 + +短码生成去掉易混字符;点击按 (ip, ua) 近似去重 UV。点击量级小、admin 低频,统计在 +Python 侧聚合,跨 PG/SQLite 无方言坑(与 admin cps repo 同风格)。 +""" +from __future__ import annotations + +import secrets +from datetime import datetime + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.models.cps_link import CpsClick, CpsLink + +# 去掉易混字符 0/O/1/l/I,运营/用户肉眼复制不易错 +_CODE_ALPHABET = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" + + +def _gen_code(n: int = 8) -> str: + return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(n)) + + +def get_by_code(db: Session, code: str) -> CpsLink | None: + return db.execute(select(CpsLink).where(CpsLink.code == code)).scalar_one_or_none() + + +def create_link( + db: Session, *, group_id: int, activity_id: int, sid: str, target_url: str, + platform: str = "meituan", commit: bool = True, +) -> CpsLink: + """生成唯一短码并存库。冲突重试 5 次,仍冲突用 12 位兜底(概率近 0)。""" + code = _gen_code() + for _ in range(5): + if get_by_code(db, code) is None: + break + code = _gen_code() + else: + code = _gen_code(12) + link = CpsLink( + code=code, group_id=group_id, activity_id=activity_id, sid=sid, + target_url=target_url, platform=platform, + ) + db.add(link) + if commit: + db.commit() + db.refresh(link) + else: + db.flush() + return link + + +def record_click(db: Session, *, link: CpsLink, ip: str | None = None, ua: str | None = None) -> None: + db.add(CpsClick( + link_id=link.id, group_id=link.group_id, sid=link.sid, + ip=ip, ua=(ua[:500] if ua else None), + )) + db.commit() + + +def click_stats_by_group( + db: Session, *, date_from: datetime | None = None, date_to: datetime | None = None, +) -> dict[int, dict]: + """按 group_id 聚合点击 → {group_id: {"pv": N, "uv": M}}。uv 按 (ip, ua) 近似去重。""" + stmt = select(CpsClick) + if date_from is not None: + stmt = stmt.where(CpsClick.clicked_at >= date_from) + if date_to is not None: + stmt = stmt.where(CpsClick.clicked_at <= date_to) + clicks = list(db.execute(stmt).scalars().all()) + + agg: dict[int, dict] = {} + seen: dict[int, set] = {} + for c in clicks: + a = agg.setdefault(c.group_id, {"pv": 0, "uv": 0}) + a["pv"] += 1 + s = seen.setdefault(c.group_id, set()) + key = (c.ip, c.ua) + if key not in s: + s.add(key) + a["uv"] += 1 + return agg