feat(ad-revenue): 接入穿山甲 GroMore 数据 API 拉取后台收益(预估+收益API) (#92)
- 新增穿山甲 GroMore「聚合数据报告 API」对接,按天 T+1 拉取后台 revenue(预估)与 api_revenue(收益API,更接近结算),在广告收益报表大盘 + 按天趋势级展示,与客户端自报 eCPM 折算的预估并列对照;逐条广告事件行不动(仍是客户端预估)。 - 链路:integrations/pangle_report.py(MD5 签名,与官方文档两个测试向量逐字节一致)→ scripts/sync_pangle_revenue.py(拉昨天/--days 回补)→ 新表 ad_pangle_daily_revenue (repositories/ad_pangle_revenue.py:upsert + 按日聚合)→ admin/repositories/ad_revenue.py 汇总,新增 total_pangle_revenue_yuan / total_pangle_api_revenue_yuan / daily[].pangle_*。 - 穿山甲无用户/类型/场景维度:仅全量视图(未按 user/ad_type/feed_scene 过滤)给值,否则置 None;join key 用 ad_unit_id(=客户端配的 104xxx),非 code_id。 - 新增配置 PANGLE_REPORT_USER_ID/ROLE_ID/SECURITY_KEY(≠发奖 m-key)+ site_id→应用映射; 含单测(签名向量+分页解析);已真连穿山甲验证。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: zzhyyyyy <2685922758@qq.com> Reviewed-on: #92 Co-authored-by: zhuzihao <zhuzihao@wonderable.ai> Co-committed-by: zhuzihao <zhuzihao@wonderable.ai>
This commit was merged in pull request #92.
This commit is contained in:
@@ -117,3 +117,16 @@ PANGLE_REWARD_SECRET=
|
|||||||
# ⚠️ 仅本地联调:true 时开放 POST /api/v1/ad/test-grant,让 debug 客户端看完广告直接发奖,
|
# ⚠️ 仅本地联调:true 时开放 POST /api/v1/ad/test-grant,让 debug 客户端看完广告直接发奖,
|
||||||
# 验证"看广告→金币到账"全链路(未部署公网、穿山甲 S2S 打不到本地时用)。生产必须 false(绕过反作弊)。
|
# 验证"看广告→金币到账"全链路(未部署公网、穿山甲 S2S 打不到本地时用)。生产必须 false(绕过反作弊)。
|
||||||
AD_REWARD_TEST_GRANT_ENABLED=false
|
AD_REWARD_TEST_GRANT_ENABLED=false
|
||||||
|
|
||||||
|
# ===== 穿山甲 GroMore 数据 API(按天拉收益报表,供后台广告收益报表的「穿山甲后台收益」)=====
|
||||||
|
# ⚠️ 与上面发奖回调的 m-key 是【两套不同凭证】:这三样在穿山甲后台「接入中心 → GroMore-API →
|
||||||
|
# 聚合数据报告 API」文档页领取。只读拉取 GroMore 天级 revenue(预估)/ api_revenue(收益Api),
|
||||||
|
# 不参与发奖。三样齐全才生效;留空 = scripts/sync_pangle_revenue 直接 no-op。
|
||||||
|
# 子账号(role_id≠user_id)需主账号在「角色管理」授予「查看全部数据」权限,否则查不到收益(接口 118);
|
||||||
|
# role_id 填成 = user_id 即查主账号数据。同步:线上每天 ~10:30 由 timer 跑 python -m scripts.sync_pangle_revenue。
|
||||||
|
PANGLE_REPORT_USER_ID=0
|
||||||
|
PANGLE_REPORT_ROLE_ID=0
|
||||||
|
PANGLE_REPORT_SECURITY_KEY=
|
||||||
|
# GroMore AppId(报表 site_id 维度)→ 应用环境;默认取现网两个应用,按需覆盖。
|
||||||
|
PANGLE_REPORT_SITE_ID_PROD=5830519
|
||||||
|
PANGLE_REPORT_SITE_ID_TEST=5832303
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""ad_pangle_daily_revenue: 穿山甲 GroMore 天级收益报表(后台结算口径)
|
||||||
|
|
||||||
|
新建表存放从 GroMore 数据 API 按天拉取的收益(revenue 预估 + api_revenue 收益Api),
|
||||||
|
粒度 = 日期 × 应用(app_env) × 代码位(our_code_id) × 广告源(adn)。供广告收益报表的
|
||||||
|
汇总/趋势级展示「穿山甲后台收益」,与客户端自报 eCPM 折算的预估互为对照。
|
||||||
|
|
||||||
|
Revision ID: ad_pangle_daily_revenue
|
||||||
|
Revises: 7db22acee504
|
||||||
|
Create Date: 2026-06-28
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "ad_pangle_daily_revenue"
|
||||||
|
down_revision = "7db22acee504"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"ad_pangle_daily_revenue",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||||
|
sa.Column("report_date", sa.String(length=10), nullable=False),
|
||||||
|
sa.Column("app_env", sa.String(length=16), nullable=False),
|
||||||
|
sa.Column("site_id", sa.String(length=32), nullable=True),
|
||||||
|
sa.Column("our_code_id", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("adn", sa.String(length=16), nullable=False, server_default=""),
|
||||||
|
sa.Column("revenue_yuan", sa.Float(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("api_revenue_yuan", sa.Float(), nullable=True),
|
||||||
|
sa.Column("ecpm", sa.String(length=32), nullable=True),
|
||||||
|
sa.Column("impressions", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("currency", sa.String(length=8), nullable=False, server_default="cny"),
|
||||||
|
sa.Column(
|
||||||
|
"synced_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"report_date", "app_env", "our_code_id", "adn", name="uq_ad_pangle_daily"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_ad_pangle_daily_revenue_report_date",
|
||||||
|
"ad_pangle_daily_revenue",
|
||||||
|
["report_date"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_ad_pangle_daily_revenue_our_code_id",
|
||||||
|
"ad_pangle_daily_revenue",
|
||||||
|
["our_code_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_ad_pangle_daily_revenue_our_code_id", table_name="ad_pangle_daily_revenue")
|
||||||
|
op.drop_index("ix_ad_pangle_daily_revenue_report_date", table_name="ad_pangle_daily_revenue")
|
||||||
|
op.drop_table("ad_pangle_daily_revenue")
|
||||||
@@ -31,6 +31,7 @@ from app.admin.repositories import stats as admin_stats
|
|||||||
from app.core import rewards
|
from app.core import rewards
|
||||||
from app.models.ad_ecpm import AdEcpmRecord
|
from app.models.ad_ecpm import AdEcpmRecord
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
|
from app.repositories import ad_pangle_revenue
|
||||||
|
|
||||||
|
|
||||||
def _cn_hour(dt: datetime) -> int:
|
def _cn_hour(dt: datetime) -> int:
|
||||||
@@ -255,6 +256,25 @@ def ad_revenue_report(
|
|||||||
for d in sorted(daily_map.values(), key=lambda x: x["date"])
|
for d in sorted(daily_map.values(), key=lambda x: x["date"])
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# 穿山甲后台收益(GroMore 数据 API,T+1 入库 ad_pangle_daily_revenue):汇总 + 按天趋势级展示,
|
||||||
|
# 与上面客户端自报 eCPM 折算的预估并列对照(看 gap)。穿山甲数据**无用户/场景/类型维度**,故仅在
|
||||||
|
# 「全量视图」(未按 user_id / ad_type / feed_scene 过滤)给值;一旦带这些过滤,穿山甲数无法对应口径
|
||||||
|
# → 置 None,前端显示「-」并提示。逐条事件行不动(仍是客户端预估)。
|
||||||
|
pangle_filterable = user_id is None and ad_type is None and feed_scene is None
|
||||||
|
total_pangle_revenue_yuan: float | None = None
|
||||||
|
total_pangle_api_revenue_yuan: float | None = None
|
||||||
|
if pangle_filterable:
|
||||||
|
pangle_aggs = ad_pangle_revenue.aggregate_by_date(db, date_from=date_from, date_to=date_to)
|
||||||
|
if pangle_aggs:
|
||||||
|
by_date = {a["date"]: a for a in pangle_aggs}
|
||||||
|
for d in daily:
|
||||||
|
pa = by_date.get(d["date"])
|
||||||
|
d["pangle_revenue_yuan"] = pa["revenue_yuan"] if pa else None
|
||||||
|
d["pangle_api_revenue_yuan"] = pa["api_revenue_yuan"] if pa else None
|
||||||
|
total_pangle_revenue_yuan = round(sum(a["revenue_yuan"] for a in pangle_aggs), 6)
|
||||||
|
api_vals = [a["api_revenue_yuan"] for a in pangle_aggs if a["api_revenue_yuan"] is not None]
|
||||||
|
total_pangle_api_revenue_yuan = round(sum(api_vals), 6) if api_vals else None
|
||||||
|
|
||||||
# 按小时汇总(全量,不受分页 limit/offset 影响):供前端按小时趋势图(单日 granularity=hour 时用)。
|
# 按小时汇总(全量,不受分页 limit/offset 影响):供前端按小时趋势图(单日 granularity=hour 时用)。
|
||||||
# 只在 by_hour 下聚合(此时每个 event 带 hour);否则空。前端按天趋势仍用 daily。
|
# 只在 by_hour 下聚合(此时每个 event 带 hour);否则空。前端按天趋势仍用 daily。
|
||||||
hourly: list[dict] = []
|
hourly: list[dict] = []
|
||||||
@@ -303,6 +323,10 @@ def ad_revenue_report(
|
|||||||
"truncated": len(events) > offset + limit,
|
"truncated": len(events) > offset + limit,
|
||||||
"total_impressions": total_impressions,
|
"total_impressions": total_impressions,
|
||||||
"total_revenue_yuan": total_revenue_yuan,
|
"total_revenue_yuan": total_revenue_yuan,
|
||||||
|
# 穿山甲后台收益合计(元):预估 revenue + 收益Api;非全量视图(带 user/类型/场景过滤)或无数据为 None。
|
||||||
|
"total_pangle_revenue_yuan": total_pangle_revenue_yuan,
|
||||||
|
"total_pangle_api_revenue_yuan": total_pangle_api_revenue_yuan,
|
||||||
|
"pangle_revenue_available": total_pangle_revenue_yuan is not None,
|
||||||
"total_expected_coin": total_expected_coin,
|
"total_expected_coin": total_expected_coin,
|
||||||
"total_actual_coin": total_actual_coin,
|
"total_actual_coin": total_actual_coin,
|
||||||
"mismatch_count": mismatch_count,
|
"mismatch_count": mismatch_count,
|
||||||
|
|||||||
@@ -89,6 +89,9 @@ def get_ad_revenue_report(
|
|||||||
truncated=result["truncated"],
|
truncated=result["truncated"],
|
||||||
total_impressions=result["total_impressions"],
|
total_impressions=result["total_impressions"],
|
||||||
total_revenue_yuan=result["total_revenue_yuan"],
|
total_revenue_yuan=result["total_revenue_yuan"],
|
||||||
|
total_pangle_revenue_yuan=result["total_pangle_revenue_yuan"],
|
||||||
|
total_pangle_api_revenue_yuan=result["total_pangle_api_revenue_yuan"],
|
||||||
|
pangle_revenue_available=result["pangle_revenue_available"],
|
||||||
total_expected_coin=result["total_expected_coin"],
|
total_expected_coin=result["total_expected_coin"],
|
||||||
total_actual_coin=result["total_actual_coin"],
|
total_actual_coin=result["total_actual_coin"],
|
||||||
mismatch_count=result["mismatch_count"],
|
mismatch_count=result["mismatch_count"],
|
||||||
|
|||||||
@@ -44,7 +44,13 @@ class AdRevenueDaily(BaseModel):
|
|||||||
|
|
||||||
date: str = Field(..., description="北京时间 YYYY-MM-DD")
|
date: str = Field(..., description="北京时间 YYYY-MM-DD")
|
||||||
impressions: int = Field(..., description="当天展示条数合计")
|
impressions: int = Field(..., description="当天展示条数合计")
|
||||||
revenue_yuan: float = Field(..., description="当天预估收益合计(元)")
|
revenue_yuan: float = Field(..., description="当天客户端预估收益合计(元;eCPM 折算)")
|
||||||
|
pangle_revenue_yuan: float | None = Field(
|
||||||
|
None, description="当天穿山甲后台预估收益(元;GroMore revenue);非全量视图/无数据为空"
|
||||||
|
)
|
||||||
|
pangle_api_revenue_yuan: float | None = Field(
|
||||||
|
None, description="当天穿山甲收益Api(元;GroMore api_revenue,更接近结算);未配/当天/无数据为空"
|
||||||
|
)
|
||||||
expected_coin: int = Field(..., description="当天应发金币合计")
|
expected_coin: int = Field(..., description="当天应发金币合计")
|
||||||
actual_coin: int = Field(..., description="当天实发金币合计")
|
actual_coin: int = Field(..., description="当天实发金币合计")
|
||||||
|
|
||||||
@@ -123,7 +129,21 @@ class AdRevenueReportOut(BaseModel):
|
|||||||
total: int = Field(..., description="广告事件总数(全量,不受分页影响;= 当前筛选下的分页总条数)")
|
total: int = Field(..., description="广告事件总数(全量,不受分页影响;= 当前筛选下的分页总条数)")
|
||||||
truncated: bool = Field(..., description="当前页之后是否还有更多事件(len(events) > offset + limit)")
|
truncated: bool = Field(..., description="当前页之后是否还有更多事件(len(events) > offset + limit)")
|
||||||
total_impressions: int = Field(..., description="全量展示条数合计")
|
total_impressions: int = Field(..., description="全量展示条数合计")
|
||||||
total_revenue_yuan: float = Field(..., description="全量收益合计(元)")
|
total_revenue_yuan: float = Field(..., description="全量客户端预估收益合计(元;eCPM 折算)")
|
||||||
|
total_pangle_revenue_yuan: float | None = Field(
|
||||||
|
None,
|
||||||
|
description="全量穿山甲后台预估收益合计(元;GroMore revenue)。穿山甲无用户/类型/场景维度,"
|
||||||
|
"仅「全量视图」(未按 user_id/ad_type/feed_scene 过滤)时有值,否则为 null",
|
||||||
|
)
|
||||||
|
total_pangle_api_revenue_yuan: float | None = Field(
|
||||||
|
None,
|
||||||
|
description="全量穿山甲收益Api合计(元;GroMore api_revenue,各 ADN 回传、更接近结算);"
|
||||||
|
"未配 Reporting / 查当天 / 非全量视图 时为 null",
|
||||||
|
)
|
||||||
|
pangle_revenue_available: bool = Field(
|
||||||
|
False,
|
||||||
|
description="本次结果是否带穿山甲后台收益(=全量视图且已同步到数据)。false 时前端「穿山甲收益」显示「-」",
|
||||||
|
)
|
||||||
total_expected_coin: int = Field(..., description="全量应发金币合计")
|
total_expected_coin: int = Field(..., description="全量应发金币合计")
|
||||||
total_actual_coin: int = Field(..., description="全量实发金币合计")
|
total_actual_coin: int = Field(..., description="全量实发金币合计")
|
||||||
mismatch_count: int = Field(..., description="应发≠实发的发奖条数(=0 说明全部按公式发放)")
|
mismatch_count: int = Field(..., description="应发≠实发的发奖条数(=0 说明全部按公式发放)")
|
||||||
|
|||||||
@@ -211,6 +211,41 @@ class Settings(BaseSettings):
|
|||||||
"""回调开关打开且至少配了一个验签密钥,才接受发奖回调。"""
|
"""回调开关打开且至少配了一个验签密钥,才接受发奖回调。"""
|
||||||
return bool(self.PANGLE_CALLBACK_ENABLED and self.pangle_reward_secrets)
|
return bool(self.PANGLE_CALLBACK_ENABLED and self.pangle_reward_secrets)
|
||||||
|
|
||||||
|
# ===== 穿山甲 GroMore 数据 API(报表收益拉取,T+1)=====
|
||||||
|
# ⚠️ 与上面发奖回调的 m-key 是【两套完全不同的凭证】:这三样在穿山甲后台
|
||||||
|
# 「接入中心 → GroMore-API → 聚合数据报告 API」文档页领取(user_id / role_id / Security Key),
|
||||||
|
# 仅用于按天拉 GroMore 收益报表(revenue 预估收益 + api_revenue 收益Api),不参与发奖。
|
||||||
|
# 该 API 只能查【GroMore 聚合代码位】的数据(=我们 useMediation 的口径),非穿山甲 SDK 数据;
|
||||||
|
# 且不提供用户/设备维度(官方明确),故收益只能落到 日期×代码位 汇总,不能挂到逐条事件。
|
||||||
|
# 子账号(role_id≠user_id)需主账号在「角色管理」授予「查看全部数据」权限,否则查不到
|
||||||
|
# ecpm/revenue(接口返回 118);role_id 填成与 user_id 一致 = 查主账号数据。
|
||||||
|
PANGLE_REPORT_USER_ID: int = 0 # 媒体账号 user_id
|
||||||
|
PANGLE_REPORT_ROLE_ID: int = 0 # 子账号 role_id(=user_id 时查主账号)
|
||||||
|
PANGLE_REPORT_SECURITY_KEY: str = "" # 该账号的 Security Key(secure_key,≠ 发奖 m-key)
|
||||||
|
# GroMore 聚合 AppId(报表 site_id 维度)→ 我们的应用环境。报表按 site_id 区分两个穿山甲应用,
|
||||||
|
# 用它把每行归到 prod(傻瓜比价正式)/ test(测试应用)。默认值取自现网两个应用,部署时按需覆盖。
|
||||||
|
PANGLE_REPORT_SITE_ID_PROD: str = "5830519" # 傻瓜比价正式应用 AppId
|
||||||
|
PANGLE_REPORT_SITE_ID_TEST: str = "5832303" # 测试应用 AppId
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pangle_report_configured(self) -> bool:
|
||||||
|
"""三样齐全(user_id / role_id / security_key)才能拉 GroMore 报表;缺任一 → 同步脚本 no-op。"""
|
||||||
|
return bool(
|
||||||
|
self.PANGLE_REPORT_USER_ID
|
||||||
|
and self.PANGLE_REPORT_ROLE_ID
|
||||||
|
and self.PANGLE_REPORT_SECURITY_KEY
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pangle_report_site_id_to_env(self) -> dict[str, str]:
|
||||||
|
"""site_id(穿山甲 AppId)→ app_env(prod/test);供同步脚本把报表行归到我们的应用。留空项忽略。"""
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
if self.PANGLE_REPORT_SITE_ID_PROD.strip():
|
||||||
|
out[self.PANGLE_REPORT_SITE_ID_PROD.strip()] = "prod"
|
||||||
|
if self.PANGLE_REPORT_SITE_ID_TEST.strip():
|
||||||
|
out[self.PANGLE_REPORT_SITE_ID_TEST.strip()] = "test"
|
||||||
|
return out
|
||||||
|
|
||||||
# ===== Pricebot 上游 (领券/比价业务透传目标) =====
|
# ===== Pricebot 上游 (领券/比价业务透传目标) =====
|
||||||
# pricebot-backend 默认跑在 8000。/api/v1/coupon/step 会透传到这里的 /api/coupon/step
|
# pricebot-backend 默认跑在 8000。/api/v1/coupon/step 会透传到这里的 /api/coupon/step
|
||||||
PRICEBOT_BASE_URL: str = "http://localhost:8000"
|
PRICEBOT_BASE_URL: str = "http://localhost:8000"
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
"""穿山甲 GroMore「聚合数据报告 API」客户端 —— 按天拉取收益报表(只读、T+1)。
|
||||||
|
|
||||||
|
⚠️ 这是 **GroMore 数据拉取 API**,与发奖回调验签(integrations/pangle.py 的 m-key)是
|
||||||
|
两套完全不同的凭证与用途。凭证在后台「接入中心 → GroMore-API → 聚合数据报告 API」领取:
|
||||||
|
媒体账号 user_id、子账号 role_id、Security Key(secure_key)。
|
||||||
|
|
||||||
|
鉴权(文档 v2.x「方法一」):
|
||||||
|
1. 去掉请求参数里的 `sign` 字段与值为空的字段;
|
||||||
|
2. 其余参数按 key 字典序升序,拼成 `k1=v1&k2=v2&...&kn=vn`;
|
||||||
|
3. 末尾直接拼接 security_key(无分隔符),对整串做 MD5,取 32 位小写十六进制 = sign。
|
||||||
|
签名有 3 分钟过期(天级用 `timestamp` 秒级时间戳;小时级才用 `current_time` 字符串)。
|
||||||
|
|
||||||
|
数据口径要点(来自官方文档):
|
||||||
|
- 只返回【GroMore 聚合代码位】在 GroMore 内的数据(=我们 useMediation 的口径),
|
||||||
|
查不到穿山甲 SDK 自身的数据;
|
||||||
|
- **不提供分用户/设备维度**(官方 FAQ 明确拒绝),最细到 日期×应用×代码位×广告源;
|
||||||
|
- `revenue` = 预估收益(元,所有 ADN 都有);`api_revenue` = 收益Api(各 ADN 经 Reporting
|
||||||
|
回传、按实时汇率折算账号币种,更接近结算),需后台为该 ADN 配置 Reporting 才有、且不支持当天;
|
||||||
|
- 「今天」与「今天以前」必须分开查;天级跨度 ≤ 1 个月、不早于 12 个月。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger("shagua.pangle_report")
|
||||||
|
|
||||||
|
HOST = "https://www.csjplatform.com"
|
||||||
|
# 天级收益报表(路径与文档代码示例一致;另有小时级 get_hour_report_data,本服务只用天级)。
|
||||||
|
DAILY_PATH = "/union_media/open/api/mediation/get_daily_income_report_data"
|
||||||
|
VERSION = "2.0"
|
||||||
|
SIGN_TYPE = "MD5"
|
||||||
|
PAGE_LIMIT = 5000 # 文档上限;一次尽量多取,减少翻页
|
||||||
|
DEFAULT_TIMEOUT = 30.0
|
||||||
|
_MAX_PAGES = 200 # 翻页安全阀(5000×200=100w 行,远超我们规模),防异常 has_next 死循环
|
||||||
|
|
||||||
|
|
||||||
|
class PangleReportError(Exception):
|
||||||
|
"""GroMore 报表接口调用失败(未配置 / 网络 / 业务码非 100)。"""
|
||||||
|
|
||||||
|
|
||||||
|
def build_sign(params: Mapping[str, Any], security_key: str) -> str:
|
||||||
|
"""按文档「方法一」生成 sign:去 sign/空值 → key 升序 → k=v& 拼接 → 末尾接 secure_key → MD5(32 位小写)。
|
||||||
|
|
||||||
|
与官方 Python/Java 示例逐字节一致(见 tests/test_pangle_report.py 的两个测试向量)。
|
||||||
|
"""
|
||||||
|
items = [
|
||||||
|
(str(k), str(v))
|
||||||
|
for k, v in params.items()
|
||||||
|
if k != "sign" and v is not None and str(v) != ""
|
||||||
|
]
|
||||||
|
items.sort(key=lambda kv: kv[0])
|
||||||
|
raw = "&".join(f"{k}={v}" for k, v in items)
|
||||||
|
return hashlib.md5((raw + security_key).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_daily_report(
|
||||||
|
*,
|
||||||
|
start_date: str,
|
||||||
|
end_date: str,
|
||||||
|
# ⚠️ 用 ad_unit_id(GroMore 广告位ID = 我们的 104xxx),不要用 code_id(底层各 ADN 代码位,对不上)
|
||||||
|
dimensions: str = "date,site_id,ad_unit_id",
|
||||||
|
metrics: str = "revenue,api_revenue,ecpm,imp_cnt",
|
||||||
|
site_ids: str | None = None,
|
||||||
|
code_ids: str | None = None,
|
||||||
|
os: str | None = None,
|
||||||
|
network: str | None = None,
|
||||||
|
time_zone: int = 8,
|
||||||
|
limit: int = PAGE_LIMIT,
|
||||||
|
timeout: float = DEFAULT_TIMEOUT,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""拉一个日期区间(北京时间,time_zone=8)的天级收益报表,自动翻页,返回 report_list 全量行。
|
||||||
|
|
||||||
|
每行是字符串字典(接口原值),形如
|
||||||
|
{"start_date": "2026-06-27", "site_id": "5830519", "ad_unit_id": "104142227",
|
||||||
|
"revenue": "1.23", "api_revenue": "1.05", "ecpm": "0.80", "imp_cnt": "1537", ...}。
|
||||||
|
业务码非 100 直接抛 PangleReportError(由调用方/脚本兜底,不静默吞)。
|
||||||
|
"""
|
||||||
|
if not settings.pangle_report_configured:
|
||||||
|
raise PangleReportError(
|
||||||
|
"PANGLE_REPORT_USER_ID / ROLE_ID / SECURITY_KEY 未配置,无法拉取 GroMore 报表"
|
||||||
|
)
|
||||||
|
|
||||||
|
base_params: dict[str, Any] = {
|
||||||
|
"user_id": settings.PANGLE_REPORT_USER_ID,
|
||||||
|
"role_id": settings.PANGLE_REPORT_ROLE_ID,
|
||||||
|
"version": VERSION,
|
||||||
|
"sign_type": SIGN_TYPE,
|
||||||
|
"start_date": start_date,
|
||||||
|
"end_date": end_date,
|
||||||
|
"dimensions": dimensions,
|
||||||
|
"metrics": metrics,
|
||||||
|
"time_zone": time_zone,
|
||||||
|
"limit": limit,
|
||||||
|
}
|
||||||
|
# 可选过滤(空则不传,避免进签名串)
|
||||||
|
for key, val in (
|
||||||
|
("site_ids", site_ids), ("code_ids", code_ids),
|
||||||
|
("os", os), ("network", network),
|
||||||
|
):
|
||||||
|
if val:
|
||||||
|
base_params[key] = val
|
||||||
|
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
offset = 0
|
||||||
|
try:
|
||||||
|
with httpx.Client(timeout=timeout) as client:
|
||||||
|
for _ in range(_MAX_PAGES):
|
||||||
|
params = dict(base_params)
|
||||||
|
params["offset"] = offset
|
||||||
|
# timestamp 每页临请求时取最新(3 分钟过期),并参与签名
|
||||||
|
params["timestamp"] = int(time.time())
|
||||||
|
params["sign"] = build_sign(params, settings.PANGLE_REPORT_SECURITY_KEY)
|
||||||
|
|
||||||
|
resp = client.get(HOST + DAILY_PATH, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
body = resp.json()
|
||||||
|
|
||||||
|
code = str(body.get("code"))
|
||||||
|
if code != "100":
|
||||||
|
raise PangleReportError(
|
||||||
|
f"GroMore 报表业务失败 code={code} message={body.get('message')!r} "
|
||||||
|
f"(101=验签失败/102=userid无效/107=无权限/118=无收益查看权限,详见文档状态码)"
|
||||||
|
)
|
||||||
|
|
||||||
|
data = body.get("data") or {}
|
||||||
|
page = data.get("report_list") or []
|
||||||
|
rows.extend(page)
|
||||||
|
|
||||||
|
# has_next=1 还有下一页;无该字段则按本页是否取满判断
|
||||||
|
has_next = str(data.get("has_next", "")) == "1"
|
||||||
|
if not page or not has_next:
|
||||||
|
break
|
||||||
|
offset += len(page)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
raise PangleReportError(f"GroMore 报表请求异常: {e}") from e
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"GroMore 天级报表拉取完成 %s~%s dims=%s 行数=%d", start_date, end_date, dimensions, len(rows)
|
||||||
|
)
|
||||||
|
return rows
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
"""所有 ORM model 必须在这里 import 一次,Alembic / metadata 才能扫到。"""
|
"""所有 ORM model 必须在这里 import 一次,Alembic / metadata 才能扫到。"""
|
||||||
from app.models.ad_ecpm import AdEcpmRecord # noqa: F401
|
from app.models.ad_ecpm import AdEcpmRecord # noqa: F401
|
||||||
from app.models.ad_feed_reward import AdFeedRewardRecord # noqa: F401
|
from app.models.ad_feed_reward import AdFeedRewardRecord # noqa: F401
|
||||||
|
from app.models.ad_pangle_revenue import AdPangleDailyRevenue # noqa: F401
|
||||||
from app.models.ad_reward import AdRewardRecord # noqa: F401
|
from app.models.ad_reward import AdRewardRecord # noqa: F401
|
||||||
from app.models.ad_watch_log import AdWatchLog # noqa: F401
|
from app.models.ad_watch_log import AdWatchLog # noqa: F401
|
||||||
from app.models.admin import AdminAuditLog, AdminUser # noqa: F401
|
from app.models.admin import AdminAuditLog, AdminUser # noqa: F401
|
||||||
@@ -19,7 +20,6 @@ from app.models.coupon_state import ( # noqa: F401
|
|||||||
CouponDailyCompletion,
|
CouponDailyCompletion,
|
||||||
CouponPromptEngagement,
|
CouponPromptEngagement,
|
||||||
)
|
)
|
||||||
from app.models.cps_order import CpsOrder # noqa: F401
|
|
||||||
from app.models.feedback import Feedback # noqa: F401
|
from app.models.feedback import Feedback # noqa: F401
|
||||||
from app.models.invite import InviteRelation # noqa: F401
|
from app.models.invite import InviteRelation # noqa: F401
|
||||||
from app.models.invite_fingerprint import InviteFingerprint # noqa: F401
|
from app.models.invite_fingerprint import InviteFingerprint # noqa: F401
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
"""穿山甲 GroMore 天级收益报表(后台结算口径,定时拉取入库)。
|
||||||
|
|
||||||
|
每行 = GroMore 数据 API 返回的一条「日期 × 应用 × 代码位」聚合收益(`integrations/pangle_report`
|
||||||
|
+ `scripts/sync_pangle_revenue` 落库)。**权威/预估收益的来源**,与 `ad_ecpm_record`(客户端自报
|
||||||
|
eCPM 折算的预估)互为对照:
|
||||||
|
|
||||||
|
- `revenue_yuan` ← 接口 `revenue`(预估收益,元;排序价×展示/1000,所有 ADN 都有);
|
||||||
|
- `api_revenue_yuan` ← 接口 `api_revenue`(收益Api,元;各 ADN 经 Reporting 回传、更接近结算;
|
||||||
|
未配置该 ADN 的 Reporting 或查当天时为空)。
|
||||||
|
|
||||||
|
⚠️ 穿山甲不提供分用户/设备维度,故本表最细只到 日期×应用×代码位,**无法挂到逐条广告事件**;
|
||||||
|
广告收益报表里只用于汇总/趋势级的「穿山甲后台收益」,不改逐条行的客户端预估。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class AdPangleDailyRevenue(Base):
|
||||||
|
__tablename__ = "ad_pangle_daily_revenue"
|
||||||
|
__table_args__ = (
|
||||||
|
# 一行 = (日期, 应用, 代码位, 广告源);adn 用 "" 表示「未分广告源、该代码位汇总」,
|
||||||
|
# 避免 NULL 在唯一约束里被视为各不相同导致 upsert 重复(SQLite/PG 行为一致)。
|
||||||
|
UniqueConstraint(
|
||||||
|
"report_date", "app_env", "our_code_id", "adn",
|
||||||
|
name="uq_ad_pangle_daily",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
# 北京时间日期串 'YYYY-MM-DD'(拉取时 time_zone=8),与 ad_ecpm_record.report_date 同口径,可直接 join。
|
||||||
|
report_date: Mapped[str] = mapped_column(String(10), index=True, nullable=False)
|
||||||
|
# 我们的应用环境:prod(傻瓜比价正式)/ test(测试);由 site_id 经 settings 映射而来。
|
||||||
|
app_env: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
# 原始 GroMore AppId(site_id),留痕便于排查映射。
|
||||||
|
site_id: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||||
|
# 广告位 ID(= 接口 ad_unit_id = 我们客户端配的 104xxx = ad_ecpm_record.our_code_id),join key。
|
||||||
|
# ⚠️ 不是接口的 code_id —— 那是底层各 ADN 的代码位(如 983674557),对不上我们的口径。
|
||||||
|
our_code_id: Mapped[str] = mapped_column(String(64), index=True, nullable=False)
|
||||||
|
# 广告源(接口 network 数字→名,如 pangle/gdt);"" = 未分广告源的代码位汇总行(当前默认口径)。
|
||||||
|
adn: Mapped[str] = mapped_column(String(16), nullable=False, default="")
|
||||||
|
# 预估收益(元)← 接口 revenue。
|
||||||
|
revenue_yuan: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||||
|
# 收益Api(元)← 接口 api_revenue;未配 Reporting / 当天 等情况接口不返回 → NULL。
|
||||||
|
api_revenue_yuan: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
# 预估 eCPM 原值(接口 ecpm,单位元/千次,**与客户端 getEcpm 的「分」不同**),参考用原样存。
|
||||||
|
ecpm: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||||
|
# 展示次数 ← 接口 imp_cnt。
|
||||||
|
impressions: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
# 货币类型(接口 currency,正常为 cny)。
|
||||||
|
currency: Mapped[str] = mapped_column(String(8), nullable=False, default="cny")
|
||||||
|
# 最近一次同步写入时间(同一行可被多次回补覆盖;T+1 数据穿山甲会订正)。
|
||||||
|
synced_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self) -> str: # pragma: no cover
|
||||||
|
return (
|
||||||
|
f"<AdPangleDailyRevenue {self.report_date} {self.app_env} "
|
||||||
|
f"code={self.our_code_id} revenue={self.revenue_yuan}>"
|
||||||
|
)
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
"""穿山甲 GroMore 天级收益 读写(`ad_pangle_daily_revenue` 表)。
|
||||||
|
|
||||||
|
`scripts/sync_pangle_revenue` 拉数后调 `upsert_daily_rows` 落库(同一(日期×应用×代码位×广告源)
|
||||||
|
幂等覆盖,T+1 订正可重跑);admin 广告收益报表调 `aggregate_by_date` 取「穿山甲后台收益」做
|
||||||
|
汇总/趋势级展示。穿山甲无用户维度,故这里不涉及 user_id。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, TypedDict
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models.ad_pangle_revenue import AdPangleDailyRevenue
|
||||||
|
|
||||||
|
# upsert 时可被覆盖更新的列(唯一键之外的业务列)。
|
||||||
|
_UPDATABLE = ("site_id", "revenue_yuan", "api_revenue_yuan", "ecpm", "impressions", "currency")
|
||||||
|
|
||||||
|
|
||||||
|
class PangleDateAgg(TypedDict):
|
||||||
|
date: str
|
||||||
|
revenue_yuan: float
|
||||||
|
api_revenue_yuan: float | None
|
||||||
|
impressions: int
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_daily_rows(db: Session, rows: list[dict[str, Any]]) -> dict[str, int]:
|
||||||
|
"""按唯一键 (report_date, app_env, our_code_id, adn) 逐行 upsert。
|
||||||
|
|
||||||
|
每个 row 须含:report_date, app_env, our_code_id;可选 adn(默认"")、site_id、
|
||||||
|
revenue_yuan、api_revenue_yuan、ecpm、impressions、currency。返回 {inserted, updated}。
|
||||||
|
规模很小(代码位数×天数),逐行 select-then-write 足够,且 SQLite/PG 通用。
|
||||||
|
"""
|
||||||
|
inserted = updated = 0
|
||||||
|
for row in rows:
|
||||||
|
adn = row.get("adn") or ""
|
||||||
|
existing = db.execute(
|
||||||
|
select(AdPangleDailyRevenue).where(
|
||||||
|
AdPangleDailyRevenue.report_date == row["report_date"],
|
||||||
|
AdPangleDailyRevenue.app_env == row["app_env"],
|
||||||
|
AdPangleDailyRevenue.our_code_id == row["our_code_id"],
|
||||||
|
AdPangleDailyRevenue.adn == adn,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
if existing is None:
|
||||||
|
db.add(AdPangleDailyRevenue(
|
||||||
|
report_date=row["report_date"],
|
||||||
|
app_env=row["app_env"],
|
||||||
|
site_id=row.get("site_id"),
|
||||||
|
our_code_id=row["our_code_id"],
|
||||||
|
adn=adn,
|
||||||
|
revenue_yuan=float(row.get("revenue_yuan") or 0.0),
|
||||||
|
api_revenue_yuan=row.get("api_revenue_yuan"),
|
||||||
|
ecpm=row.get("ecpm"),
|
||||||
|
impressions=int(row.get("impressions") or 0),
|
||||||
|
currency=row.get("currency") or "cny",
|
||||||
|
))
|
||||||
|
inserted += 1
|
||||||
|
else:
|
||||||
|
for col in _UPDATABLE:
|
||||||
|
if col in row:
|
||||||
|
setattr(existing, col, row[col])
|
||||||
|
updated += 1
|
||||||
|
db.commit()
|
||||||
|
return {"inserted": inserted, "updated": updated}
|
||||||
|
|
||||||
|
|
||||||
|
def aggregate_by_date(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
date_from: str,
|
||||||
|
date_to: str,
|
||||||
|
app_env: str | None = None,
|
||||||
|
our_code_id: str | None = None,
|
||||||
|
) -> list[PangleDateAgg]:
|
||||||
|
"""按日期汇总穿山甲收益(闭区间,北京时间),供报表趋势 + 合计。
|
||||||
|
|
||||||
|
revenue_yuan = Σrevenue;api_revenue_yuan = Σapi_revenue(SQL SUM 忽略 NULL,
|
||||||
|
全为空则返回 None,前端显示「-」)。可选按应用 / 代码位过滤。返回按日期升序。
|
||||||
|
"""
|
||||||
|
stmt = (
|
||||||
|
select(
|
||||||
|
AdPangleDailyRevenue.report_date,
|
||||||
|
func.sum(AdPangleDailyRevenue.revenue_yuan),
|
||||||
|
func.sum(AdPangleDailyRevenue.api_revenue_yuan),
|
||||||
|
func.sum(AdPangleDailyRevenue.impressions),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
AdPangleDailyRevenue.report_date >= date_from,
|
||||||
|
AdPangleDailyRevenue.report_date <= date_to,
|
||||||
|
)
|
||||||
|
.group_by(AdPangleDailyRevenue.report_date)
|
||||||
|
.order_by(AdPangleDailyRevenue.report_date)
|
||||||
|
)
|
||||||
|
if app_env is not None:
|
||||||
|
stmt = stmt.where(AdPangleDailyRevenue.app_env == app_env)
|
||||||
|
if our_code_id is not None:
|
||||||
|
stmt = stmt.where(AdPangleDailyRevenue.our_code_id == our_code_id)
|
||||||
|
|
||||||
|
out: list[PangleDateAgg] = []
|
||||||
|
for report_date, rev, api_rev, imp in db.execute(stmt).all():
|
||||||
|
out.append(PangleDateAgg(
|
||||||
|
date=report_date,
|
||||||
|
revenue_yuan=round(float(rev or 0.0), 6),
|
||||||
|
api_revenue_yuan=(round(float(api_rev), 6) if api_rev is not None else None),
|
||||||
|
impressions=int(imp or 0),
|
||||||
|
))
|
||||||
|
return out
|
||||||
@@ -46,7 +46,10 @@
|
|||||||
| `total` | int | 当前筛选下的**分页总条数**(全量,不受分页影响;= 前端分页器 total) |
|
| `total` | int | 当前筛选下的**分页总条数**(全量,不受分页影响;= 前端分页器 total) |
|
||||||
| `truncated` | bool | 当前页之后是否还有更多事件(`len(events) > offset + limit`) |
|
| `truncated` | bool | 当前页之后是否还有更多事件(`len(events) > offset + limit`) |
|
||||||
| `total_impressions` | int | 全量展示条数合计 |
|
| `total_impressions` | int | 全量展示条数合计 |
|
||||||
| `total_revenue_yuan` | float | 全量收益合计(元) |
|
| `total_revenue_yuan` | float | 全量**客户端预估**收益合计(元;eCPM 折算) |
|
||||||
|
| `total_pangle_revenue_yuan` | float \| null | 全量**穿山甲后台预估**收益合计(元;GroMore `revenue`)。穿山甲无用户/类型/场景维度,**仅全量视图**(未按 `user_id`/`ad_type`/`feed_scene` 过滤)有值,否则 `null` |
|
||||||
|
| `total_pangle_api_revenue_yuan` | float \| null | 全量**穿山甲收益Api**合计(元;GroMore `api_revenue`,各 ADN 回传更接近结算);未配 Reporting / 查当天 / 非全量视图为 `null` |
|
||||||
|
| `pangle_revenue_available` | bool | 本次是否带穿山甲后台收益(=全量视图且对应日期已同步到 `ad_pangle_daily_revenue`) |
|
||||||
| `total_expected_coin` | int | 全量应发金币合计 |
|
| `total_expected_coin` | int | 全量应发金币合计 |
|
||||||
| `total_actual_coin` | int | 全量实发金币合计 |
|
| `total_actual_coin` | int | 全量实发金币合计 |
|
||||||
| `mismatch_count` | int | 应发≠实发的组数(=0 说明全部按公式发放) |
|
| `mismatch_count` | int | 应发≠实发的组数(=0 说明全部按公式发放) |
|
||||||
@@ -57,7 +60,9 @@
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `date` | string | 北京时间 `YYYY-MM-DD` |
|
| `date` | string | 北京时间 `YYYY-MM-DD` |
|
||||||
| `impressions` | int | 当天展示条数合计 |
|
| `impressions` | int | 当天展示条数合计 |
|
||||||
| `revenue_yuan` | float | 当天预估收益合计(元) |
|
| `revenue_yuan` | float | 当天客户端预估收益合计(元;eCPM 折算) |
|
||||||
|
| `pangle_revenue_yuan` | float \| null | 当天穿山甲后台预估收益(元;GroMore `revenue`);非全量视图 / 无数据为 `null` |
|
||||||
|
| `pangle_api_revenue_yuan` | float \| null | 当天穿山甲收益Api(元;GroMore `api_revenue`);未配 / 当天 / 无数据为 `null` |
|
||||||
| `expected_coin` | int | 当天应发金币合计 |
|
| `expected_coin` | int | 当天应发金币合计 |
|
||||||
| `actual_coin` | int | 当天实发金币合计 |
|
| `actual_coin` | int | 当天实发金币合计 |
|
||||||
|
|
||||||
@@ -125,5 +130,6 @@
|
|||||||
- **展示 vs 发奖分离**:信息流轮播一会话可展示多条(都计入 `impressions`),但发奖仍按现规则(一会话发一次),`coin` 不因展示条数变化——这是有意设计(用户中途关只记展示不发奖)。
|
- **展示 vs 发奖分离**:信息流轮播一会话可展示多条(都计入 `impressions`),但发奖仍按现规则(一会话发一次),`coin` 不因展示条数变化——这是有意设计(用户中途关只记展示不发奖)。
|
||||||
- **历史 Draw 不可拆**:迁移(Draw→普通信息流)前,Draw 发奖混在 `ad_feed_reward_record` 且无类型标记,金币侧统一记 `feed`;迁移后 Draw 不再产生新数据。展示侧 `ad_type` 由客户端上报区分,故 `draw` 桶基本为空。
|
- **历史 Draw 不可拆**:迁移(Draw→普通信息流)前,Draw 发奖混在 `ad_feed_reward_record` 且无类型标记,金币侧统一记 `feed`;迁移后 Draw 不再产生新数据。展示侧 `ad_type` 由客户端上报区分,故 `draw` 桶基本为空。
|
||||||
- **来源字段从上线起齐全**:`app_env`/`our_code_id` 是本期新增列,历史记录为 NULL(报表来源列留空)。
|
- **来源字段从上线起齐全**:`app_env`/`our_code_id` 是本期新增列,历史记录为 NULL(报表来源列留空)。
|
||||||
- **收益是预估**:基于客户端上报的 eCPM,非穿山甲后台结算值;以后台报表为结算权威。
|
- **逐条/明细的收益是预估**:`items[].revenue_yuan` 基于客户端上报的 eCPM 折算,非穿山甲后台结算值。
|
||||||
|
- **穿山甲后台收益(汇总/趋势级)**:`total_pangle_revenue_yuan`(预估 `revenue`)与 `total_pangle_api_revenue_yuan`(收益Api `api_revenue`,更接近结算)来自穿山甲 **GroMore 数据 API**(`integrations/pangle_report` + `scripts/sync_pangle_revenue` 按天 T+1 拉取入 [ad_pangle_daily_revenue](../database/ad_pangle_daily_revenue.md))。穿山甲**不提供分用户/设备/类型/场景维度**(官方明确),最细到 日期×应用×代码位,故只用于汇总与按天趋势的对照,**不挂到逐条事件行**;且仅在全量视图(未按 user/类型/场景过滤)展示。配置见 `.env` 的 `PANGLE_REPORT_*`。
|
||||||
- **对账聚合级 + 逐条下钻**:行级 `matched` 给出该组(用户×类型×应用×代码位)应发是否==实发;**展开 `records` 即可看该组逐条明细**(eCPM/因子1/份数/LT/因子2/应发/实发/一致)定位到具体记录。独立逐条审计接口 [admin-ad-coin-audit](./admin-ad-coin-audit.md) 仍保留(同一复算口径,可全局按场景/只看不符筛选)。
|
- **对账聚合级 + 逐条下钻**:行级 `matched` 给出该组(用户×类型×应用×代码位)应发是否==实发;**展开 `records` 即可看该组逐条明细**(eCPM/因子1/份数/LT/因子2/应发/实发/一致)定位到具体记录。独立逐条审计接口 [admin-ad-coin-audit](./admin-ad-coin-audit.md) 仍保留(同一复算口径,可全局按场景/只看不符筛选)。
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# ad_pangle_daily_revenue — 穿山甲 GroMore 天级收益
|
||||||
|
|
||||||
|
> 模型:[app/models/ad_pangle_revenue.py](../../app/models/ad_pangle_revenue.py) | 读写:[app/repositories/ad_pangle_revenue.py](../../app/repositories/ad_pangle_revenue.py) | 拉取:[app/integrations/pangle_report.py](../../app/integrations/pangle_report.py) + [scripts/sync_pangle_revenue.py](../../scripts/sync_pangle_revenue.py)
|
||||||
|
|
||||||
|
存放从穿山甲 **GroMore 数据 API**(聚合数据报告 API,天级、T+1)按天拉取的收益,供 admin
|
||||||
|
[广告收益报表](../api/admin-ad-revenue-report.md) 的「穿山甲后台收益」做汇总/趋势级展示,
|
||||||
|
与客户端自报 eCPM 折算的预估互为对照(看 gap)。
|
||||||
|
|
||||||
|
**粒度 = 日期 × 应用(app_env) × 代码位(our_code_id) × 广告源(adn)**。穿山甲**不提供分用户/设备维度**
|
||||||
|
(官方明确),故本表无 `user_id`,也无法挂到逐条广告事件;报表逐条行仍用客户端预估。
|
||||||
|
|
||||||
|
## 数据流
|
||||||
|
|
||||||
|
- **写**:`scripts/sync_pangle_revenue`(线上每天 ~10:30 由 systemd timer 跑)调
|
||||||
|
`pangle_report.fetch_daily_report` 拉昨天(可 `--days N` 回补),维度 `date,site_id,ad_unit_id`,
|
||||||
|
映射 `site_id→app_env`、`ad_unit_id→our_code_id` 后 `upsert_daily_rows` 落库。按唯一键幂等,T+1 订正可重跑。
|
||||||
|
⚠️ 用 `ad_unit_id`(广告位ID=104xxx)而非 `code_id`:实测 `code_id` 返回底层各 ADN 代码位(如 `983674557`),对不上我们的口径。
|
||||||
|
- **读**:`ad_pangle_revenue.aggregate_by_date` 按日期汇总 → admin 报表
|
||||||
|
`total_pangle_revenue_yuan` / `total_pangle_api_revenue_yuan` / `daily[].pangle_*`。
|
||||||
|
|
||||||
|
## 字段
|
||||||
|
|
||||||
|
| 列 | 类型 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | int PK | |
|
||||||
|
| `report_date` | str(10) idx | 北京时间 `YYYY-MM-DD`(拉取 `time_zone=8`),与 `ad_ecpm_record.report_date` 同口径可 join |
|
||||||
|
| `app_env` | str(16) | `prod`(傻瓜比价正式)/ `test`(测试);由 `site_id` 经 `settings.pangle_report_site_id_to_env` 映射;未映射记 `site:<id>` |
|
||||||
|
| `site_id` | str(32) \| null | 原始 GroMore AppId(留痕) |
|
||||||
|
| `our_code_id` | str(64) idx | 广告位 ID(= 接口 `ad_unit_id` = 客户端配的 104xxx = `ad_ecpm_record.our_code_id`),join key。⚠️ **非** `code_id`(那是底层各 ADN 代码位,对不上);`-1` 为 GroMore 未归因桶 |
|
||||||
|
| `adn` | str(16) | 广告源(接口 `network`);当前默认口径不分广告源(按 `ad_unit_id` 汇总),统一 `""` |
|
||||||
|
| `revenue_yuan` | float | 预估收益(元)← 接口 `revenue`(排序价×展示/1000,所有 ADN) |
|
||||||
|
| `api_revenue_yuan` | float \| null | 收益Api(元)← 接口 `api_revenue`(各 ADN 经 Reporting 回传、更接近结算);未配 / 当天为 `null` |
|
||||||
|
| `ecpm` | str(32) \| null | 预估 eCPM 原值(接口 `ecpm`,单位**元/千次**,与客户端 getEcpm 的「分」不同) |
|
||||||
|
| `impressions` | int | 展示次数 ← 接口 `imp_cnt` |
|
||||||
|
| `currency` | str(8) | 货币(接口 `currency`,正常 `cny`) |
|
||||||
|
| `synced_at` | datetime | 最近同步写入时间(可被回补覆盖) |
|
||||||
|
|
||||||
|
唯一约束 `uq_ad_pangle_daily (report_date, app_env, our_code_id, adn)`;`adn` 用 `""` 而非
|
||||||
|
NULL,避免 NULL 在唯一约束里互不相同导致 upsert 重复。
|
||||||
|
|
||||||
|
## 局限
|
||||||
|
|
||||||
|
- **T+1**:当天数据穿山甲不出 `api_revenue`,`revenue` 也次日才稳;报表「今天」此两列多为空。
|
||||||
|
- **`revenue` vs `api_revenue`**:前者所有 ADN 都有(预估口径);后者需后台为该 ADN 配置 Reporting
|
||||||
|
才回传,部分 ADN 可能长期为空。结算仍以各 ADN 结算单 / 穿山甲后台为准。
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
"""每日拉取穿山甲 GroMore 天级收益报表入库(供 admin 广告收益报表的「穿山甲后台收益」)。
|
||||||
|
|
||||||
|
GroMore 数据 API 为 T+1:次日穿山甲约 10:00 出数。建议线上每天 ~10:30 由 systemd timer 跑一次
|
||||||
|
(默认拉【昨天】);穿山甲对历史数据可能订正,故支持回补近 N 天(幂等 upsert,重跑无害)。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
python -m scripts.sync_pangle_revenue # 拉昨天(北京时间)
|
||||||
|
python -m scripts.sync_pangle_revenue --days 3 # 回补昨天起往前 3 天
|
||||||
|
python -m scripts.sync_pangle_revenue --date 2026-06-27 # 指定单天
|
||||||
|
python -m scripts.sync_pangle_revenue --start 2026-06-01 --end 2026-06-27 # 指定区间
|
||||||
|
|
||||||
|
约束(来自官方文档):天级跨度 ≤ 1 个月、不早于 12 个月;「今天」与「今天以前」必须分开查,
|
||||||
|
本脚本默认只拉昨天及更早,不混查今天。未配置 PANGLE_REPORT_* 时直接 no-op 退出。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
from datetime import date, timedelta
|
||||||
|
|
||||||
|
# Windows 控制台按 UTF-8 输出中文/¥
|
||||||
|
try:
|
||||||
|
sys.stdout.reconfigure(encoding="utf-8") # type: ignore[attr-defined]
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.core.rewards import cn_today
|
||||||
|
from app.db.session import SessionLocal
|
||||||
|
from app.integrations import pangle_report
|
||||||
|
from app.integrations.pangle_report import PangleReportError
|
||||||
|
from app.repositories import ad_pangle_revenue as repo
|
||||||
|
|
||||||
|
# 拉取维度固定:date×site_id×ad_unit_id。⚠️ 用 ad_unit_id(GroMore 广告位ID = 我们客户端配的 104xxx
|
||||||
|
# = ad_ecpm_record.our_code_id)而**非** code_id —— 实测 code_id 返回的是底层各 ADN 的代码位
|
||||||
|
# (如 983674557 / youlianghui_xinxiliu),对不上我们的口径。指标取预估 + Api 两个收益。
|
||||||
|
_DIMENSIONS = "date,site_id,ad_unit_id"
|
||||||
|
_METRICS = "revenue,api_revenue,ecpm,imp_cnt"
|
||||||
|
|
||||||
|
|
||||||
|
def _to_float(v: object) -> float | None:
|
||||||
|
"""接口数值字段都是字符串;空/非法 → None(api_revenue 缺失时即为 None,前端显示「-」)。"""
|
||||||
|
if v is None or str(v).strip() == "":
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(str(v).strip())
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _row_date(row: dict) -> str | None:
|
||||||
|
"""报表行的日期:优先 date 维度,回退 start_date;取前 10 位 'YYYY-MM-DD'。"""
|
||||||
|
raw = row.get("date") or row.get("start_date") or ""
|
||||||
|
raw = str(raw).strip()
|
||||||
|
return raw[:10] if len(raw) >= 10 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize(rows: list[dict]) -> tuple[list[dict], int, set[str]]:
|
||||||
|
"""接口原始行 → 入库行;返回 (入库行, 跳过数, 未映射 site_id 集合)。
|
||||||
|
|
||||||
|
site_id 经 settings 映射成 app_env(prod/test);未映射的仍入库(app_env=site:<id>)但提示,
|
||||||
|
避免静默丢收益。缺 code_id / 日期的行跳过。
|
||||||
|
"""
|
||||||
|
site_map = settings.pangle_report_site_id_to_env
|
||||||
|
out: list[dict] = []
|
||||||
|
skipped = 0
|
||||||
|
unmapped: set[str] = set()
|
||||||
|
for row in rows:
|
||||||
|
rdate = _row_date(row)
|
||||||
|
# 用 ad_unit_id(GroMore 广告位ID = 我们的 104xxx),不是 code_id(底层 ADN 代码位)
|
||||||
|
code_id = str(row.get("ad_unit_id") or "").strip()
|
||||||
|
if not rdate or not code_id:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
site_id = str(row.get("site_id") or "").strip()
|
||||||
|
app_env = site_map.get(site_id)
|
||||||
|
if app_env is None:
|
||||||
|
app_env = f"site:{site_id}" if site_id else "unknown"
|
||||||
|
if site_id:
|
||||||
|
unmapped.add(site_id)
|
||||||
|
out.append({
|
||||||
|
"report_date": rdate,
|
||||||
|
"app_env": app_env,
|
||||||
|
"site_id": site_id or None,
|
||||||
|
"our_code_id": code_id,
|
||||||
|
"adn": "",
|
||||||
|
"revenue_yuan": _to_float(row.get("revenue")) or 0.0,
|
||||||
|
"api_revenue_yuan": _to_float(row.get("api_revenue")),
|
||||||
|
"ecpm": (str(row.get("ecpm")).strip() or None) if row.get("ecpm") is not None else None,
|
||||||
|
"impressions": int(_to_float(row.get("imp_cnt")) or 0),
|
||||||
|
"currency": "cny",
|
||||||
|
})
|
||||||
|
return out, skipped, unmapped
|
||||||
|
|
||||||
|
|
||||||
|
def sync_range(start_date: str, end_date: str) -> None:
|
||||||
|
if not settings.pangle_report_configured:
|
||||||
|
print("PANGLE_REPORT_* 未配置,跳过(no-op)。需在后台「接入中心 → GroMore-API」领取 "
|
||||||
|
"user_id/role_id/Security Key 并填入 .env。")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"拉取 GroMore 天级收益 {start_date} ~ {end_date} (dims={_DIMENSIONS}) ...")
|
||||||
|
try:
|
||||||
|
raw = pangle_report.fetch_daily_report(
|
||||||
|
start_date=start_date, end_date=end_date,
|
||||||
|
dimensions=_DIMENSIONS, metrics=_METRICS, time_zone=8,
|
||||||
|
)
|
||||||
|
except PangleReportError as e:
|
||||||
|
print(f"❌ 拉取失败:{e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
rows, skipped, unmapped = _normalize(raw)
|
||||||
|
if unmapped:
|
||||||
|
print(f"⚠️ 有未映射 site_id(收益仍入库,app_env=site:<id>):{sorted(unmapped)};"
|
||||||
|
f"如需归入 prod/test,补 PANGLE_REPORT_SITE_ID_* 配置。")
|
||||||
|
with SessionLocal() as db:
|
||||||
|
stats = repo.upsert_daily_rows(db, rows)
|
||||||
|
total_rev = round(sum(r["revenue_yuan"] for r in rows), 4)
|
||||||
|
print(f"✅ 完成:接口 {len(raw)} 行 → 入库 {len(rows)} 行(跳过 {skipped}),"
|
||||||
|
f"新增 {stats['inserted']} / 更新 {stats['updated']};预估收益合计 ¥{total_rev}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser(description="拉取穿山甲 GroMore 天级收益报表入库")
|
||||||
|
ap.add_argument("--date", help="指定单天 YYYY-MM-DD")
|
||||||
|
ap.add_argument("--start", help="区间起始 YYYY-MM-DD(与 --end 同用)")
|
||||||
|
ap.add_argument("--end", help="区间结束 YYYY-MM-DD(与 --start 同用)")
|
||||||
|
ap.add_argument("--days", type=int, help="回补:昨天起往前 N 天(含昨天)")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
yesterday = cn_today() - timedelta(days=1)
|
||||||
|
if args.date:
|
||||||
|
start = end = args.date
|
||||||
|
elif args.start and args.end:
|
||||||
|
start, end = args.start, args.end
|
||||||
|
elif args.days:
|
||||||
|
start = (yesterday - timedelta(days=args.days - 1)).isoformat()
|
||||||
|
end = yesterday.isoformat()
|
||||||
|
else:
|
||||||
|
start = end = yesterday.isoformat()
|
||||||
|
|
||||||
|
# 防呆:跨度 > 31 天直接拒(接口上限 1 个月,超了会返回 114)
|
||||||
|
if (date.fromisoformat(end) - date.fromisoformat(start)).days > 31:
|
||||||
|
print("❌ 区间跨度超过 31 天(接口上限 1 个月),请分批拉。")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
sync_range(start, end)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""穿山甲 GroMore 数据 API 客户端测试。
|
||||||
|
|
||||||
|
重点锚定**签名算法**:用官方文档 Python / Java 示例里给出的「参数 + secure_key → sign」两个
|
||||||
|
测试向量做断言,保证我们的 build_sign 与穿山甲服务端逐字节一致(签错=接口直接 101 验签失败)。
|
||||||
|
另测拉取的翻页、业务码非 100 抛错、响应解析。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.integrations import pangle_report
|
||||||
|
from app.integrations.pangle_report import PangleReportError, build_sign
|
||||||
|
|
||||||
|
# ---- 签名:官方文档测试向量(完整 key 与期望 sign 来自文档代码注释)----
|
||||||
|
|
||||||
|
def test_build_sign_matches_doc_python_vector():
|
||||||
|
params = {
|
||||||
|
"current_time": "2022-01-06 16:10:54",
|
||||||
|
"sign_type": "MD5",
|
||||||
|
"version": "2.0",
|
||||||
|
"user_id": 459,
|
||||||
|
"role_id": 459,
|
||||||
|
}
|
||||||
|
sign = build_sign(params, "7f2ddb34d999ea61c00246ea3971529c")
|
||||||
|
assert sign == "57b9fed9b6c09cefdced7fa770fe341d"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_sign_matches_doc_java_vector():
|
||||||
|
params = {
|
||||||
|
"user_id": 459,
|
||||||
|
"role_id": 459,
|
||||||
|
"version": "2.0",
|
||||||
|
"sign_type": "MD5",
|
||||||
|
"current_time": "2022-01-06 16:05:49",
|
||||||
|
}
|
||||||
|
sign = build_sign(params, "1c589c70f7bb2746027ce90c33d2544c")
|
||||||
|
assert sign == "4d47b3a7679448c3da5c1d05e5291533"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_sign_drops_sign_and_empty_fields():
|
||||||
|
"""sign 字段与空值字段不参与签名:带上它们应得到与不带时相同的 sign。"""
|
||||||
|
base = {"version": "2.0", "user_id": 459, "role_id": 459, "sign_type": "MD5"}
|
||||||
|
with_noise = {**base, "sign": "deadbeef", "code_ids": "", "os": None}
|
||||||
|
assert build_sign(with_noise, "key123") == build_sign(base, "key123")
|
||||||
|
|
||||||
|
|
||||||
|
# ---- 拉取:伪造 httpx 客户端,验证翻页 / 解析 / 业务码 ----
|
||||||
|
|
||||||
|
class _FakeResp:
|
||||||
|
def __init__(self, payload: dict):
|
||||||
|
self._payload = payload
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def json(self) -> dict:
|
||||||
|
return self._payload
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeClient:
|
||||||
|
"""按 offset 返回分页响应;记录每次请求的 params 供断言。"""
|
||||||
|
|
||||||
|
def __init__(self, pages: list[dict]):
|
||||||
|
self._pages = pages
|
||||||
|
self.calls: list[dict] = []
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get(self, url: str, params: dict):
|
||||||
|
self.calls.append(params)
|
||||||
|
# 第 N 次调用取第 N 页(has_next 驱动翻页);超出用最后一页兜底
|
||||||
|
page = self._pages[min(len(self.calls) - 1, len(self._pages) - 1)]
|
||||||
|
return _FakeResp(page)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def _configured(monkeypatch):
|
||||||
|
monkeypatch.setattr(settings, "PANGLE_REPORT_USER_ID", 459)
|
||||||
|
monkeypatch.setattr(settings, "PANGLE_REPORT_ROLE_ID", 459)
|
||||||
|
monkeypatch.setattr(settings, "PANGLE_REPORT_SECURITY_KEY", "key123")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_daily_report_paginates_and_parses(monkeypatch, _configured):
|
||||||
|
page1 = {
|
||||||
|
"code": "100", "message": "",
|
||||||
|
"data": {
|
||||||
|
"has_next": "1", "currency": "cny", "total": 3,
|
||||||
|
"report_list": [
|
||||||
|
{"start_date": "2026-06-27", "site_id": "5830519", "ad_unit_id": "104142227",
|
||||||
|
"revenue": "1.23", "api_revenue": "1.05", "ecpm": "0.80", "imp_cnt": "1537"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
page2 = {
|
||||||
|
"code": "100", "message": "",
|
||||||
|
"data": {
|
||||||
|
"has_next": "0", "currency": "cny", "total": 3,
|
||||||
|
"report_list": [
|
||||||
|
{"start_date": "2026-06-27", "site_id": "5832303", "ad_unit_id": "104137445",
|
||||||
|
"revenue": "0.00", "api_revenue": "", "ecpm": "0", "imp_cnt": "12"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fake = _FakeClient([page1, page2])
|
||||||
|
monkeypatch.setattr(pangle_report.httpx, "Client", lambda *a, **k: fake)
|
||||||
|
|
||||||
|
rows = pangle_report.fetch_daily_report(start_date="2026-06-27", end_date="2026-06-27")
|
||||||
|
|
||||||
|
assert len(rows) == 2 # 两页都被取到(has_next 驱动翻页)
|
||||||
|
assert rows[0]["ad_unit_id"] == "104142227"
|
||||||
|
assert rows[1]["api_revenue"] == ""
|
||||||
|
# 每次请求都带上签名与必填鉴权参数
|
||||||
|
assert "sign" in fake.calls[0]
|
||||||
|
assert fake.calls[0]["user_id"] == 459 and fake.calls[0]["version"] == "2.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_daily_report_raises_on_business_error(monkeypatch, _configured):
|
||||||
|
fail = {"code": "118", "message": "no revenue permission", "data": {}}
|
||||||
|
monkeypatch.setattr(
|
||||||
|
pangle_report.httpx, "Client", lambda *a, **k: _FakeClient([fail])
|
||||||
|
)
|
||||||
|
with pytest.raises(PangleReportError):
|
||||||
|
pangle_report.fetch_daily_report(start_date="2026-06-27", end_date="2026-06-27")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_daily_report_no_config_raises(monkeypatch):
|
||||||
|
monkeypatch.setattr(settings, "PANGLE_REPORT_USER_ID", 0)
|
||||||
|
with pytest.raises(PangleReportError):
|
||||||
|
pangle_report.fetch_daily_report(start_date="2026-06-27", end_date="2026-06-27")
|
||||||
Reference in New Issue
Block a user