Compare commits

...

2 Commits

Author SHA1 Message Date
pure ea3cbc5d82 Merge origin/main into feat/price-observation-pipeline
解决 app/core/config.py 冲突:同时保留 main 的 PRICEBOT_INSTANCES 多实例透传
(pricebot_instances 属性)与本分支的 INTERNAL_API_SECRET(价格观测内部端点密钥)。
.env.example 自动合并(两段并存)。main 未新增 alembic 迁移,price_observation_table
仍是唯一 head;app/main.py 未被 main 改动,内部路由注册不受影响。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-07 03:31:04 +08:00
pure 14b1e2b727 feat(price-observation): 价格观测资产表 + pricebot 内部上报端点
比价沉淀的价格事实(平台/门店/菜篮/到手价/时间/来源)无条件落库,server 侧存储、
与登录无关,独立于用户视角的 comparison_record。是未来"别人查过同店→秒回价格"
价格大数据的源头(先存数据,用法后续)。

- models/price_observation.py: 新增 price_observation 表。结构化列(平台/门店/价格/
  红包/地理/来源)+ dishes/attrs 两个 JSONB 兜底列;(trace_id,platform,scope) 幂等
  唯一约束;observed_at/store_name/geohash/source_device_id/source_user_id 等索引
- alembic/versions/price_observation_table.py: 建表迁移,down_revision=wx_transfer_auth
  (已核为当前唯一 head,无多 head 风险)
- schemas/ + repositories/price_observation.py: 批量收发模型 + 跨方言(PG/SQLite)幂等
  批量插入(先查 trace 已有 (platform,scope) 只插新的,并发撞唯一约束则回滚跳过)
- api/internal/price.py: 新增 POST /internal/price-observation,X-Internal-Secret 头校验
  (常量时间比较,未配密钥→503),server→server 专用,不走用户 JWT
- core/config.py + main.py: 新增 INTERNAL_API_SECRET 配置 + 注册内部路由
- .env.example: INTERNAL_API_SECRET(须与 pricebot 侧同值,留空=内部写端点关闭)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-07 02:11:30 +08:00
10 changed files with 401 additions and 0 deletions
+7
View File
@@ -54,6 +54,13 @@ PRICEBOT_REQUEST_TIMEOUT_SEC=30
# 比价(intent/recognize + price/step)透传超时:大上下文 LLM + 逐帧 LLM,给 60s
PRICEBOT_COMPARE_TIMEOUT_SEC=60
# ===== 内部端点 (server→server, pricebot 上报价格观测) =====
# pricebot 比价 done 后把各平台到手价 POST 到本服务 /internal/price-observation 落库
# (price_observation 表,比价资产沉淀层),靠共享密钥头 X-Internal-Secret 校验。
# 必须与 pricebot 侧的 INTERNAL_API_SECRET **同值**;留空 = 内部写端点关闭(返 503)。
# 启用前两边都填同一高熵串:python -c "import secrets; print(secrets.token_urlsafe(48))"
INTERNAL_API_SECRET=
# ===== CORS =====
# 逗号分隔,生产留空(只让 app 调,不开放 web)。本地开发可加 http://localhost:5173 之类
CORS_ALLOW_ORIGINS=
@@ -0,0 +1,74 @@
"""price_observation table (价格观测:比价沉淀的价格资产层,server 侧无条件落库)
Revision ID: price_observation_table
Revises: wx_transfer_auth
Create Date: 2026-06-07 00:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'price_observation_table'
down_revision: Union[str, Sequence[str], None] = 'wx_transfer_auth'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
'price_observation',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('observed_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
sa.Column('trace_id', sa.String(length=64), nullable=False),
sa.Column('business_type', sa.String(length=16), nullable=False),
sa.Column('platform', sa.String(length=32), nullable=False),
sa.Column('platform_store_id', sa.String(length=64), nullable=True),
sa.Column('store_name', sa.String(length=128), nullable=True),
sa.Column('city', sa.String(length=64), nullable=True),
sa.Column('geohash', sa.String(length=16), nullable=True),
sa.Column('lng', sa.Float(), nullable=True),
sa.Column('lat', sa.Float(), nullable=True),
sa.Column('is_source', sa.Boolean(), nullable=False),
sa.Column('scope', sa.String(length=16), nullable=False),
sa.Column('price_cents', sa.Integer(), nullable=True),
sa.Column('coupon_saved_cents', sa.Integer(), nullable=True),
sa.Column('coupon_name', sa.String(length=64), nullable=True),
sa.Column('store_closed', sa.String(length=32), nullable=True),
sa.Column('rank', sa.Integer(), nullable=True),
# PG 上为 JSONB,其它(SQLite)为 JSON——与模型层 with_variant 对齐
sa.Column('dishes', sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), 'postgresql'), nullable=True),
sa.Column('attrs', sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), 'postgresql'), nullable=True),
sa.Column('source_device_id', sa.String(length=64), nullable=True),
sa.Column('source_user_id', sa.Integer(), nullable=True),
sa.Column('confidence', sa.Float(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('trace_id', 'platform', 'scope', name='uq_price_obs_trace_platform_scope'),
)
with op.batch_alter_table('price_observation', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_price_observation_observed_at'), ['observed_at'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_trace_id'), ['trace_id'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_business_type'), ['business_type'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_platform'), ['platform'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_store_name'), ['store_name'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_geohash'), ['geohash'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_source_device_id'), ['source_device_id'], unique=False)
batch_op.create_index(batch_op.f('ix_price_observation_source_user_id'), ['source_user_id'], unique=False)
def downgrade() -> None:
with op.batch_alter_table('price_observation', schema=None) as batch_op:
batch_op.drop_index(batch_op.f('ix_price_observation_source_user_id'))
batch_op.drop_index(batch_op.f('ix_price_observation_source_device_id'))
batch_op.drop_index(batch_op.f('ix_price_observation_geohash'))
batch_op.drop_index(batch_op.f('ix_price_observation_store_name'))
batch_op.drop_index(batch_op.f('ix_price_observation_platform'))
batch_op.drop_index(batch_op.f('ix_price_observation_business_type'))
batch_op.drop_index(batch_op.f('ix_price_observation_trace_id'))
batch_op.drop_index(batch_op.f('ix_price_observation_observed_at'))
op.drop_table('price_observation')
+1
View File
@@ -0,0 +1 @@
"""内部(server→server)端点。不对客户端开放,靠共享密钥头校验,不走用户 JWT。"""
+63
View File
@@ -0,0 +1,63 @@
"""价格观测内部上报端点(pricebot → app-server)。
pricebot 在比价 done 帧把整批价格事实 POST 到这里落库。**不是给客户端的接口**:
不走用户 JWT,靠 server 间共享密钥头 `X-Internal-Secret` 校验(== settings.INTERNAL_API_SECRET)。
密钥未配置(默认空)时直接 503,避免裸奔的写端点。
与不鉴权的比价透传(compare.py / coupon.py)无关:那两个是客户端→server 的透传壳,
这个是 server→server 的内部写库。
"""
from __future__ import annotations
import hmac
import logging
from typing import Annotated
from fastapi import APIRouter, Header, HTTPException, status
from app.api.deps import DbSession
from app.core.config import settings
from app.repositories import price_observation as repo
from app.schemas.price_observation import (
PriceObservationBatchIn,
PriceObservationBatchOut,
)
logger = logging.getLogger("shagua.internal.price")
router = APIRouter(prefix="/internal", tags=["internal"])
def _check_secret(x_internal_secret: str | None) -> None:
"""共享密钥校验。未配置 → 503(挡住裸奔写端点);不匹配 → 401(常量时间比较)。"""
configured = settings.INTERNAL_API_SECRET
if not configured:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="internal api not configured",
)
if not x_internal_secret or not hmac.compare_digest(x_internal_secret, configured):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="invalid internal secret",
)
@router.post(
"/price-observation",
response_model=PriceObservationBatchOut,
summary="价格观测内部上报(pricebot→app-server,落 price_observation)",
)
def report_price_observation(
payload: PriceObservationBatchIn,
db: DbSession,
x_internal_secret: Annotated[str | None, Header()] = None,
) -> PriceObservationBatchOut:
_check_secret(x_internal_secret)
inserted, skipped = repo.insert_batch(db, payload)
logger.info(
"price_observation trace=%s inserted=%d skipped=%d device=%s user=%s",
payload.trace_id, inserted, skipped,
payload.source_device_id, payload.source_user_id,
)
return PriceObservationBatchOut(inserted=inserted, skipped=skipped)
+6
View File
@@ -157,6 +157,12 @@ class Settings(BaseSettings):
return [self.PRICEBOT_BASE_URL]
return [u.strip() for u in self.PRICEBOT_INSTANCES.split(",") if u.strip()]
# ===== 内部(server→server)端点密钥 =====
# pricebot 比价 done 后把价格观测 POST 到 /internal/price-observation 落库,
# 靠这个共享密钥头(X-Internal-Secret)校验,与 pricebot 侧 INTERNAL_API_SECRET 同值。
# 默认空 = 内部写端点关闭(返 503),启用前两边都要配上同一高熵串。
INTERNAL_API_SECRET: str = ""
# ===== 媒体文件(用户头像上传)=====
# 落盘根目录(data/ 已 gitignore,上传不进库);对外经 StaticFiles 挂在 MEDIA_URL_PREFIX。
# 生产可改由 nginx 直接 serve MEDIA_ROOT,绕过应用进程。
+3
View File
@@ -20,6 +20,7 @@ from app.api.v1.compare import router as compare_router
from app.api.v1.compare_milestone import router as compare_milestone_router
from app.api.v1.compare_record import router as compare_record_router
from app.api.v1.coupon import router as coupon_router
from app.api.internal.price import router as internal_price_router
from app.api.v1.feedback import router as feedback_router
from app.api.v1.meituan import router as meituan_router
from app.api.v1.order import router as order_router
@@ -90,6 +91,8 @@ app.include_router(savings_router)
app.include_router(ad_router)
app.include_router(order_router)
app.include_router(report_router)
# 内部(server→server)端点:pricebot 上报价格观测,靠共享密钥头校验,不对客户端开放。
app.include_router(internal_price_router)
# 用户上传文件(头像)静态服务。生产可改由 nginx 直接 serve MEDIA_ROOT。
_media_root = Path(settings.MEDIA_ROOT)
+1
View File
@@ -7,6 +7,7 @@ from app.models.app_config import AppConfig # noqa: F401
from app.models.comparison import ComparisonRecord # noqa: F401
from app.models.comparison_milestone import ComparisonMilestoneClaim # noqa: F401
from app.models.feedback import Feedback # noqa: F401
from app.models.price_observation import PriceObservation # noqa: F401
from app.models.price_report import PriceReport # noqa: F401
from app.models.savings import SavingsRecord # noqa: F401
from app.models.signin import SigninRecord # noqa: F401
+111
View File
@@ -0,0 +1,111 @@
"""价格观测表(price_observation)—— 比价沉淀的价格资产层。
每完成一次比价,pricebot 在 done 帧把"在某平台某店,这一篮菜/这单到手价多少钱"
按平台逐条算出来,server→server 内部上报落这里。**与登录无关、不依赖客户端上报**——
比价透传链路当前不鉴权,只要比价跑到 done 就记,匿名用户也记(来源记 device_id,
user_id 客户端带上时一并记)。
与 comparison_record 的区别:
- comparison_record:用户视角的「我的比价记录」,客户端登录后主动上报,按 user_id 存。
- price_observation:**平台/门店视角的价格事实**,server 侧无条件沉淀,按 (平台,门店,菜,时间)
组织,是未来"别人查过同店→直接秒回价格"大数据的源头。两表独立,互不影响。
「先存下来、用法后说」:结构化列给将来按 平台/门店/菜/时间/地理 查询聚合用;dishes / attrs
(JSONB)兜底存灵活明细,免得每多记一个字段就迁移 schema。raw trace(逐 step 全过程)
另存 pricebot 本地 jsonl.gz,本表只存提炼后的价格事实(本表可从 raw 重算)。
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import (
JSON,
Boolean,
DateTime,
Float,
Integer,
String,
UniqueConstraint,
func,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base
# PG 上用 JSONB(可建 GIN 索引),SQLite(本地/测试)退化为通用 JSON(同 comparison_record)。
_JSON = JSON().with_variant(JSONB(), "postgresql")
class PriceObservation(Base):
__tablename__ = "price_observation"
__table_args__ = (
# 一次比价(trace)里,同一平台、同一口径(scope)只记一条:pricebot 重试 / 客户端
# replay 重复上报时幂等去重(insert-or-ignore)。一次外卖比价 = 源 + N 个目标平台
# 各一条 scope='order'。
UniqueConstraint(
"trace_id", "platform", "scope",
name="uq_price_obs_trace_platform_scope",
),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
# 观测时刻(≈ 比价 done 时;服务端落库时取 now)。按时间查"最近有效价"用,建索引。
observed_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), index=True, nullable=False
)
# pricebot 侧 trace_id:回指 A 层原始 trace(能溯源 / 重算)+ 幂等去重键
trace_id: Mapped[str] = mapped_column(String(64), index=True, nullable=False)
# 业务类型:food(外卖,当前唯一接通)/ ecom(电商,二期)。预留扩展。
business_type: Mapped[str] = mapped_column(
String(16), nullable=False, default="food", index=True
)
# ===== 门店维度(先 denormalize 成文本,不建维度表;实体归一二期再做)=====
platform: Mapped[str] = mapped_column(String(32), index=True, nullable=False)
# 平台内门店 ID(抓得到就存,将来归一最稳的 key;现在多为空)
platform_store_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
store_name: Mapped[str | None] = mapped_column(String(128), index=True, nullable=True)
city: Mapped[str | None] = mapped_column(String(64), nullable=True)
# 地理(现在拿不到 → 留空;二期把 device 经纬度透传进比价请求后回填)
geohash: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
lng: Mapped[float | None] = mapped_column(Float, nullable=True)
lat: Mapped[float | None] = mapped_column(Float, nullable=True)
# ===== 价格事实 =====
# 是否源平台(发起比价那家)。源平台价也是真实观测,照记。
is_source: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# 口径:order(整单到手价,外卖当前唯一)/ dish(单菜单价,引擎暂不逐菜读,二期)
scope: Mapped[str] = mapped_column(String(16), nullable=False, default="order")
# 该口径的到手价(分);None = 该平台采集失败 / 门店打烊,无有效价
price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True)
coupon_saved_cents: Mapped[int | None] = mapped_column(Integer, nullable=True)
coupon_name: Mapped[str | None] = mapped_column(String(64), nullable=True)
# 该平台目标店打烊原因(非空时 price_cents 为 None)
store_closed: Mapped[str | None] = mapped_column(String(32), nullable=True)
# 本次比价里的全局名次(1=最便宜;按到手价升序,源/目标统一排)
rank: Mapped[int | None] = mapped_column(Integer, nullable=True)
# ===== 明细(JSON)=====
# 菜篮 [{name, qty, subtotal?}](外卖)。dish 级单价二期从这里 + 引擎增强推。
dishes: Mapped[list | None] = mapped_column(_JSON, nullable=True)
# 灵活字段兜底(配送费 / 起送 / 活动名 / 跳过菜数 等),免得加字段就迁移。
attrs: Mapped[dict | None] = mapped_column(_JSON, nullable=True)
# ===== 来源(溯源 / 去重 / 置信)=====
source_device_id: Mapped[str | None] = mapped_column(String(64), index=True, nullable=True)
# user_id 当前比价链路不鉴权拿不到,客户端带上时才有;先可空。
source_user_id: Mapped[int | None] = mapped_column(Integer, index=True, nullable=True)
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=1.0)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
def __repr__(self) -> str: # pragma: no cover
return (
f"<PriceObservation id={self.id} platform={self.platform} "
f"store={self.store_name!r} price_cents={self.price_cents} "
f"trace_id={self.trace_id}>"
)
+86
View File
@@ -0,0 +1,86 @@
"""价格观测落库:一次比价的整批观测幂等写入。
幂等键 = (trace_id, platform, scope)。pricebot 重试 / replay 会重复上报同一 trace,
这里先查该 trace 已存在的 (platform, scope),只插新的——跨方言(PG / SQLite dev)都安全,
不依赖 ON CONFLICT。极端并发下若仍撞唯一约束,IntegrityError 回滚后忽略(本批本就幂等)。
"""
from __future__ import annotations
import logging
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.models.price_observation import PriceObservation
from app.schemas.price_observation import PriceObservationBatchIn
logger = logging.getLogger("shagua.price_obs")
def insert_batch(db: Session, payload: PriceObservationBatchIn) -> tuple[int, int]:
"""写入一次比价的整批观测,返回 (inserted, skipped)。
skipped = 因 (trace_id, platform, scope) 已存在而幂等跳过的条数。
"""
if not payload.observations:
return 0, 0
# 该 trace 已有的 (platform, scope),用于幂等过滤
existing: set[tuple[str, str]] = set(
db.execute(
select(PriceObservation.platform, PriceObservation.scope).where(
PriceObservation.trace_id == payload.trace_id
)
).all()
)
inserted = 0
skipped = 0
for obs in payload.observations:
key = (obs.platform, obs.scope)
if key in existing:
skipped += 1
continue
existing.add(key) # 同批内去重(防 pricebot 同帧给重复平台)
db.add(
PriceObservation(
trace_id=payload.trace_id,
business_type=payload.business_type,
platform=obs.platform,
platform_store_id=obs.platform_store_id,
store_name=payload.store_name,
city=payload.city,
geohash=payload.geohash,
lng=payload.lng,
lat=payload.lat,
is_source=obs.is_source,
scope=obs.scope,
price_cents=obs.price_cents,
coupon_saved_cents=obs.coupon_saved_cents,
coupon_name=obs.coupon_name,
store_closed=obs.store_closed,
rank=obs.rank,
dishes=payload.dishes,
attrs=obs.attrs,
source_device_id=payload.source_device_id,
source_user_id=payload.source_user_id,
)
)
inserted += 1
if inserted == 0:
return 0, skipped
try:
db.commit()
except IntegrityError:
# 并发下另一个请求刚插了同 (trace,platform,scope) → 唯一约束撞了。
# 本批本就幂等,回滚后当作全跳过(不重试,失败就失败,见 P0 约定)。
db.rollback()
logger.warning(
"price_observation 并发幂等冲突 trace=%s,本批回滚跳过", payload.trace_id
)
return 0, skipped + inserted
return inserted, skipped
+49
View File
@@ -0,0 +1,49 @@
"""价格观测内部上报的收发模型。
pricebot 在比价 done 帧 server→server POST 一批观测(一次比价 = 源 + N 目标平台各一条)。
批级字段(门店 / 菜篮 / 地理 / 来源)所有观测共享,放外层;逐平台字段放 observations[]。
"""
from __future__ import annotations
from pydantic import BaseModel, Field
class PriceObservationItem(BaseModel):
"""单个平台的价格观测(一次比价里的一行)。"""
platform: str
is_source: bool = False
scope: str = "order" # order / dish
price_cents: int | None = None # 到手价(分);None=该平台失败/打烊
coupon_saved_cents: int | None = None
coupon_name: str | None = None
store_closed: str | None = None # 打烊原因(非空时 price_cents=None)
rank: int | None = None # 全局名次(1=最便宜)
platform_store_id: str | None = None
attrs: dict | None = None # 该平台的灵活明细兜底
class PriceObservationBatchIn(BaseModel):
"""一次比价的整批价格观测上报体。"""
trace_id: str
business_type: str = "food"
# 门店 / 菜篮 / 地理 —— 一次比价同一份,批级共享
store_name: str | None = None
city: str | None = None
geohash: str | None = None
lng: float | None = None
lat: float | None = None
dishes: list | None = None # [{name, qty, subtotal?}]
# 来源
source_device_id: str | None = None
source_user_id: int | None = None
# 逐平台观测(源 + 各目标)
observations: list[PriceObservationItem] = Field(default_factory=list)
class PriceObservationBatchOut(BaseModel):
"""上报结果:本批新写入条数 / 因幂等跳过条数。"""
inserted: int
skipped: int