feat(meituan-cps): 经纬度→城市离线反查 + rec/销量最高按城市过滤 (#116)
## 主要功能 新增离线「经纬度 → 美团 cityId」反查,让 `rec`(智能推荐)与 `top-sales`(销量最高)从离线库只返**同城券**(此前会混返异地券)。 - `app/utils/geo.py` + `app/utils/meituan_city.py`:坐标 → 美团 `city_id`(reverse_geocoder 离线反查,零网络)。 - `feed?tab=rec` / `/top-sales`:按 `city_id` 过滤;解析不出 / 老客户端不带坐标 → 降级返空。 - `top-sales` 与 `rec` 一致置空库内距离(相对城市默认点、对用户无意义)。 --------- Co-authored-by: guke <guke@autohome.com.cn> Reviewed-on: #116 Co-authored-by: guke <guke@wonderable.ai> Co-committed-by: guke <guke@wonderable.ai>
This commit was merged in pull request #116.
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
"""把 meituan_coupon 的线上采样 TSV 灌进本地 SQLite。
|
||||
|
||||
用途:本地开发/调试时,把线上 `meituan_coupon` 表的采样数据(tests/meituan_coupon_data.tsv)
|
||||
灌进 dev 库(默认 `./data/app.db`),免得每次都实时打美团接口。
|
||||
|
||||
TSV 说明:
|
||||
- 制表符分隔,每行一条记录,列顺序与线上 PostgreSQL 物理列一致
|
||||
(image_size / image_type 是后加的迁移,排在最后两列 —— 与本地 SQLite 一致)。
|
||||
- 空字段 = NULL(文件里没有 `\\N` 标记)。
|
||||
- 个别记录的文本/JSON 字段内含换行,会把一条逻辑行拆成多物理行 —— 按“累计到 26 列”重组。
|
||||
- 文件尾部可能有一条被导出截断的残行(列数不足 / raw JSON 不完整),直接跳过。
|
||||
|
||||
datetime 三列(first_seen/last_seen/updated_at)去掉尾部时区偏移(`+08`),
|
||||
存成 SQLAlchemy 在 SQLite 上用的朴素格式 `YYYY-MM-DD HH:MM:SS.ffffff`,保证 ORM 能读回。
|
||||
|
||||
用法:
|
||||
python scripts/load_meituan_coupon_tsv.py # 默认 TSV + .env 里的库
|
||||
python scripts/load_meituan_coupon_tsv.py path/to.tsv # 指定 TSV
|
||||
DATABASE_URL=sqlite:///./data/app.db python scripts/load_meituan_coupon_tsv.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
# 列顺序 = TSV 字段顺序 = 本地 SQLite 物理列顺序
|
||||
COLS = [
|
||||
"id", "source", "platform", "biz_line", "city_id", "product_view_sign",
|
||||
"sku_view_id", "name", "brand_name", "sell_price_cents", "original_price_cents",
|
||||
"head_url", "sale_volume", "sale_volume_num", "commission_percent",
|
||||
"commission_amount_cents", "poi_name", "available_poi_num", "delivery_distance_m",
|
||||
"dedup_key", "raw", "first_seen", "last_seen", "updated_at", "image_size", "image_type",
|
||||
]
|
||||
NCOL = len(COLS)
|
||||
|
||||
# 按列做类型转换(空串 -> None)。未列出的列 = 原样字符串(source/city_id/... 等 NOT NULL 文本)。
|
||||
_INT_COLS = {0, 2, 3, 9, 10, 13, 15, 17, 24} # id, platform, biz_line, prices, ...
|
||||
_FLOAT_COLS = {14, 18} # commission_percent, delivery_distance_m
|
||||
_NULLABLE_STR_COLS = {6, 7, 8, 11, 12, 16, 25} # sku_view_id, name, brand_name, ...
|
||||
_DT_COLS = {21, 22, 23} # first_seen, last_seen, updated_at
|
||||
_RAW_COL = 20
|
||||
|
||||
_TZ_SUFFIX = re.compile(r"[+-]\d{2}(:?\d{2})?$") # 尾部时区偏移 +08 / +08:00 / +0800
|
||||
|
||||
|
||||
def _resolve_sqlite_path() -> Path:
|
||||
"""从 DATABASE_URL(env 或 .env)解析出 SQLite 文件路径。只支持 sqlite://。"""
|
||||
url = os.environ.get("DATABASE_URL", "")
|
||||
if not url:
|
||||
env = _PROJECT_ROOT / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("DATABASE_URL="):
|
||||
url = line.split("=", 1)[1].strip()
|
||||
break
|
||||
if not url:
|
||||
url = "sqlite:///./data/app.db"
|
||||
if not url.startswith("sqlite:"):
|
||||
sys.exit(f"仅支持 sqlite:// 库,当前 DATABASE_URL={url!r}")
|
||||
rest = url.split("sqlite:///", 1)[1] if "sqlite:///" in url else url.split("sqlite://", 1)[1]
|
||||
p = Path(rest)
|
||||
if not p.is_absolute():
|
||||
p = (_PROJECT_ROOT / rest).resolve()
|
||||
return p
|
||||
|
||||
|
||||
def _reconstruct_rows(text: str) -> tuple[list[list[str]], int]:
|
||||
"""把文件文本重组成一条条 26 列的逻辑行。返回 (rows, skipped)。
|
||||
|
||||
单个字段内含换行 -> 一条逻辑行被拆成多物理行:累计字段,拆点用 \\n 重新拼回,
|
||||
直到凑满 26 列。列数溢出(内嵌 TAB / 错位)或文件尾残行 -> 跳过并计数。
|
||||
"""
|
||||
lines = text.split("\n")
|
||||
while lines and lines[-1] == "":
|
||||
lines.pop()
|
||||
rows: list[list[str]] = []
|
||||
skipped = 0
|
||||
buf: list[str] = []
|
||||
for raw_line in lines:
|
||||
parts = raw_line.split("\t")
|
||||
if not buf:
|
||||
buf = parts
|
||||
else:
|
||||
buf[-1] += "\n" + parts[0] # 拼回被换行拆开的字段
|
||||
buf.extend(parts[1:])
|
||||
if len(buf) == NCOL:
|
||||
rows.append(buf)
|
||||
buf = []
|
||||
elif len(buf) > NCOL: # 溢出:数据异常,丢弃这段重新开始
|
||||
print(f" [skip] 列数溢出({len(buf)}>{NCOL}),field0={buf[0][:20]!r}")
|
||||
skipped += 1
|
||||
buf = []
|
||||
if buf: # 文件尾被截断的残行
|
||||
print(f" [skip] 尾部残行不足 {NCOL} 列(实 {len(buf)} 列),field0={buf[0][:20]!r}")
|
||||
skipped += 1
|
||||
return rows, skipped
|
||||
|
||||
|
||||
def _convert(row: list[str]) -> tuple | None:
|
||||
"""按列类型转换一行;非法(必填 int 为空 / raw 非 JSON)返回 None。"""
|
||||
out: list = []
|
||||
for i, v in enumerate(row):
|
||||
if i in _DT_COLS:
|
||||
out.append(_TZ_SUFFIX.sub("", v))
|
||||
continue
|
||||
if i == _RAW_COL:
|
||||
try:
|
||||
json.loads(v)
|
||||
except Exception as e:
|
||||
print(f" [skip] id={row[0]} raw 非法 JSON: {e}")
|
||||
return None
|
||||
out.append(v)
|
||||
continue
|
||||
if i in _INT_COLS:
|
||||
out.append(int(v) if v != "" else None)
|
||||
elif i in _FLOAT_COLS:
|
||||
out.append(float(v) if v != "" else None)
|
||||
elif i in _NULLABLE_STR_COLS:
|
||||
out.append(v if v != "" else None)
|
||||
else: # 必填文本列,原样
|
||||
out.append(v)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tsv = Path(sys.argv[1]) if len(sys.argv) > 1 else _PROJECT_ROOT / "tests" / "meituan_coupon_data.tsv"
|
||||
if not tsv.is_absolute():
|
||||
tsv = (_PROJECT_ROOT / tsv).resolve()
|
||||
db = _resolve_sqlite_path()
|
||||
print(f"TSV: {tsv}")
|
||||
print(f"DB : {db}")
|
||||
if not tsv.exists():
|
||||
sys.exit(f"TSV 不存在: {tsv}")
|
||||
if not db.exists():
|
||||
sys.exit(f"SQLite 库不存在: {db}(先跑 alembic upgrade head 建表)")
|
||||
|
||||
rows, skipped = _reconstruct_rows(tsv.read_text(encoding="utf-8"))
|
||||
print(f"重组逻辑行: {len(rows)} 跳过(残/异常): {skipped}")
|
||||
|
||||
records = []
|
||||
bad = 0
|
||||
for r in rows:
|
||||
rec = _convert(r)
|
||||
if rec is None:
|
||||
bad += 1
|
||||
continue
|
||||
records.append(rec)
|
||||
print(f"可入库: {len(records)} 转换失败: {bad}")
|
||||
|
||||
placeholders = ",".join(["?"] * NCOL)
|
||||
sql = f"INSERT OR REPLACE INTO meituan_coupon ({','.join(COLS)}) VALUES ({placeholders})"
|
||||
con = sqlite3.connect(str(db))
|
||||
try:
|
||||
before = con.execute("SELECT count(*) FROM meituan_coupon").fetchone()[0]
|
||||
con.executemany(sql, records)
|
||||
con.commit()
|
||||
after = con.execute("SELECT count(*) FROM meituan_coupon").fetchone()[0]
|
||||
finally:
|
||||
con.close()
|
||||
print(f"入库前 {before} 行 -> 入库后 {after} 行(本次 {len(records)} 条)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,189 @@
|
||||
"""给指定用户灌一批 mock 比价记录(开发 / 真机走查用)。
|
||||
|
||||
为什么:首页「上次比价」横幅(4 分钟新鲜窗口)、比价记录页、「我的」省钱战绩卡都读
|
||||
`comparison_record`。dev 库里这个用户没数据时这些 UI 是空的,本脚本灌一批真实感的外卖
|
||||
比价记录,方便端上走查。
|
||||
|
||||
用法(项目根、已 pip install -e . 的环境):
|
||||
python scripts/mock_compare_records.py # 默认 user 昵称后缀 DlSKoc5S7, 10 条
|
||||
python scripts/mock_compare_records.py --user DlSKoc5S7 # 按 昵称后缀/用户名/邀请码/手机号
|
||||
python scripts/mock_compare_records.py --user 5 # 直接按 user.id
|
||||
python scripts/mock_compare_records.py --user 5 --count 6
|
||||
|
||||
要点:
|
||||
- 幂等:trace_id 固定为 mock-<标识>-NN,重跑覆盖同号记录(不会越灌越多)。
|
||||
- 第 1 条 created_at = 跑脚本当下(4 分钟内),其余铺在近 7 天;重跑会刷新时间戳
|
||||
(所以重跑一次即可让「上次比价」横幅重新进入 4 分钟窗口)。
|
||||
- best_* / saved / status 由真实的 repositories.comparison.upsert_record 从 comparison_results 派生。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from app.core.rewards import CN_TZ
|
||||
from app.db.session import SessionLocal
|
||||
from app.models.user import User
|
||||
from app.repositories.comparison import upsert_record
|
||||
from app.schemas.compare_record import ComparisonRecordIn
|
||||
|
||||
# 平台名 → id(展示用;横幅按 platform_name 渲染,id 仅留痕)
|
||||
_PLATFORM_ID = {"美团": "meituan", "京东": "jd", "淘宝闪购": "taobao", "饿了么": "ele"}
|
||||
|
||||
# 10 单真实感外卖订单:source=美团,targets=(平台, 到手价/元, 优惠来源名|None, 红包额/元)。
|
||||
# failed=True 表示目标平台没采到价(targets 价置 None → 派生 status=failed)。
|
||||
_CATALOG = [
|
||||
dict(store="瑞幸咖啡(国贸店)", items=[("生椰拿铁(大杯)", 2)],
|
||||
source=("美团", 39.8),
|
||||
targets=[("淘宝闪购", 29.9, "平台红包", 6.0), ("京东", 33.0, None, 0)]),
|
||||
dict(store="茶百道(大悦城店)", items=[("杨梅冰萃(大杯)", 1), ("脆波波奶茶", 1)],
|
||||
source=("美团", 31.0),
|
||||
targets=[("京东", 24.5, "百亿补贴", 4.5), ("饿了么", 27.9, None, 0)]),
|
||||
dict(store="麦当劳(西单店)", items=[("板烧鸡腿堡套餐", 1)],
|
||||
source=("美团", 32.0),
|
||||
targets=[("京东", 25.9, "神券", 3.0), ("淘宝闪购", 28.0, None, 0)]),
|
||||
dict(store="肯德基(朝阳大悦城店)", items=[("疯狂星期四全家桶", 1)],
|
||||
source=("美团", 89.0),
|
||||
targets=[("京东", 79.9, "百亿补贴", 9.0)]),
|
||||
dict(store="张亮麻辣烫(双井店)", items=[("自选麻辣烫", 1)],
|
||||
source=("美团", 35.0),
|
||||
targets=[("京东", None, None, 0), ("淘宝闪购", None, None, 0)],
|
||||
failed=True, fail_reason="京东外卖、淘宝闪购均未找到该店"),
|
||||
dict(store="必胜客(王府井店)", items=[("超级至尊比萨(9寸)", 1), ("香辣鸡翅", 1)],
|
||||
source=("美团", 78.0),
|
||||
targets=[("淘宝闪购", 62.0, "跨店满减", 10.0), ("京东", 69.0, None, 0)]),
|
||||
dict(store="蜜雪冰城(中关村店)", items=[("多肉葡萄", 2), ("冰鲜柠檬水", 1)],
|
||||
source=("美团", 21.0),
|
||||
targets=[("京东", 16.5, None, 0)]),
|
||||
dict(store="海底捞外送(三里屯店)", items=[("番茄锅底", 1), ("鲜毛肚", 2), ("虾滑", 1)],
|
||||
source=("美团", 168.0),
|
||||
targets=[("京东", 155.0, "大额神券", 13.0), ("饿了么", 162.0, None, 0)]),
|
||||
dict(store="华莱士(回龙观店)", items=[("全鸡汉堡套餐", 1)],
|
||||
source=("美团", 26.0),
|
||||
targets=[("京东", None, None, 0)],
|
||||
failed=True, fail_reason="该商品在京东外卖未上架"),
|
||||
dict(store="星巴克(国贸店)", items=[("燕麦拿铁(大杯)", 2)],
|
||||
source=("美团", 66.0),
|
||||
targets=[("饿了么", 58.0, "会员券", 8.0), ("京东", 61.0, None, 0)]),
|
||||
]
|
||||
|
||||
# 第 2..N 条的 created_at 相对当下的回退量(都 > 4 分钟,确保只有第 1 条落在新鲜窗口)。
|
||||
_OFFSETS = [
|
||||
timedelta(minutes=25), timedelta(hours=2, minutes=10), timedelta(hours=6),
|
||||
timedelta(days=1, hours=3), timedelta(days=1, hours=20), timedelta(days=2, hours=9),
|
||||
timedelta(days=3, hours=14), timedelta(days=5, hours=7), timedelta(days=7, hours=2),
|
||||
]
|
||||
|
||||
|
||||
def _resolve_user(db, ident: str) -> User | None:
|
||||
"""按 id(纯数字)/ 用户名 / 邀请码 / 手机号 / 昵称(或昵称去掉「用户」前缀的后缀)解析用户。"""
|
||||
if ident.isdigit():
|
||||
u = db.get(User, int(ident))
|
||||
if u is not None:
|
||||
return u
|
||||
return db.execute(
|
||||
select(User).where(
|
||||
or_(
|
||||
User.username == ident,
|
||||
User.invite_code == ident,
|
||||
User.phone == ident,
|
||||
User.nickname == ident,
|
||||
User.nickname == f"用户{ident}",
|
||||
User.nickname.like(f"%{ident}"),
|
||||
)
|
||||
)
|
||||
).scalars().first()
|
||||
|
||||
|
||||
def _build_results(order: dict) -> tuple[list[dict], str]:
|
||||
"""造 comparison_results + information 文案;返回 (results, information)。"""
|
||||
src_name, src_price = order["source"]
|
||||
failed = order.get("failed", False)
|
||||
rows: list[dict] = [dict(
|
||||
platform_id=_PLATFORM_ID.get(src_name), platform_name=src_name,
|
||||
price=src_price, is_source=True, status="success",
|
||||
)]
|
||||
for pname, price, cname, csaved in order["targets"]:
|
||||
row = dict(
|
||||
platform_id=_PLATFORM_ID.get(pname), platform_name=pname,
|
||||
price=(None if failed else price), is_source=False,
|
||||
status=("store_not_found" if failed else "success"),
|
||||
)
|
||||
if not failed and csaved:
|
||||
row["coupon_saved"] = csaved
|
||||
row["coupon_name"] = cname
|
||||
rows.append(row)
|
||||
|
||||
# rank:有价的按升序 1..N;无价的不排
|
||||
priced = sorted((r for r in rows if r["price"] is not None), key=lambda r: r["price"])
|
||||
for i, r in enumerate(priced, start=1):
|
||||
r["rank"] = i
|
||||
|
||||
if failed:
|
||||
return rows, order.get("fail_reason", "目标平台未找到该商品")
|
||||
best = priced[0]
|
||||
saved = src_price - best["price"]
|
||||
if best["is_source"]:
|
||||
info = f"{src_name}已是最低价,本单未比出更低"
|
||||
else:
|
||||
info = f"已在{best['platform_name']}比出最低价 ¥{best['price']:.2f}(比{src_name}省 ¥{saved:.2f})"
|
||||
return rows, info
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="给用户灌一批 mock 比价记录")
|
||||
parser.add_argument("--user", default="DlSKoc5S7", help="user.id / 用户名 / 邀请码 / 手机号 / 昵称后缀")
|
||||
parser.add_argument("--count", type=int, default=10, help="条数(1..10),默认 10")
|
||||
args = parser.parse_args()
|
||||
|
||||
count = max(1, min(args.count, len(_CATALOG)))
|
||||
now = datetime.now(CN_TZ).replace(tzinfo=None)
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
user = _resolve_user(db, args.user)
|
||||
if user is None:
|
||||
raise SystemExit(f"❌ 找不到用户:{args.user}(试试 --user <id>)")
|
||||
print(f"→ 用户 id={user.id} nickname={user.nickname} phone={user.phone}")
|
||||
|
||||
ok = failed = 0
|
||||
for idx in range(count):
|
||||
order = _CATALOG[idx]
|
||||
src_name, src_price = order["source"]
|
||||
results, info = _build_results(order)
|
||||
payload = ComparisonRecordIn(
|
||||
trace_id=f"mock-{args.user}-{idx:02d}",
|
||||
business_type="food",
|
||||
device_id="mock-seed",
|
||||
store_name=order["store"],
|
||||
source_platform_id=_PLATFORM_ID.get(src_name),
|
||||
source_platform_name=src_name,
|
||||
source_price=src_price,
|
||||
items=[{"name": n, "qty": q} for n, q in order["items"]],
|
||||
comparison_results=results,
|
||||
total_dish_count=sum(q for _, q in order["items"]),
|
||||
information=info,
|
||||
app_version="mock",
|
||||
)
|
||||
rec = upsert_record(db, user_id=user.id, payload=payload)
|
||||
# created_at:第 1 条 = 当下(4 分钟新鲜窗口);其余铺近 7 天。重跑刷新时间戳。
|
||||
rec.created_at = now if idx == 0 else now - _OFFSETS[idx - 1]
|
||||
db.commit()
|
||||
if rec.status == "success":
|
||||
ok += 1
|
||||
else:
|
||||
failed += 1
|
||||
tag = "🆕now" if idx == 0 else f"-{_OFFSETS[idx - 1]}"
|
||||
print(f" [{rec.status:7}] id={rec.id} {order['store']} best={rec.best_platform_name} "
|
||||
f"¥{(rec.best_price_cents or 0)/100:.2f} saved=¥{(rec.saved_amount_cents or 0)/100:.2f} {tag}")
|
||||
|
||||
print(f"✅ 已为 user id={user.id} 写入 {count} 条(success {ok} / failed {failed});"
|
||||
f"第 1 条 created_at=当下,4 分钟内进 app 可见「上次比价」横幅。")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user