diff --git a/alembic/versions/meituan_coupon_image_meta.py b/alembic/versions/meituan_coupon_image_meta.py new file mode 100644 index 0000000..e143c62 --- /dev/null +++ b/alembic/versions/meituan_coupon_image_meta.py @@ -0,0 +1,34 @@ +"""meituan_coupon image size & type + +给 meituan_coupon 增加 image_size(字节)/ image_type(MIME)两列,采集时 HEAD 头图得到, +供读取侧按图片大小决定是否拼缩放参数提速。两列均 nullable,不动存量行;下一轮全量 ETL +upsert 自动回填,无需手动 backfill。 + +Revision ID: meituan_coupon_image_meta +Revises: feedback_review_fields +Create Date: 2026-06-23 00:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "meituan_coupon_image_meta" +down_revision: str | Sequence[str] | None = "feedback_review_fields" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + with op.batch_alter_table("meituan_coupon", schema=None) as batch_op: + batch_op.add_column(sa.Column("image_size", sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column("image_type", sa.String(length=32), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("meituan_coupon", schema=None) as batch_op: + batch_op.drop_column("image_type") + batch_op.drop_column("image_size") diff --git a/app/models/meituan_coupon.py b/app/models/meituan_coupon.py index df6cd10..216f10f 100644 --- a/app/models/meituan_coupon.py +++ b/app/models/meituan_coupon.py @@ -55,6 +55,10 @@ class MeituanCoupon(Base): sell_price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) original_price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True) head_url: Mapped[str | None] = mapped_column(String(512), nullable=True) + # 头图字节大小 / MIME 类型:采集时 HEAD head_url 得到(Content-Length / Content-Type)。 + # 读取侧已统一压缩(不再按大小判定),这两列保留用于分析/监控(GIF 占比、体积分布等)。 + image_size: Mapped[int | None] = mapped_column(Integer, nullable=True) # 字节 + image_type: Mapped[str | None] = mapped_column(String(32), nullable=True) # 如 image/jpeg # ===== 销量(美团只给粗档位:热销1w+;num=排序用的下界数值,如 1w+ → 10000) ===== sale_volume: Mapped[str | None] = mapped_column(String(32), nullable=True) diff --git a/app/schemas/meituan.py b/app/schemas/meituan.py index 14ce264..b40f3ed 100644 --- a/app/schemas/meituan.py +++ b/app/schemas/meituan.py @@ -13,6 +13,21 @@ from pydantic import BaseModel, Field FeedStatus = Literal["ok", "empty", "degraded"] +# ───────────────── 头图传输优化(统一缩放 + 转 WebP) ───────────────── +# 美团图片 CDN 支持在 URL 后拼 @w_h_1e_1c[.webp] 让服务端缩放/转码。feed 卡片只占 +# 124dp(≈372px@3x),而库里 head_url 是原图(实测中位 137KB、长尾到 17MB、GIF 均 4.8MB), +# 是首页 feed 图片加载偏慢的来源。按产品要求,头图**统一**缩到 124dp + 转 WebP 再下发 +# (全部压缩、不设大小阈值),由美团 CDN 服务端缩放/转码,实测省 76–99% 体积。 +FEED_THUMB_PARAM = "@375w_375h_1e_1c.webp" # 124dp@3x 缩略 + 转 WebP + + +def feed_image_url(head_url: str) -> str: + """统一给头图 URL 拼缩放参数(缩到 124dp + 转 WebP),让美团 CDN 出小图;空 URL 原样返回。""" + if not head_url: + return head_url + return head_url.split("@")[0] + FEED_THUMB_PARAM + + # ───────────────── 券卡片(归一化后给客户端) ───────────────── class CouponCard(BaseModel): @@ -94,7 +109,7 @@ class CouponCard(BaseModel): platform=platform, biz_line=biz_line, name=cpd.get("name") or "", - head_image_url=head_url, + head_image_url=feed_image_url(head_url), brand_name=brand.get("brandName"), brand_logo_url=brand.get("brandLogoUrl"), sell_price=sell, diff --git a/deploy/backfill_image_meta.sh b/deploy/backfill_image_meta.sh new file mode 100644 index 0000000..e2f3577 --- /dev/null +++ b/deploy/backfill_image_meta.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# 美团券头图大小/类型回填 —— 全量重跑一轮 ETL,把 meituan_coupon.image_size / image_type 灌满。 +# 上线本 PR 后在服务器手动跑一次,立即回填存量券(不必等定时器逐轮 upsert 自然补齐)。 +# +# 用法(服务器, root): +# cd /opt/shaguabijia-app-server +# sudo bash deploy/backfill_image_meta.sh +# # 全量一轮 ~70min,建议在 tmux/screen 里跑,防 SSH 断开中断 +# +# 前提:已拉到含本 PR 的 main、已跑 `alembic upgrade head`(脚本会自检列是否存在)、 +# 已 restart shaguabijia-app-server.service。服务器 MT_CPS_PROXY 留空(直连 CDN)。 +set -euo pipefail + +APP_DIR="${APP_DIR:-/opt/shaguabijia-app-server}" +PY="$APP_DIR/.venv/bin/python" +cd "$APP_DIR" +mkdir -p data +LOG="data/etl_backfill_$(date +%Y%m%d_%H%M%S).log" + +# 退出时务必恢复定时器(即使 ETL 中途失败,也不把 timer 留在停用态) +restore_timer() { systemctl start meituan-etl.timer 2>/dev/null || true; } +trap restore_timer EXIT + +echo "[0/3] 自检 image_size/image_type 列(缺列说明没迁移,先跑 alembic upgrade head)..." +"$PY" -c "from sqlalchemy import inspect; from app.db.session import engine; \ +cols=[c['name'] for c in inspect(engine).get_columns('meituan_coupon')]; \ +import sys; (('image_size' in cols) and ('image_type' in cols)) or sys.exit(' ❌ 缺列,请先: sudo .venv/bin/python -m alembic upgrade head'); \ +print(' ✓ 列已就绪')" + +echo "[1/3] 停定时器,防与定时轮次并发占锁..." +systemctl stop meituan-etl.timer 2>/dev/null || true + +echo "[2/3] 全量重跑 ETL(~70min;日志 $LOG)..." +# 不走 systemctl:meituan-etl.service 有 20min 硬超时,全量一轮跑不完会被掐断。 +"$PY" -m scripts.pull_meituan_coupons --once --prune-hours 24 2>&1 | tee "$LOG" + +echo "[3/3] 验证填充率..." +"$PY" - <<'PYEOF' +from sqlalchemy import text +from app.db.session import engine +engine.echo = False +with engine.connect() as c: + tot = c.execute(text("SELECT count(*) FROM meituan_coupon")).scalar() + enr = c.execute(text("SELECT count(*) FROM meituan_coupon WHERE image_size IS NOT NULL")).scalar() + gif = c.execute(text("SELECT count(*) FROM meituan_coupon WHERE image_type='image/gif'")).scalar() + print(f" 总 {tot} 行 | 有 image_size {enr} ({enr*100//max(tot,1)}%) | GIF {gif} 张") +PYEOF +echo "✅ 回填完成(定时器已恢复)。" diff --git a/docs/api/README.md b/docs/api/README.md index 642afa1..1eabe1f 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -173,7 +173,7 @@ | `platform` | int | `1`=外卖/到家, `2`=到店 | | `biz_line` | int \| null | 到店子类:1到餐 2到综 3酒店 4门票 | | `name` | string | 商品名 | -| `head_image_url` | string | 头图 | +| `head_image_url` | string | 头图;后端已**统一**拼缩放参数 `@375w_375h_1e_1c.webp`(美团 CDN 服务端缩到 124dp + 转 WebP,实测省 76–99%),客户端直接用 | | `brand_name` | string \| null | 品牌名 | | `brand_logo_url` | string \| null | 品牌 logo | | `sell_price` | string | 现价 | diff --git a/scripts/pull_meituan_coupons.py b/scripts/pull_meituan_coupons.py index 3431cc8..a9fced4 100644 --- a/scripts/pull_meituan_coupons.py +++ b/scripts/pull_meituan_coupons.py @@ -52,6 +52,7 @@ try: except Exception: # noqa: BLE001 pass +import httpx # noqa: E402 from sqlalchemy import delete, func, select # noqa: E402 from sqlalchemy.dialects.postgresql import insert as pg_insert # noqa: E402 @@ -85,6 +86,13 @@ DEFAULT_CONCURRENCY = 12 # 并发城市数(实测 15 并发 402 占 3% 可退 STARTUP_STAGGER = 0.3 # 首批城市启动错峰间隔秒(削平瞬时峰值,实测能压低 402) PRUNE_FAIL_RATIO_MAX = 0.05 # 失败城占比超此值则本轮跳过 prune(避免大面积抓取失败误删库) +# 头图大小/类型富集:每城抓完后,按 head_url 去重并发 HEAD 取 Content-Length / Content-Type, +# 写进 image_size / image_type。图片走美团图片 CDN(p*.meituan.net / img.meituan.net),直连可达、 +# 与 CPS 网关 402 限流无关;任何失败 → None,不阻断入库。城市级已 12 并发,这里每城再开 8, +# 叠加峰值 ~96 个 HEAD,CDN 扛得住(实测单进程 50 并发稳)。 +IMAGE_META_WORKERS = 8 # 每城 HEAD 并发数 +IMAGE_META_TIMEOUT = 8.0 # 单图 HEAD 超时秒 + SOURCES = [ {"code": "search_waimai", "label": "外卖·搜外卖", "kind": "search", "platform": 1, "keyword": "外卖"}, {"code": "search_meishi", "label": "外卖·搜美食", "kind": "search", "platform": 1, "keyword": "美食"}, @@ -256,6 +264,9 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: "sell_price_cents": price_cents, "original_price_cents": _to_cents(cpd.get("originalPrice")), "head_url": ((cpd.get("headUrl") or "").split("@")[0][:512] or None), + # 占位;由 _fill_image_meta 抓完本城后并发 HEAD 回填(键必须在,upsert 才会带上这两列) + "image_size": None, + "image_type": None, "sale_volume": cpd.get("saleVolume"), "sale_volume_num": _sale_volume_num(cpd.get("saleVolume")), "commission_percent": comm_pct, @@ -268,6 +279,38 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: }) +def _head_image_meta(client: httpx.Client, url: str) -> tuple[int | None, str | None]: + """HEAD 单张头图,取 (字节大小, MIME 类型);任何失败返回 (None, None),不阻断入库。""" + try: + r = client.head(url, timeout=IMAGE_META_TIMEOUT, follow_redirects=True) + if r.status_code >= 400: + return None, None + cl = r.headers.get("content-length") + size = int(cl) if cl and cl.isdigit() else None + ctype = (r.headers.get("content-type") or "").split(";")[0].strip()[:32] or None + return size, ctype + except Exception: # noqa: BLE001 + return None, None + + +def _fill_image_meta(rows: list[dict]) -> None: + """就地为本城 rows 填 image_size / image_type:按 head_url 去重后并发 HEAD,再回填每行。 + trust_env=False 直连图片 CDN(绕本机代理,CDN 无需走 CPS 代理)。""" + urls = {r["head_url"] for r in rows if r.get("head_url")} + if not urls: + return + meta: dict[str, tuple[int | None, str | None]] = {} + with httpx.Client(trust_env=False, timeout=IMAGE_META_TIMEOUT) as client: + with ThreadPoolExecutor(max_workers=IMAGE_META_WORKERS) as pool: + futs = {pool.submit(_head_image_meta, client, u): u for u in urls} + for f in as_completed(futs): + meta[futs[f]] = f.result() + for r in rows: + m = meta.get(r.get("head_url")) + if m: + r["image_size"], r["image_type"] = m + + def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> list[dict]: """抓单个城市的 3 路券并解析。worker 线程内执行(只抓取+解析,不碰 DB)。 @@ -286,6 +329,7 @@ def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> p = _parse_item(it, src, cid) if p: parsed.append(p) + _fill_image_meta(parsed) # 抓完本城 → 并发 HEAD 回填头图 image_size/image_type return parsed