feat(meituan-coupon): 头图改为统一压缩(去掉大小阈值,全部拼缩放参数)

产品要求所有头图都压缩,不再按 image_size 阈值判定: - feed_image_url 改为无条件给图片 URL 拼 @375w_375h_1e_1c.webp(缩 124dp + 转 WebP) - 去掉 FEED_THUMB_SIZE_THRESHOLD 阈值常量;from_raw / 两处 DB 路不再传 image_size - image_size/image_type 两列保留用于分析/监控(GIF 占比、体积分布),采集逻辑不变 - 文档更新:head_image_url 现统一带缩放参数注:读取侧统一压缩,部署后即对全部 feed 图生效,无需依赖 image_size 回填。
feat(meituan-coupon): 采集记录头图大小/类型 + 读取侧缩放口子 + 回填脚本
2026-06-23 20:23:02 +08:00 · 2026-06-23 20:10:35 +08:00
6 changed files with 147 additions and 2 deletions
@@ -0,0 +1,34 @@
+"""meituan_coupon image size & type
+
+给 meituan_coupon 增加 image_size(字节)/ image_type(MIME)两列,采集时 HEAD 头图得到,
+供读取侧按图片大小决定是否拼缩放参数提速。两列均 nullable,不动存量行;下一轮全量 ETL
+upsert 自动回填,无需手动 backfill。
+
+Revision ID: meituan_coupon_image_meta
+Revises: feedback_review_fields
+Create Date: 2026-06-23 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "meituan_coupon_image_meta"
+down_revision: str | Sequence[str] | None = "feedback_review_fields"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("meituan_coupon", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("image_size", sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column("image_type", sa.String(length=32), nullable=True))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("meituan_coupon", schema=None) as batch_op:
+        batch_op.drop_column("image_type")
+        batch_op.drop_column("image_size")
@@ -55,6 +55,10 @@ class MeituanCoupon(Base):
    sell_price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True)
    original_price_cents: Mapped[int | None] = mapped_column(Integer, nullable=True)
    head_url: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    # 头图字节大小 / MIME 类型:采集时 HEAD head_url 得到(Content-Length / Content-Type)。
+    # 读取侧已统一压缩(不再按大小判定),这两列保留用于分析/监控(GIF 占比、体积分布等)。
+    image_size: Mapped[int | None] = mapped_column(Integer, nullable=True)         # 字节
+    image_type: Mapped[str | None] = mapped_column(String(32), nullable=True)      # 如 image/jpeg

    # ===== 销量(美团只给粗档位:热销1w+;num=排序用的下界数值,如 1w+ → 10000) =====
    sale_volume: Mapped[str | None] = mapped_column(String(32), nullable=True)
@@ -13,6 +13,21 @@ from pydantic import BaseModel, Field
 FeedStatus = Literal["ok", "empty", "degraded"]


+# ───────────────── 头图传输优化（统一缩放 + 转 WebP） ─────────────────
+# 美团图片 CDN 支持在 URL 后拼 @<w>w_<h>h_1e_1c[.webp] 让服务端缩放/转码。feed 卡片只占
+# 124dp(≈372px@3x),而库里 head_url 是原图(实测中位 137KB、长尾到 17MB、GIF 均 4.8MB),
+# 是首页 feed 图片加载偏慢的来源。按产品要求,头图**统一**缩到 124dp + 转 WebP 再下发
+# (全部压缩、不设大小阈值),由美团 CDN 服务端缩放/转码,实测省 76–99% 体积。
+FEED_THUMB_PARAM = "@375w_375h_1e_1c.webp"          # 124dp@3x 缩略 + 转 WebP
+
+
+def feed_image_url(head_url: str) -> str:
+    """统一给头图 URL 拼缩放参数(缩到 124dp + 转 WebP),让美团 CDN 出小图;空 URL 原样返回。"""
+    if not head_url:
+        return head_url
+    return head_url.split("@")[0] + FEED_THUMB_PARAM
+
+
 # ───────────────── 券卡片（归一化后给客户端） ─────────────────

 class CouponCard(BaseModel):
@@ -94,7 +109,7 @@ class CouponCard(BaseModel):
            platform=platform,
            biz_line=biz_line,
            name=cpd.get("name") or "",
-            head_image_url=head_url,
+            head_image_url=feed_image_url(head_url),
            brand_name=brand.get("brandName"),
            brand_logo_url=brand.get("brandLogoUrl"),
            sell_price=sell,
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# 美团券头图大小/类型回填 —— 全量重跑一轮 ETL,把 meituan_coupon.image_size / image_type 灌满。
+# 上线本 PR 后在服务器手动跑一次,立即回填存量券(不必等定时器逐轮 upsert 自然补齐)。
+#
+# 用法(服务器, root):
+#   cd /opt/shaguabijia-app-server
+#   sudo bash deploy/backfill_image_meta.sh
+#   # 全量一轮 ~70min,建议在 tmux/screen 里跑,防 SSH 断开中断
+#
+# 前提:已拉到含本 PR 的 main、已跑 `alembic upgrade head`(脚本会自检列是否存在)、
+#       已 restart shaguabijia-app-server.service。服务器 MT_CPS_PROXY 留空(直连 CDN)。
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/opt/shaguabijia-app-server}"
+PY="$APP_DIR/.venv/bin/python"
+cd "$APP_DIR"
+mkdir -p data
+LOG="data/etl_backfill_$(date +%Y%m%d_%H%M%S).log"
+
+# 退出时务必恢复定时器(即使 ETL 中途失败,也不把 timer 留在停用态)
+restore_timer() { systemctl start meituan-etl.timer 2>/dev/null || true; }
+trap restore_timer EXIT
+
+echo "[0/3] 自检 image_size/image_type 列(缺列说明没迁移,先跑 alembic upgrade head)..."
+"$PY" -c "from sqlalchemy import inspect; from app.db.session import engine; \
+cols=[c['name'] for c in inspect(engine).get_columns('meituan_coupon')]; \
+import sys; (('image_size' in cols) and ('image_type' in cols)) or sys.exit('  ❌ 缺列,请先: sudo .venv/bin/python -m alembic upgrade head'); \
+print('  ✓ 列已就绪')"
+
+echo "[1/3] 停定时器,防与定时轮次并发占锁..."
+systemctl stop meituan-etl.timer 2>/dev/null || true
+
+echo "[2/3] 全量重跑 ETL(~70min;日志 $LOG)..."
+# 不走 systemctl:meituan-etl.service 有 20min 硬超时,全量一轮跑不完会被掐断。
+"$PY" -m scripts.pull_meituan_coupons --once --prune-hours 24 2>&1 | tee "$LOG"
+
+echo "[3/3] 验证填充率..."
+"$PY" - <<'PYEOF'
+from sqlalchemy import text
+from app.db.session import engine
+engine.echo = False
+with engine.connect() as c:
+    tot = c.execute(text("SELECT count(*) FROM meituan_coupon")).scalar()
+    enr = c.execute(text("SELECT count(*) FROM meituan_coupon WHERE image_size IS NOT NULL")).scalar()
+    gif = c.execute(text("SELECT count(*) FROM meituan_coupon WHERE image_type='image/gif'")).scalar()
+    print(f"  总 {tot} 行 | 有 image_size {enr} ({enr*100//max(tot,1)}%) | GIF {gif} 张")
+PYEOF
+echo "✅ 回填完成(定时器已恢复)。"
@@ -173,7 +173,7 @@
 | `platform` | int | `1`=外卖/到家, `2`=到店 |
 | `biz_line` | int \| null | 到店子类:1到餐 2到综 3酒店 4门票 |
 | `name` | string | 商品名 |
-| `head_image_url` | string | 头图 |
+| `head_image_url` | string | 头图;后端已**统一**拼缩放参数 `@375w_375h_1e_1c.webp`(美团 CDN 服务端缩到 124dp + 转 WebP,实测省 76–99%),客户端直接用 |
 | `brand_name` | string \| null | 品牌名 |
 | `brand_logo_url` | string \| null | 品牌 logo |
 | `sell_price` | string | 现价 |
@@ -52,6 +52,7 @@ try:
 except Exception:  # noqa: BLE001
    pass

+import httpx  # noqa: E402
 from sqlalchemy import delete, func, select  # noqa: E402
 from sqlalchemy.dialects.postgresql import insert as pg_insert  # noqa: E402

@@ -85,6 +86,13 @@ DEFAULT_CONCURRENCY = 12   # 并发城市数(实测 15 并发 402 占 3% 可退
 STARTUP_STAGGER = 0.3      # 首批城市启动错峰间隔秒(削平瞬时峰值,实测能压低 402)
 PRUNE_FAIL_RATIO_MAX = 0.05  # 失败城占比超此值则本轮跳过 prune(避免大面积抓取失败误删库)

+# 头图大小/类型富集:每城抓完后,按 head_url 去重并发 HEAD 取 Content-Length / Content-Type,
+# 写进 image_size / image_type。图片走美团图片 CDN(p*.meituan.net / img.meituan.net),直连可达、
+# 与 CPS 网关 402 限流无关;任何失败 → None,不阻断入库。城市级已 12 并发,这里每城再开 8,
+# 叠加峰值 ~96 个 HEAD,CDN 扛得住(实测单进程 50 并发稳)。
+IMAGE_META_WORKERS = 8     # 每城 HEAD 并发数
+IMAGE_META_TIMEOUT = 8.0   # 单图 HEAD 超时秒
+
 SOURCES = [
    {"code": "search_waimai", "label": "外卖·搜外卖", "kind": "search", "platform": 1, "keyword": "外卖"},
    {"code": "search_meishi", "label": "外卖·搜美食", "kind": "search", "platform": 1, "keyword": "美食"},
@@ -256,6 +264,9 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None:
        "sell_price_cents": price_cents,
        "original_price_cents": _to_cents(cpd.get("originalPrice")),
        "head_url": ((cpd.get("headUrl") or "").split("@")[0][:512] or None),
+        # 占位;由 _fill_image_meta 抓完本城后并发 HEAD 回填(键必须在,upsert 才会带上这两列)
+        "image_size": None,
+        "image_type": None,
        "sale_volume": cpd.get("saleVolume"),
        "sale_volume_num": _sale_volume_num(cpd.get("saleVolume")),
        "commission_percent": comm_pct,
@@ -268,6 +279,38 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None:
    })


+def _head_image_meta(client: httpx.Client, url: str) -> tuple[int | None, str | None]:
+    """HEAD 单张头图,取 (字节大小, MIME 类型);任何失败返回 (None, None),不阻断入库。"""
+    try:
+        r = client.head(url, timeout=IMAGE_META_TIMEOUT, follow_redirects=True)
+        if r.status_code >= 400:
+            return None, None
+        cl = r.headers.get("content-length")
+        size = int(cl) if cl and cl.isdigit() else None
+        ctype = (r.headers.get("content-type") or "").split(";")[0].strip()[:32] or None
+        return size, ctype
+    except Exception:  # noqa: BLE001
+        return None, None
+
+
+def _fill_image_meta(rows: list[dict]) -> None:
+    """就地为本城 rows 填 image_size / image_type:按 head_url 去重后并发 HEAD,再回填每行。
+    trust_env=False 直连图片 CDN(绕本机代理,CDN 无需走 CPS 代理)。"""
+    urls = {r["head_url"] for r in rows if r.get("head_url")}
+    if not urls:
+        return
+    meta: dict[str, tuple[int | None, str | None]] = {}
+    with httpx.Client(trust_env=False, timeout=IMAGE_META_TIMEOUT) as client:
+        with ThreadPoolExecutor(max_workers=IMAGE_META_WORKERS) as pool:
+            futs = {pool.submit(_head_image_meta, client, u): u for u in urls}
+            for f in as_completed(futs):
+                meta[futs[f]] = f.result()
+    for r in rows:
+        m = meta.get(r.get("head_url"))
+        if m:
+            r["image_size"], r["image_type"] = m
+
+
 def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> list[dict]:
    """抓单个城市的 3 路券并解析。worker 线程内执行(只抓取+解析,不碰 DB)。

@@ -286,6 +329,7 @@ def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) ->
            p = _parse_item(it, src, cid)
            if p:
                parsed.append(p)
+    _fill_image_meta(parsed)  # 抓完本城 → 并发 HEAD 回填头图 image_size/image_type
    return parsed