feat(meituan-coupon): 采集记录头图大小/类型 + 读取侧缩放口子 + 回填脚本 (#71)

Reviewed-on: #71
Co-authored-by: chenshuobo <chenshuobo@wonderable.ai>
Co-committed-by: chenshuobo <chenshuobo@wonderable.ai>

scripts/pull_meituan_coupons.py

@@ -52,6 +52,7 @@ try:
 except Exception:  # noqa: BLE001
     pass
 
+import httpx  # noqa: E402
 from sqlalchemy import delete, func, select  # noqa: E402
 from sqlalchemy.dialects.postgresql import insert as pg_insert  # noqa: E402
 
@@ -85,6 +86,13 @@ DEFAULT_CONCURRENCY = 12   # 并发城市数(实测 15 并发 402 占 3% 可退
 STARTUP_STAGGER = 0.3      # 首批城市启动错峰间隔秒(削平瞬时峰值,实测能压低 402)
 PRUNE_FAIL_RATIO_MAX = 0.05  # 失败城占比超此值则本轮跳过 prune(避免大面积抓取失败误删库)
 
+# 头图大小/类型富集:每城抓完后,按 head_url 去重并发 HEAD 取 Content-Length / Content-Type,
+# 写进 image_size / image_type。图片走美团图片 CDN(p*.meituan.net / img.meituan.net),直连可达、
+# 与 CPS 网关 402 限流无关;任何失败 → None,不阻断入库。城市级已 12 并发,这里每城再开 8,
+# 叠加峰值 ~96 个 HEAD,CDN 扛得住(实测单进程 50 并发稳)。
+IMAGE_META_WORKERS = 8     # 每城 HEAD 并发数
+IMAGE_META_TIMEOUT = 8.0   # 单图 HEAD 超时秒
+
 SOURCES = [
     {"code": "search_waimai", "label": "外卖·搜外卖", "kind": "search", "platform": 1, "keyword": "外卖"},
     {"code": "search_meishi", "label": "外卖·搜美食", "kind": "search", "platform": 1, "keyword": "美食"},
@@ -256,6 +264,9 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None:
         "sell_price_cents": price_cents,
         "original_price_cents": _to_cents(cpd.get("originalPrice")),
         "head_url": ((cpd.get("headUrl") or "").split("@")[0][:512] or None),
+        # 占位;由 _fill_image_meta 抓完本城后并发 HEAD 回填(键必须在,upsert 才会带上这两列)
+        "image_size": None,
+        "image_type": None,
         "sale_volume": cpd.get("saleVolume"),
         "sale_volume_num": _sale_volume_num(cpd.get("saleVolume")),
         "commission_percent": comm_pct,
@@ -268,6 +279,38 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None:
     })
 
 
+def _head_image_meta(client: httpx.Client, url: str) -> tuple[int | None, str | None]:
+    """HEAD 单张头图,取 (字节大小, MIME 类型);任何失败返回 (None, None),不阻断入库。"""
+    try:
+        r = client.head(url, timeout=IMAGE_META_TIMEOUT, follow_redirects=True)
+        if r.status_code >= 400:
+            return None, None
+        cl = r.headers.get("content-length")
+        size = int(cl) if cl and cl.isdigit() else None
+        ctype = (r.headers.get("content-type") or "").split(";")[0].strip()[:32] or None
+        return size, ctype
+    except Exception:  # noqa: BLE001
+        return None, None
+
+
+def _fill_image_meta(rows: list[dict]) -> None:
+    """就地为本城 rows 填 image_size / image_type:按 head_url 去重后并发 HEAD,再回填每行。
+    trust_env=False 直连图片 CDN(绕本机代理,CDN 无需走 CPS 代理)。"""
+    urls = {r["head_url"] for r in rows if r.get("head_url")}
+    if not urls:
+        return
+    meta: dict[str, tuple[int | None, str | None]] = {}
+    with httpx.Client(trust_env=False, timeout=IMAGE_META_TIMEOUT) as client:
+        with ThreadPoolExecutor(max_workers=IMAGE_META_WORKERS) as pool:
+            futs = {pool.submit(_head_image_meta, client, u): u for u in urls}
+            for f in as_completed(futs):
+                meta[futs[f]] = f.result()
+    for r in rows:
+        m = meta.get(r.get("head_url"))
+        if m:
+            r["image_size"], r["image_type"] = m
+
+
 def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> list[dict]:
     """抓单个城市的 3 路券并解析。worker 线程内执行(只抓取+解析,不碰 DB)。
 
@@ -286,6 +329,7 @@ def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) ->
             p = _parse_item(it, src, cid)
             if p:
                 parsed.append(p)
+    _fill_image_meta(parsed)  # 抓完本城 → 并发 HEAD 回填头图 image_size/image_type
     return parsed