diff --git a/scripts/pull_meituan_coupons.py b/scripts/pull_meituan_coupons.py index 3bc2977..3431cc8 100644 --- a/scripts/pull_meituan_coupons.py +++ b/scripts/pull_meituan_coupons.py @@ -202,6 +202,18 @@ def _to_cents(yuan) -> int | None: return None +def _strip_nul(v): + """递归去掉字符串里的 NUL(0x00):PostgreSQL 的 text / jsonb 字段都不接受该字节, + 美团偶有脏数据(某券文本含 NUL)会让整批 upsert 抛 DataError。入库前统一清洗。""" + if isinstance(v, str): + return v.replace(chr(0), "") + if isinstance(v, dict): + return {k: _strip_nul(x) for k, x in v.items()} + if isinstance(v, list): + return [_strip_nul(x) for x in v] + return v + + def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: cpd = item.get("couponPackDetail") or {} br = item.get("brandInfo") or {} @@ -231,7 +243,8 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: dist = None dedup_raw = f"{brand}|{name}|{price_cents}" - return { + # 入库前递归清洗 NUL(美团脏数据偶含 0x00,PostgreSQL text/jsonb 拒绝整批 → DataError) + return _strip_nul({ "source": source["code"], "platform": source["platform"], "biz_line": item.get("bizLine") or cpd.get("bizLine"), @@ -252,7 +265,7 @@ def _parse_item(item: dict, source: dict, city_id: str) -> dict | None: "delivery_distance_m": dist, "dedup_key": hashlib.md5(dedup_raw.encode("utf-8")).hexdigest(), "raw": item, - } + }) def _pull_one_city(city: dict, index: int, concurrency: int, stagger: float) -> list[dict]: @@ -372,7 +385,13 @@ def run_once( fails.append(city["name"]) print(f" [fail] {city['name']}: {type(e).__name__}: {str(e)[:60]}") continue - up, _dup = _upsert(db, parsed, now) + try: + up, _dup = _upsert(db, parsed, now) + except Exception as e: # noqa: BLE001 + db.rollback() # 事务已 abort,必须 rollback 才能继续给下一城复用 Session + fails.append(city["name"]) + print(f" [入库失败] {city['name']}: {type(e).__name__}: {str(e)[:80]}") + continue total_up += up done += 1 if done % 20 == 0 or done == n_city: