dish-image-toolkit/crawler/scrape_bing.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Bing 图片菜品爬虫(每菜多图版,下厨房限速后切换的备用源)。

每道菜去 Bing 图片搜索,取**前 N 张**结果,下载 Bing CDN 缩略图(turl,ts*.mm.bing.net
极耐爬/统一可靠;原图 murl 也一并记录备用),每菜一行 JSON 到 jsonl,images:[...] 记录
该菜对应的所有图。与 scrape_xiachufang.py 共用 dishes.jsonl + images/:
  - 续抓只跳过 found:true 的菜(失败的会重试)→ 自动只补下厨房没爬到的 ~2082 道
  - 同样的并行 + 全局自适应限速闸(任一线程被限速→全员冷却,根治退避雪崩)

用法:
  python scrape_bing.py --out dishes.jsonl --download --per-dish 3 --workers 8 --input todo.txt
"""
from __future__ import annotations

import argparse
import html
import json
import os
import random
import re
import sys
import threading
import time
import urllib.parse
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone

import requests

for _s in (sys.stdout, sys.stderr):
    try:
        _s.reconfigure(encoding="utf-8", errors="replace")
    except Exception:
        pass

SEARCH_URL = "https://cn.bing.com/images/search"

UA_POOL = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Edg/123.0.0.0",
]
ACCEPT_LANGS = ["zh-CN,zh;q=0.9,en;q=0.8", "zh-CN,zh;q=0.9", "zh-CN,zh;q=0.8,en-US;q=0.5"]


def build_headers() -> dict:
    return {
        "User-Agent": random.choice(UA_POOL),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": random.choice(ACCEPT_LANGS),
        "Referer": "https://cn.bing.com/images/",
    }


_tls = threading.local()


def session() -> requests.Session:
    s = getattr(_tls, "s", None)
    if s is None:
        s = _tls.s = requests.Session()
    return s


# ---------- 全局限速治理(同 scrape_xiachufang:节流 + 全员冷却闸,根治雪崩) ----------
_pace_lock = threading.Lock()
_next_search = 0.0
_resume_at = 0.0
_penalty = 0
SEARCH_SPACING = 0.5  # Bing 耐爬,默认间隔小;可被 --spacing 覆盖


def search_pace() -> None:
    global _next_search
    with _pace_lock:
        now = time.time()
        slot = max(now, _next_search)
        _next_search = slot + SEARCH_SPACING + random.uniform(0, 0.3)
        wait = slot - now
    if wait > 0:
        time.sleep(wait)


def gate_wait() -> None:
    while True:
        with _pace_lock:
            wait = _resume_at - time.time()
        if wait <= 0:
            return
        time.sleep(min(wait, 3) + random.uniform(0, 0.4))


def trip_limit() -> None:
    global _resume_at, _penalty
    with _pace_lock:
        _penalty += 1
        cool = min(45 * (1.5 ** (_penalty - 1)), 600)
        nr = time.time() + cool
        if nr > _resume_at:
            _resume_at = nr
            print(f"  [!] 限速#{_penalty} 全局冷却 {cool:.0f}s", file=sys.stderr, flush=True)


def ease_limit() -> None:
    global _penalty
    if _penalty:
        with _pace_lock:
            _penalty = max(0, _penalty - 1)


def fetch_search(dish: str, proxy: str | None, retries: int = 5, timeout: int = 25) -> str | None:
    proxies = {"http": proxy, "https": proxy} if proxy else None
    params = {"q": dish, "first": 1, "count": 35}
    url = SEARCH_URL + "?" + urllib.parse.urlencode(params)
    for attempt in range(retries):
        gate_wait()
        search_pace()
        try:
            r = session().get(url, headers=build_headers(), proxies=proxies, timeout=timeout)
            if r.status_code == 200:
                ease_limit()
                return r.text
            if r.status_code in (403, 429, 503):
                trip_limit()
            else:
                print(f"  [!] 状态 {r.status_code}", file=sys.stderr)
                time.sleep(3 * (attempt + 1))
        except requests.RequestException as e:
            print(f"  [!] 请求异常 {e}", file=sys.stderr)
            time.sleep(5 + random.uniform(0, 5))
    return None


def parse_images(html_text: str, n: int) -> list[dict]:
    """从 Bing 结果页抽前 n 张图的 {image_url(=turl), murl, turl, title, host_page}。"""
    out: list[dict] = []
    seen: set[str] = set()
    for m in re.finditer(r'class="iusc"[^>]+?\sm="([^"]+)"', html_text):
        try:
            d = json.loads(html.unescape(m.group(1)))
        except Exception:
            continue
        url = d.get("turl") or d.get("murl")
        if not url or url in seen:
            continue
        seen.add(url)
        out.append({
            "image_url": url,
            "murl": d.get("murl"),
            "turl": d.get("turl"),
            "title": (d.get("t") or "").strip() or None,
            "host_page": d.get("purl"),
        })
        if len(out) >= n:
            break
    return out


def is_image(b: bytes) -> bool:
    return (b[:3] == b"\xff\xd8\xff" or b[:8] == b"\x89PNG\r\n\x1a\n"
            or b[:6] in (b"GIF87a", b"GIF89a")
            or (b[:4] == b"RIFF" and b[8:12] == b"WEBP") or b[:2] == b"BM")


def safe_name(dish: str) -> str:
    return re.sub(r"[^\w一-鿿]+", "_", dish).strip("_") or "dish"


def download(url: str, dish: str, idx: int, img_dir: str,
             proxy: str | None) -> tuple[str | None, int]:
    proxies = {"http": proxy, "https": proxy} if proxy else None
    headers = {"User-Agent": random.choice(UA_POOL), "Referer": "https://cn.bing.com/",
               "Accept": "image/avif,image/webp,image/png,image/*;q=0.8,*/*;q=0.5"}
    try:
        r = session().get(url, headers=headers, proxies=proxies, timeout=20)
        if r.status_code == 200 and len(r.content) >= 800 and is_image(r.content):
            path = os.path.join(img_dir, f"{safe_name(dish)}_{idx:02d}.jpg")
            with open(path, "wb") as f:
                f.write(r.content)
            rel = os.path.relpath(path, os.path.dirname(img_dir) or ".").replace(os.sep, "/")
            return rel, len(r.content)
    except requests.RequestException as e:
        print(f"  [!] 下图失败 {e}", file=sys.stderr)
    return None, 0


def scrape_one(dish: str, proxy: str | None, do_download: bool, img_dir: str,
               per_dish: int, jitter: float = 0.0) -> dict:
    if jitter:
        time.sleep(random.uniform(0, jitter))
    now = datetime.now(timezone.utc).isoformat(timespec="seconds")
    base = {"dish": dish, "source": "bing", "fetched_at": now}
    html_text = fetch_search(dish, proxy)
    if html_text is None:
        return {**base, "found": False, "count": 0, "images": [], "reason": "fetch_failed"}
    recs = parse_images(html_text, per_dish)
    if not recs:
        return {**base, "found": False, "count": 0, "images": [], "reason": "no_result"}
    images = []
    for i, rec in enumerate(recs, 1):
        e = dict(rec)
        if do_download:
            rel, size = download(rec["image_url"], dish, i, img_dir, proxy)
            e["image_file"] = rel
            e["bytes"] = size
            if i < len(recs):
                time.sleep(random.uniform(0.05, 0.2))
        images.append(e)
    saved = sum(1 for e in images if not do_download or e.get("image_file"))
    if do_download and saved == 0:
        return {**base, "found": False, "count": 0, "images": images, "reason": "download_failed"}
    return {**base, "found": True, "count": saved, "images": images}


def load_done(out_path: str) -> set[str]:
    done = set()
    if os.path.exists(out_path):
        with open(out_path, encoding="utf-8") as f:
            for line in f:
                try:
                    rec = json.loads(line)
                    if rec.get("found"):  # 只跳过成功的;失败行会重试
                        done.add(rec["dish"])
                except Exception:
                    pass
    return done


def main():
    global SEARCH_SPACING
    ap = argparse.ArgumentParser()
    ap.add_argument("dishes", nargs="*")
    ap.add_argument("--input", help="菜名文件,一行一道")
    ap.add_argument("--out", default="dishes.jsonl")
    ap.add_argument("--download", action="store_true")
    ap.add_argument("--img-dir", default="images")
    ap.add_argument("--per-dish", type=int, default=3)
    ap.add_argument("--proxy", default=os.environ.get("CRAWL_PROXY"))
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--jitter", type=float, default=1.0)
    ap.add_argument("--spacing", type=float, default=SEARCH_SPACING)
    args = ap.parse_args()
    SEARCH_SPACING = args.spacing

    dishes = list(args.dishes)
    if args.input:
        with open(args.input, encoding="utf-8") as f:
            dishes += [ln.strip() for ln in f if ln.strip()]
    if not dishes:
        ap.error("没给菜名")
    dishes = list(dict.fromkeys(dishes))

    os.makedirs(args.img_dir, exist_ok=True)
    done = load_done(args.out)
    todo = [d for d in dishes if d not in done]
    print(f"共 {len(dishes)} 道,已完成 {len(done)},本次待抓 {len(todo)},并行 {args.workers} 线程,源 Bing",
          flush=True)
    if not todo:
        print("没有待抓的菜。")
        return

    ok = miss = 0
    t0 = time.time()
    lock = threading.Lock()
    with open(args.out, "a", encoding="utf-8") as fout, \
            ThreadPoolExecutor(max_workers=args.workers) as ex:
        futs = {ex.submit(scrape_one, d, args.proxy, args.download,
                          args.img_dir, args.per_dish, args.jitter): d for d in todo}
        for i, fut in enumerate(as_completed(futs), 1):
            dish = futs[fut]
            try:
                rec = fut.result()
            except Exception as e:
                now = datetime.now(timezone.utc).isoformat(timespec="seconds")
                rec = {"dish": dish, "source": "bing", "found": False, "count": 0,
                       "images": [], "reason": f"error:{type(e).__name__}", "fetched_at": now}
            with lock:
                fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
                fout.flush()
            if rec["found"]:
                ok += 1
            else:
                miss += 1
            if i % 25 == 0 or i == len(todo):
                rate = i / max(time.time() - t0, 1e-6)
                eta = (len(todo) - i) / rate if rate else 0
                print(f"[{i}/{len(todo)}] ✓{ok} ✗{miss}  {rate*60:.0f}道/分  ETA {eta/60:.0f}分",
                      flush=True)
    print(f"\n完成:成功 {ok}  缺图 {miss}  → {args.out}", flush=True)


if __name__ == "__main__":
    main()