初始化 dish-image:菜品图爬虫 + 图片名称检索平台
- crawler/:豆果(主)/下厨房/Bing 爬虫 + dishes.jsonl(3056 菜→9167 图映射)+ verify_repair 按 URL 重下 - retrieval/:三路检索(BM25 + 本地 BGE-M3 向量 + RRF 融合),FastAPI + 前端;写死图片目录(默认 crawler/images,可 IMAGE_DIR 覆盖)、绑 0.0.0.0 局域网访问、启动自动建索引、服务器 serve 图片 - 图片(1.7G)与向量模型(2.3G)不进 git Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
陈世睿
@@ -0,0 +1,23 @@
|
||||
# crawler — 菜品图爬虫
|
||||
|
||||
给菜名爬多图,记录"菜→哪些图"到 `dishes.jsonl`。
|
||||
|
||||
## 源(优先级从高到低)
|
||||
- **`scrape_douguo.py` — 豆果 App 接口(最终采用,质量最好)**:`POST api.douguo.net/recipe/search`,960px 高清,无需签名。⚠ 图床 `i1.douguo.com` 有防盗链(下约 1100 张后返 HTTP 493),脚本已自动改走镜像 `i1.douguo.net`(http)。
|
||||
- `scrape_xiachufang.py` — 下厨房(菜谱真实图,质量好;但搜索接口硬限速,备用)。
|
||||
- `scrape_bing.py` — Bing 图搜(通用图搜,质量差、很多不是菜品,已弃用)。
|
||||
|
||||
## 用法
|
||||
```
|
||||
python scrape_douguo.py --input 菜名.txt --out dishes.jsonl --download --per-dish 3 --workers 8 --replace
|
||||
```
|
||||
- 图片 → `images/<菜名>_NN.jpg`;映射 → `dishes.jsonl`,每行:
|
||||
`{dish, found, count, images:[{image_url, image_file, ...}], ...}`
|
||||
- **断点续爬**:只跳 `found:true` 的菜,失败的会重试。菜名也可直接作命令行参数传。
|
||||
|
||||
## 工具
|
||||
- `verify_repair.py [--repair]` — 校验 `dishes.jsonl` 里 `image_file` 的完好率;`--repair` 按 `image_url` 重下缺失/损坏图(应对杀软定期清图)。
|
||||
- `probe_status.py` — 统计 `dishes.jsonl` 的成功/失败数。
|
||||
|
||||
## 现状
|
||||
`dishes.jsonl` 已含 **3056 道**(下厨房 974 + 豆果 2082),对应 **9167 张图**。图不在 git 里 —— `--download` 重爬,或把已有 `images/` 拷进来。
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,10 @@
|
||||
import requests, sys
|
||||
H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9", "Referer": "https://www.xiachufang.com/"}
|
||||
kw = sys.argv[1] if len(sys.argv) > 1 else "番茄炒蛋"
|
||||
try:
|
||||
r = requests.get("https://www.xiachufang.com/search/", params={"keyword": kw, "cat": 1001},
|
||||
headers=H, timeout=20)
|
||||
print(r.status_code)
|
||||
except Exception:
|
||||
print("ERR")
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Bing 图片菜品爬虫(每菜多图版,下厨房限速后切换的备用源)。
|
||||
|
||||
每道菜去 Bing 图片搜索,取**前 N 张**结果,下载 Bing CDN 缩略图(turl,ts*.mm.bing.net
|
||||
极耐爬/统一可靠;原图 murl 也一并记录备用),每菜一行 JSON 到 jsonl,images:[...] 记录
|
||||
该菜对应的所有图。与 scrape_xiachufang.py 共用 dishes.jsonl + images/:
|
||||
- 续抓只跳过 found:true 的菜(失败的会重试)→ 自动只补下厨房没爬到的 ~2082 道
|
||||
- 同样的并行 + 全局自适应限速闸(任一线程被限速→全员冷却,根治退避雪崩)
|
||||
|
||||
用法:
|
||||
python scrape_bing.py --out dishes.jsonl --download --per-dish 3 --workers 8 --input todo.txt
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
for _s in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_s.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
SEARCH_URL = "https://cn.bing.com/images/search"
|
||||
|
||||
UA_POOL = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Edg/123.0.0.0",
|
||||
]
|
||||
ACCEPT_LANGS = ["zh-CN,zh;q=0.9,en;q=0.8", "zh-CN,zh;q=0.9", "zh-CN,zh;q=0.8,en-US;q=0.5"]
|
||||
|
||||
|
||||
def build_headers() -> dict:
|
||||
return {
|
||||
"User-Agent": random.choice(UA_POOL),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": random.choice(ACCEPT_LANGS),
|
||||
"Referer": "https://cn.bing.com/images/",
|
||||
}
|
||||
|
||||
|
||||
_tls = threading.local()
|
||||
|
||||
|
||||
def session() -> requests.Session:
|
||||
s = getattr(_tls, "s", None)
|
||||
if s is None:
|
||||
s = _tls.s = requests.Session()
|
||||
return s
|
||||
|
||||
|
||||
# ---------- 全局限速治理(同 scrape_xiachufang:节流 + 全员冷却闸,根治雪崩) ----------
|
||||
_pace_lock = threading.Lock()
|
||||
_next_search = 0.0
|
||||
_resume_at = 0.0
|
||||
_penalty = 0
|
||||
SEARCH_SPACING = 0.5 # Bing 耐爬,默认间隔小;可被 --spacing 覆盖
|
||||
|
||||
|
||||
def search_pace() -> None:
|
||||
global _next_search
|
||||
with _pace_lock:
|
||||
now = time.time()
|
||||
slot = max(now, _next_search)
|
||||
_next_search = slot + SEARCH_SPACING + random.uniform(0, 0.3)
|
||||
wait = slot - now
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
|
||||
|
||||
def gate_wait() -> None:
|
||||
while True:
|
||||
with _pace_lock:
|
||||
wait = _resume_at - time.time()
|
||||
if wait <= 0:
|
||||
return
|
||||
time.sleep(min(wait, 3) + random.uniform(0, 0.4))
|
||||
|
||||
|
||||
def trip_limit() -> None:
|
||||
global _resume_at, _penalty
|
||||
with _pace_lock:
|
||||
_penalty += 1
|
||||
cool = min(45 * (1.5 ** (_penalty - 1)), 600)
|
||||
nr = time.time() + cool
|
||||
if nr > _resume_at:
|
||||
_resume_at = nr
|
||||
print(f" [!] 限速#{_penalty} 全局冷却 {cool:.0f}s", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def ease_limit() -> None:
|
||||
global _penalty
|
||||
if _penalty:
|
||||
with _pace_lock:
|
||||
_penalty = max(0, _penalty - 1)
|
||||
|
||||
|
||||
def fetch_search(dish: str, proxy: str | None, retries: int = 5, timeout: int = 25) -> str | None:
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
params = {"q": dish, "first": 1, "count": 35}
|
||||
url = SEARCH_URL + "?" + urllib.parse.urlencode(params)
|
||||
for attempt in range(retries):
|
||||
gate_wait()
|
||||
search_pace()
|
||||
try:
|
||||
r = session().get(url, headers=build_headers(), proxies=proxies, timeout=timeout)
|
||||
if r.status_code == 200:
|
||||
ease_limit()
|
||||
return r.text
|
||||
if r.status_code in (403, 429, 503):
|
||||
trip_limit()
|
||||
else:
|
||||
print(f" [!] 状态 {r.status_code}", file=sys.stderr)
|
||||
time.sleep(3 * (attempt + 1))
|
||||
except requests.RequestException as e:
|
||||
print(f" [!] 请求异常 {e}", file=sys.stderr)
|
||||
time.sleep(5 + random.uniform(0, 5))
|
||||
return None
|
||||
|
||||
|
||||
def parse_images(html_text: str, n: int) -> list[dict]:
|
||||
"""从 Bing 结果页抽前 n 张图的 {image_url(=turl), murl, turl, title, host_page}。"""
|
||||
out: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for m in re.finditer(r'class="iusc"[^>]+?\sm="([^"]+)"', html_text):
|
||||
try:
|
||||
d = json.loads(html.unescape(m.group(1)))
|
||||
except Exception:
|
||||
continue
|
||||
url = d.get("turl") or d.get("murl")
|
||||
if not url or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
out.append({
|
||||
"image_url": url,
|
||||
"murl": d.get("murl"),
|
||||
"turl": d.get("turl"),
|
||||
"title": (d.get("t") or "").strip() or None,
|
||||
"host_page": d.get("purl"),
|
||||
})
|
||||
if len(out) >= n:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def is_image(b: bytes) -> bool:
|
||||
return (b[:3] == b"\xff\xd8\xff" or b[:8] == b"\x89PNG\r\n\x1a\n"
|
||||
or b[:6] in (b"GIF87a", b"GIF89a")
|
||||
or (b[:4] == b"RIFF" and b[8:12] == b"WEBP") or b[:2] == b"BM")
|
||||
|
||||
|
||||
def safe_name(dish: str) -> str:
|
||||
return re.sub(r"[^\w一-鿿]+", "_", dish).strip("_") or "dish"
|
||||
|
||||
|
||||
def download(url: str, dish: str, idx: int, img_dir: str,
|
||||
proxy: str | None) -> tuple[str | None, int]:
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
headers = {"User-Agent": random.choice(UA_POOL), "Referer": "https://cn.bing.com/",
|
||||
"Accept": "image/avif,image/webp,image/png,image/*;q=0.8,*/*;q=0.5"}
|
||||
try:
|
||||
r = session().get(url, headers=headers, proxies=proxies, timeout=20)
|
||||
if r.status_code == 200 and len(r.content) >= 800 and is_image(r.content):
|
||||
path = os.path.join(img_dir, f"{safe_name(dish)}_{idx:02d}.jpg")
|
||||
with open(path, "wb") as f:
|
||||
f.write(r.content)
|
||||
rel = os.path.relpath(path, os.path.dirname(img_dir) or ".").replace(os.sep, "/")
|
||||
return rel, len(r.content)
|
||||
except requests.RequestException as e:
|
||||
print(f" [!] 下图失败 {e}", file=sys.stderr)
|
||||
return None, 0
|
||||
|
||||
|
||||
def scrape_one(dish: str, proxy: str | None, do_download: bool, img_dir: str,
|
||||
per_dish: int, jitter: float = 0.0) -> dict:
|
||||
if jitter:
|
||||
time.sleep(random.uniform(0, jitter))
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
base = {"dish": dish, "source": "bing", "fetched_at": now}
|
||||
html_text = fetch_search(dish, proxy)
|
||||
if html_text is None:
|
||||
return {**base, "found": False, "count": 0, "images": [], "reason": "fetch_failed"}
|
||||
recs = parse_images(html_text, per_dish)
|
||||
if not recs:
|
||||
return {**base, "found": False, "count": 0, "images": [], "reason": "no_result"}
|
||||
images = []
|
||||
for i, rec in enumerate(recs, 1):
|
||||
e = dict(rec)
|
||||
if do_download:
|
||||
rel, size = download(rec["image_url"], dish, i, img_dir, proxy)
|
||||
e["image_file"] = rel
|
||||
e["bytes"] = size
|
||||
if i < len(recs):
|
||||
time.sleep(random.uniform(0.05, 0.2))
|
||||
images.append(e)
|
||||
saved = sum(1 for e in images if not do_download or e.get("image_file"))
|
||||
if do_download and saved == 0:
|
||||
return {**base, "found": False, "count": 0, "images": images, "reason": "download_failed"}
|
||||
return {**base, "found": True, "count": saved, "images": images}
|
||||
|
||||
|
||||
def load_done(out_path: str) -> set[str]:
|
||||
done = set()
|
||||
if os.path.exists(out_path):
|
||||
with open(out_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
if rec.get("found"): # 只跳过成功的;失败行会重试
|
||||
done.add(rec["dish"])
|
||||
except Exception:
|
||||
pass
|
||||
return done
|
||||
|
||||
|
||||
def main():
|
||||
global SEARCH_SPACING
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("dishes", nargs="*")
|
||||
ap.add_argument("--input", help="菜名文件,一行一道")
|
||||
ap.add_argument("--out", default="dishes.jsonl")
|
||||
ap.add_argument("--download", action="store_true")
|
||||
ap.add_argument("--img-dir", default="images")
|
||||
ap.add_argument("--per-dish", type=int, default=3)
|
||||
ap.add_argument("--proxy", default=os.environ.get("CRAWL_PROXY"))
|
||||
ap.add_argument("--workers", type=int, default=8)
|
||||
ap.add_argument("--jitter", type=float, default=1.0)
|
||||
ap.add_argument("--spacing", type=float, default=SEARCH_SPACING)
|
||||
args = ap.parse_args()
|
||||
SEARCH_SPACING = args.spacing
|
||||
|
||||
dishes = list(args.dishes)
|
||||
if args.input:
|
||||
with open(args.input, encoding="utf-8") as f:
|
||||
dishes += [ln.strip() for ln in f if ln.strip()]
|
||||
if not dishes:
|
||||
ap.error("没给菜名")
|
||||
dishes = list(dict.fromkeys(dishes))
|
||||
|
||||
os.makedirs(args.img_dir, exist_ok=True)
|
||||
done = load_done(args.out)
|
||||
todo = [d for d in dishes if d not in done]
|
||||
print(f"共 {len(dishes)} 道,已完成 {len(done)},本次待抓 {len(todo)},并行 {args.workers} 线程,源 Bing",
|
||||
flush=True)
|
||||
if not todo:
|
||||
print("没有待抓的菜。")
|
||||
return
|
||||
|
||||
ok = miss = 0
|
||||
t0 = time.time()
|
||||
lock = threading.Lock()
|
||||
with open(args.out, "a", encoding="utf-8") as fout, \
|
||||
ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
futs = {ex.submit(scrape_one, d, args.proxy, args.download,
|
||||
args.img_dir, args.per_dish, args.jitter): d for d in todo}
|
||||
for i, fut in enumerate(as_completed(futs), 1):
|
||||
dish = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
rec = {"dish": dish, "source": "bing", "found": False, "count": 0,
|
||||
"images": [], "reason": f"error:{type(e).__name__}", "fetched_at": now}
|
||||
with lock:
|
||||
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
fout.flush()
|
||||
if rec["found"]:
|
||||
ok += 1
|
||||
else:
|
||||
miss += 1
|
||||
if i % 25 == 0 or i == len(todo):
|
||||
rate = i / max(time.time() - t0, 1e-6)
|
||||
eta = (len(todo) - i) / rate if rate else 0
|
||||
print(f"[{i}/{len(todo)}] ✓{ok} ✗{miss} {rate*60:.0f}道/分 ETA {eta/60:.0f}分",
|
||||
flush=True)
|
||||
print(f"\n完成:成功 {ok} 缺图 {miss} → {args.out}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""豆果美食 菜品图爬虫(每菜多图版,最高质量源)。
|
||||
|
||||
走豆果 App JSON 接口 POST api.douguo.net/recipe/search/0/20(无需签名),每菜取前 N 个
|
||||
菜谱的 photo_path(960px 高清真实菜品图),下载到 images/,每菜一行 JSON 到 jsonl。
|
||||
与 scrape_xiachufang/scrape_bing 共用 dishes.jsonl + images/:续抓只跳 found:true。
|
||||
--replace: 下载前先删该菜已有的 <safe>_*.jpg(用于把旧 Bing 图替换成豆果图)。
|
||||
|
||||
用法:
|
||||
python scrape_douguo.py --out dishes.jsonl --download --per-dish 3 --workers 8 --replace --input todo.txt
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
for _s in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_s.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
API = "https://api.douguo.net/recipe/search/0/20"
|
||||
|
||||
UA_POOL = [
|
||||
"Mozilla/5.0 (Linux; Android 9; MI 8 Build/PKQ1.181121.001) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 10; Redmi K30) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.91 Mobile Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
def build_headers() -> dict:
|
||||
return {"User-Agent": random.choice(UA_POOL),
|
||||
"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"}
|
||||
|
||||
|
||||
_tls = threading.local()
|
||||
|
||||
|
||||
def session() -> requests.Session:
|
||||
s = getattr(_tls, "s", None)
|
||||
if s is None:
|
||||
s = _tls.s = requests.Session()
|
||||
return s
|
||||
|
||||
|
||||
# ---------- 全局限速治理(同前:节流 + 全员冷却闸) ----------
|
||||
_pace_lock = threading.Lock()
|
||||
_next_search = 0.0
|
||||
_resume_at = 0.0
|
||||
_penalty = 0
|
||||
SEARCH_SPACING = 0.3 # 豆果接口快且耐用,默认间隔小
|
||||
|
||||
|
||||
def search_pace() -> None:
|
||||
global _next_search
|
||||
with _pace_lock:
|
||||
now = time.time()
|
||||
slot = max(now, _next_search)
|
||||
_next_search = slot + SEARCH_SPACING + random.uniform(0, 0.2)
|
||||
wait = slot - now
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
|
||||
|
||||
def gate_wait() -> None:
|
||||
while True:
|
||||
with _pace_lock:
|
||||
wait = _resume_at - time.time()
|
||||
if wait <= 0:
|
||||
return
|
||||
time.sleep(min(wait, 3) + random.uniform(0, 0.4))
|
||||
|
||||
|
||||
def trip_limit() -> None:
|
||||
global _resume_at, _penalty
|
||||
with _pace_lock:
|
||||
_penalty += 1
|
||||
cool = min(45 * (1.5 ** (_penalty - 1)), 600)
|
||||
nr = time.time() + cool
|
||||
if nr > _resume_at:
|
||||
_resume_at = nr
|
||||
print(f" [!] 限速#{_penalty} 全局冷却 {cool:.0f}s", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def ease_limit() -> None:
|
||||
global _penalty
|
||||
if _penalty:
|
||||
with _pace_lock:
|
||||
_penalty = max(0, _penalty - 1)
|
||||
|
||||
|
||||
def fetch_search(dish: str, proxy: str | None, retries: int = 5) -> dict | None:
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
body = urllib.parse.urlencode({"client": 4, "keyword": dish, "order": 0, "_vs": 400}).encode()
|
||||
for attempt in range(retries):
|
||||
gate_wait()
|
||||
search_pace()
|
||||
try:
|
||||
r = session().post(API, data=body, headers=build_headers(), proxies=proxies, timeout=20)
|
||||
if r.status_code == 200:
|
||||
ease_limit()
|
||||
return r.json()
|
||||
if r.status_code in (403, 429, 503):
|
||||
trip_limit()
|
||||
else:
|
||||
print(f" [!] 状态 {r.status_code}", file=sys.stderr)
|
||||
time.sleep(3 * (attempt + 1))
|
||||
except (requests.RequestException, ValueError) as e:
|
||||
print(f" [!] 请求异常 {type(e).__name__}", file=sys.stderr)
|
||||
time.sleep(5 + random.uniform(0, 5))
|
||||
return None
|
||||
|
||||
|
||||
def parse_images(data: dict | None, n: int) -> list[dict]:
|
||||
recs = ((data or {}).get("result") or {}).get("recipes") or []
|
||||
out: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for rc in recs:
|
||||
url = rc.get("photo_path") or rc.get("thumb_path") or rc.get("image")
|
||||
if not url:
|
||||
continue
|
||||
key = str(rc.get("cook_id") or url)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
# .com 图床有 hotlink/限流(HTTP 493),.net 镜像同图可用,但 .net 证书不匹配→走 http
|
||||
net_url = url.replace("douguo.com", "douguo.net").replace("https://", "http://")
|
||||
out.append({
|
||||
"image_url": net_url,
|
||||
"src_com": url,
|
||||
"title": (rc.get("title") or "").strip() or None,
|
||||
"cook_id": rc.get("cook_id"),
|
||||
})
|
||||
if len(out) >= n:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def is_image(b: bytes) -> bool:
|
||||
return (b[:3] == b"\xff\xd8\xff" or b[:8] == b"\x89PNG\r\n\x1a\n"
|
||||
or b[:6] in (b"GIF87a", b"GIF89a")
|
||||
or (b[:4] == b"RIFF" and b[8:12] == b"WEBP") or b[:2] == b"BM")
|
||||
|
||||
|
||||
def safe_name(dish: str) -> str:
|
||||
return re.sub(r"[^\w一-鿿]+", "_", dish).strip("_") or "dish"
|
||||
|
||||
|
||||
def download(url: str, dish: str, idx: int, img_dir: str,
|
||||
proxy: str | None, retries: int = 4) -> tuple[str | None, int]:
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
headers = {"User-Agent": random.choice(UA_POOL), "Referer": "https://www.douguo.com/",
|
||||
"Accept": "image/avif,image/webp,image/png,image/*;q=0.8,*/*;q=0.5"}
|
||||
for attempt in range(retries):
|
||||
gate_wait() # 图床被限(493/429)时也走全局冷却
|
||||
try:
|
||||
r = session().get(url, headers=headers, proxies=proxies, timeout=25)
|
||||
if r.status_code == 200 and len(r.content) >= 800 and is_image(r.content):
|
||||
ease_limit()
|
||||
path = os.path.join(img_dir, f"{safe_name(dish)}_{idx:02d}.jpg")
|
||||
with open(path, "wb") as f:
|
||||
f.write(r.content)
|
||||
rel = os.path.relpath(path, os.path.dirname(img_dir) or ".").replace(os.sep, "/")
|
||||
return rel, len(r.content)
|
||||
if r.status_code in (403, 429, 493, 503):
|
||||
trip_limit()
|
||||
else:
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
except requests.RequestException as e:
|
||||
print(f" [!] 下图失败 {e}", file=sys.stderr)
|
||||
time.sleep(2)
|
||||
return None, 0
|
||||
|
||||
|
||||
def scrape_one(dish: str, proxy: str | None, do_download: bool, img_dir: str,
|
||||
per_dish: int, jitter: float = 0.0, replace: bool = False) -> dict:
|
||||
if jitter:
|
||||
time.sleep(random.uniform(0, jitter))
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
base = {"dish": dish, "source": "douguo", "fetched_at": now}
|
||||
data = fetch_search(dish, proxy)
|
||||
if data is None:
|
||||
return {**base, "found": False, "count": 0, "images": [], "reason": "fetch_failed"}
|
||||
recs = parse_images(data, per_dish)
|
||||
if not recs:
|
||||
return {**base, "found": False, "count": 0, "images": [], "reason": "no_result"}
|
||||
if do_download and replace: # 先删旧图(把该菜的 Bing 图替换掉)
|
||||
for old in glob.glob(os.path.join(img_dir, safe_name(dish) + "_*.jpg")):
|
||||
try:
|
||||
os.remove(old)
|
||||
except OSError:
|
||||
pass
|
||||
images = []
|
||||
for i, rec in enumerate(recs, 1):
|
||||
e = dict(rec)
|
||||
if do_download:
|
||||
rel, size = download(rec["image_url"], dish, i, img_dir, proxy)
|
||||
e["image_file"] = rel
|
||||
e["bytes"] = size
|
||||
if i < len(recs):
|
||||
time.sleep(random.uniform(0.05, 0.2))
|
||||
images.append(e)
|
||||
saved = sum(1 for e in images if not do_download or e.get("image_file"))
|
||||
if do_download and saved == 0:
|
||||
return {**base, "found": False, "count": 0, "images": images, "reason": "download_failed"}
|
||||
return {**base, "found": True, "count": saved, "images": images}
|
||||
|
||||
|
||||
def load_done(out_path: str) -> set[str]:
|
||||
done = set()
|
||||
if os.path.exists(out_path):
|
||||
with open(out_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
if rec.get("found"):
|
||||
done.add(rec["dish"])
|
||||
except Exception:
|
||||
pass
|
||||
return done
|
||||
|
||||
|
||||
def main():
|
||||
global SEARCH_SPACING
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("dishes", nargs="*")
|
||||
ap.add_argument("--input")
|
||||
ap.add_argument("--out", default="dishes.jsonl")
|
||||
ap.add_argument("--download", action="store_true")
|
||||
ap.add_argument("--img-dir", default="images")
|
||||
ap.add_argument("--per-dish", type=int, default=3)
|
||||
ap.add_argument("--proxy", default=os.environ.get("CRAWL_PROXY"))
|
||||
ap.add_argument("--workers", type=int, default=8)
|
||||
ap.add_argument("--jitter", type=float, default=0.8)
|
||||
ap.add_argument("--spacing", type=float, default=SEARCH_SPACING)
|
||||
ap.add_argument("--replace", action="store_true", help="下载前删该菜已有图(替换旧源)")
|
||||
args = ap.parse_args()
|
||||
SEARCH_SPACING = args.spacing
|
||||
|
||||
dishes = list(args.dishes)
|
||||
if args.input:
|
||||
with open(args.input, encoding="utf-8") as f:
|
||||
dishes += [ln.strip() for ln in f if ln.strip()]
|
||||
if not dishes:
|
||||
ap.error("没给菜名")
|
||||
dishes = list(dict.fromkeys(dishes))
|
||||
|
||||
os.makedirs(args.img_dir, exist_ok=True)
|
||||
done = load_done(args.out)
|
||||
todo = [d for d in dishes if d not in done]
|
||||
print(f"共 {len(dishes)} 道,已完成 {len(done)},本次待抓 {len(todo)},并行 {args.workers} 线程,源 豆果",
|
||||
flush=True)
|
||||
if not todo:
|
||||
print("没有待抓的菜。")
|
||||
return
|
||||
|
||||
ok = miss = 0
|
||||
t0 = time.time()
|
||||
lock = threading.Lock()
|
||||
with open(args.out, "a", encoding="utf-8") as fout, \
|
||||
ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
futs = {ex.submit(scrape_one, d, args.proxy, args.download, args.img_dir,
|
||||
args.per_dish, args.jitter, args.replace): d for d in todo}
|
||||
for i, fut in enumerate(as_completed(futs), 1):
|
||||
dish = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
rec = {"dish": dish, "source": "douguo", "found": False, "count": 0,
|
||||
"images": [], "reason": f"error:{type(e).__name__}", "fetched_at": now}
|
||||
with lock:
|
||||
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
fout.flush()
|
||||
if rec["found"]:
|
||||
ok += 1
|
||||
else:
|
||||
miss += 1
|
||||
if i % 25 == 0 or i == len(todo):
|
||||
rate = i / max(time.time() - t0, 1e-6)
|
||||
eta = (len(todo) - i) / rate if rate else 0
|
||||
print(f"[{i}/{len(todo)}] ✓{ok} ✗{miss} {rate*60:.0f}道/分 ETA {eta/60:.0f}分",
|
||||
flush=True)
|
||||
print(f"\n完成:成功 {ok} 缺图 {miss} → {args.out}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,375 @@
|
||||
#!/usr/bin/env python3
|
||||
"""下厨房菜品图爬虫(每菜多图版)。
|
||||
|
||||
给一批菜名,逐个去下厨房搜索,取**前 N 个 recipe 的首图**(真图在 data-src,不是 src,
|
||||
N 由 --per-dish 控制,默认 3 → 即同名菜的不同做法各一张),每道菜输出一行 JSON 到 jsonl,
|
||||
行内 images:[...] 记录该菜对应的所有图(文件名 + 原图 URL + 做法标题/链接)。
|
||||
|
||||
反爬策略(用户要求"悠着点"):
|
||||
- 每请求随机换 User-Agent + Accept-Language + 轻微变化的 header 组合
|
||||
- 请求间随机延迟(默认 4~9s),每 N 条额外长歇一次
|
||||
- 非 200 / 异常 → 指数退避重试;疑似被封(403/503/429)歇更久
|
||||
- 不持久化 cookie(每次请求像"新访客",避免同 cookie 配不同 UA 的机器人特征)
|
||||
- 代理可选(--proxy / 文件轮换);国内站默认直连
|
||||
|
||||
断点续抓:已在输出 jsonl 里的菜名自动跳过。
|
||||
|
||||
用法:
|
||||
python3 scrape_xiachufang.py --out dishes.jsonl --download --per-dish 3 鸡尾酒 布朗尼 牛肉拉面
|
||||
python3 scrape_xiachufang.py --out dishes.jsonl --download --per-dish 3 --input dishes.txt
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
# Windows 控制台默认 GBK,无法输出 ✓/✗/… 等字符 → 强制 UTF-8(长跑任务必须稳)
|
||||
for _s in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_s.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
SEARCH_URL = "https://www.xiachufang.com/search/"
|
||||
|
||||
# 真实浏览器 UA 池(桌面 + 移动混合),每请求随机取一条
|
||||
UA_POOL = [
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1",
|
||||
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/536.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Edg/123.0.0.0",
|
||||
]
|
||||
ACCEPT_LANGS = [
|
||||
"zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"zh-CN,zh;q=0.9",
|
||||
"zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
|
||||
]
|
||||
|
||||
|
||||
def build_headers(referer: str = "https://www.xiachufang.com/") -> dict:
|
||||
"""每次随机组装 header,制造"不同访客"特征。"""
|
||||
h = {
|
||||
"User-Agent": random.choice(UA_POOL),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": random.choice(ACCEPT_LANGS),
|
||||
"Referer": referer,
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
# 随机带或不带这些可选头,进一步打散指纹
|
||||
if random.random() < 0.6:
|
||||
h["DNT"] = "1"
|
||||
if random.random() < 0.5:
|
||||
h["Sec-Fetch-Dest"] = "document"
|
||||
h["Sec-Fetch-Mode"] = "navigate"
|
||||
h["Sec-Fetch-Site"] = "same-origin"
|
||||
return h
|
||||
|
||||
|
||||
# 每线程独立 Session(连接复用更快,并行更稳;requests.Session 非线程安全→线程本地)
|
||||
_tls = threading.local()
|
||||
|
||||
|
||||
def session() -> requests.Session:
|
||||
s = getattr(_tls, "s", None)
|
||||
if s is None:
|
||||
s = _tls.s = requests.Session()
|
||||
return s
|
||||
|
||||
|
||||
# ---------- 全局限速治理(根治多线程退避雪崩) ----------
|
||||
# 1) 搜索节流:全局把"搜索请求"按最小间隔排队(限速的是搜索域名,不是图片CDN)
|
||||
# 2) 冷却闸:任一线程被 429/503 → 设全局冷却,所有线程一起等,避免罚时还有线程乱戳
|
||||
_pace_lock = threading.Lock()
|
||||
_next_search = 0.0 # 下一次允许搜索的时刻
|
||||
_resume_at = 0.0 # 全局冷却到期时刻
|
||||
_penalty = 0 # 连续被限速次数(自适应升级/消解)
|
||||
SEARCH_SPACING = 2.0 # 全局两次搜索最小间隔(秒),可被 --spacing 覆盖
|
||||
|
||||
|
||||
def search_pace() -> None:
|
||||
"""全局给搜索请求排队,保证两次搜索至少间隔 SEARCH_SPACING 秒。"""
|
||||
global _next_search
|
||||
with _pace_lock:
|
||||
now = time.time()
|
||||
slot = max(now, _next_search)
|
||||
_next_search = slot + SEARCH_SPACING + random.uniform(0, 0.5)
|
||||
wait = slot - now
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
|
||||
|
||||
def gate_wait() -> None:
|
||||
"""被限速期间所有线程在此一起等到冷却结束。"""
|
||||
while True:
|
||||
with _pace_lock:
|
||||
wait = _resume_at - time.time()
|
||||
if wait <= 0:
|
||||
return
|
||||
time.sleep(min(wait, 3) + random.uniform(0, 0.4))
|
||||
|
||||
|
||||
def trip_limit() -> None:
|
||||
"""被 429/503 时触发全局冷却,连续触发自适应升级(60s→封顶600s)。"""
|
||||
global _resume_at, _penalty
|
||||
with _pace_lock:
|
||||
_penalty += 1
|
||||
cool = min(60 * (1.5 ** (_penalty - 1)), 600)
|
||||
nr = time.time() + cool
|
||||
if nr > _resume_at:
|
||||
_resume_at = nr
|
||||
print(f" [!] 限速#{_penalty} 全局冷却 {cool:.0f}s", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def ease_limit() -> None:
|
||||
"""成功一次就消解一点惩罚,持续顺利时自动恢复速度。"""
|
||||
global _penalty
|
||||
if _penalty:
|
||||
with _pace_lock:
|
||||
_penalty = max(0, _penalty - 1)
|
||||
|
||||
|
||||
def fetch(url: str, proxy: str | None, retries: int = 6, timeout: int = 25) -> str | None:
|
||||
"""GET 搜索页:全局节流 + 全局冷却闸 + 自适应重试。"""
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
for attempt in range(retries):
|
||||
gate_wait() # 冷却期一起等
|
||||
search_pace() # 全局排队,控制搜索速率
|
||||
try:
|
||||
resp = session().get(
|
||||
url, headers=build_headers(), proxies=proxies, timeout=timeout
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
ease_limit()
|
||||
return resp.text
|
||||
if resp.status_code in (403, 429, 503):
|
||||
trip_limit() # 设全局冷却,下轮 gate_wait 一起等
|
||||
else:
|
||||
print(f" [!] 状态 {resp.status_code}", file=sys.stderr)
|
||||
time.sleep(5 * (attempt + 1))
|
||||
except requests.RequestException as e:
|
||||
print(f" [!] 请求异常 {e}", file=sys.stderr)
|
||||
time.sleep(8 + random.uniform(0, 5))
|
||||
return None
|
||||
|
||||
|
||||
def normalize_img(url: str, width: int = 600) -> str:
|
||||
"""把 215px 缩略图参数换成更大尺寸。chuimg 走七牛 imageView2:
|
||||
base.jpg?imageView2/2/w/600 = 等比缩到宽 600。"""
|
||||
base = url.split("?", 1)[0]
|
||||
return f"{base}?imageView2/2/w/{width}/interlace/1/q/85"
|
||||
|
||||
|
||||
def parse_top_recipes(html: str, n: int) -> list[dict]:
|
||||
"""从搜索结果页抽**前 n 个** recipe 的 {image_url, recipe_title, recipe_url}。
|
||||
一个搜索页通常有 ~15 个 recipe,各带一张真图 → 即每菜多图(同名菜的不同做法)。
|
||||
无结果 / 结构变化 → []。"""
|
||||
# 先定位结果列表区,避免误抓页面装饰图(IE 提示图等)
|
||||
start = html.find('class="normal-recipe-list"')
|
||||
if start == -1:
|
||||
return []
|
||||
region = html[start:]
|
||||
|
||||
out: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
# 逐个 recipe 块:<a href="/recipe/ID/">...内含 <img data-src=chuimg alt=标题>...</a>
|
||||
for m in re.finditer(r'<a href="(/recipe/(\d+)/)"[^>]*>(.*?)</a>', region, re.S):
|
||||
block = m.group(0)
|
||||
img = re.search(r'data-src="([^"]+chuimg[^"]*)"', block)
|
||||
if not img:
|
||||
continue
|
||||
rid = m.group(2)
|
||||
if rid in seen: # 同一 recipe 在页面可能出现两次,去重
|
||||
continue
|
||||
seen.add(rid)
|
||||
image_raw = img.group(1)
|
||||
alt = re.search(r'alt="([^"]*)"', block)
|
||||
out.append({
|
||||
"image_url": normalize_img(image_raw),
|
||||
"image_url_raw": image_raw,
|
||||
"recipe_title": (alt.group(1).strip() if alt else None),
|
||||
"recipe_url": "https://www.xiachufang.com" + m.group(1),
|
||||
})
|
||||
if len(out) >= n:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def safe_name(dish: str) -> str:
|
||||
return re.sub(r"[^\w一-鿿]+", "_", dish).strip("_") or "dish"
|
||||
|
||||
|
||||
def download_image(url: str, dish: str, idx: int, img_dir: str,
|
||||
proxy: str | None) -> tuple[str | None, int]:
|
||||
"""下图到 img_dir/<safe>_<idx>.jpg。chuimg 可能校验 Referer,带上。
|
||||
返回 (相对路径, 字节数);失败 (None, 0)。"""
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
headers = build_headers()
|
||||
headers["Referer"] = "https://www.xiachufang.com/"
|
||||
headers["Accept"] = "image/avif,image/webp,image/png,image/*;q=0.8,*/*;q=0.5"
|
||||
try:
|
||||
r = session().get(url, headers=headers, proxies=proxies, timeout=25)
|
||||
if r.status_code == 200 and r.content:
|
||||
path = os.path.join(img_dir, f"{safe_name(dish)}_{idx:02d}.jpg")
|
||||
with open(path, "wb") as f:
|
||||
f.write(r.content)
|
||||
rel = os.path.relpath(path, os.path.dirname(img_dir) or ".").replace(os.sep, "/")
|
||||
return rel, len(r.content)
|
||||
except requests.RequestException as e:
|
||||
print(f" [!] 下图失败 {e}", file=sys.stderr)
|
||||
return None, 0
|
||||
|
||||
|
||||
def scrape_one(dish: str, proxy: str | None, do_download: bool, img_dir: str,
|
||||
per_dish: int, jitter: float = 0.0) -> dict:
|
||||
if jitter: # 并行时每个任务起点随机错开,避免同时齐刷刷打过去
|
||||
time.sleep(random.uniform(0, jitter))
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
url = SEARCH_URL + "?" + urllib.parse.urlencode({"keyword": dish, "cat": 1001})
|
||||
html = fetch(url, proxy)
|
||||
if html is None:
|
||||
return {"dish": dish, "found": False, "count": 0, "images": [],
|
||||
"reason": "fetch_failed", "fetched_at": now}
|
||||
recs = parse_top_recipes(html, per_dish)
|
||||
if not recs:
|
||||
return {"dish": dish, "found": False, "count": 0, "images": [],
|
||||
"reason": "no_result", "fetched_at": now}
|
||||
|
||||
images = []
|
||||
for i, rec in enumerate(recs, 1):
|
||||
entry = dict(rec)
|
||||
if do_download:
|
||||
rel, size = download_image(rec["image_url"], dish, i, img_dir, proxy)
|
||||
entry["image_file"] = rel
|
||||
entry["bytes"] = size
|
||||
if i < len(recs): # 同菜多图之间轻微间隔,别猛打 CDN
|
||||
time.sleep(random.uniform(0.1, 0.3))
|
||||
images.append(entry)
|
||||
saved = sum(1 for e in images if not do_download or e.get("image_file"))
|
||||
return {"dish": dish, "found": True, "count": saved, "images": images,
|
||||
"fetched_at": now}
|
||||
|
||||
|
||||
def load_done(out_path: str) -> set[str]:
|
||||
done = set()
|
||||
if os.path.exists(out_path):
|
||||
with open(out_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
if rec.get("found"): # 只把成功的算"已完成";失败行会被重试
|
||||
done.add(rec["dish"])
|
||||
except Exception:
|
||||
pass
|
||||
return done
|
||||
|
||||
|
||||
def main():
|
||||
global SEARCH_SPACING
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("dishes", nargs="*", help="菜名(也可用 --input 从文件读)")
|
||||
ap.add_argument("--input", help="菜名文件,一行一道")
|
||||
ap.add_argument("--out", default="dishes.jsonl", help="输出 jsonl")
|
||||
ap.add_argument("--download", action="store_true", help="同时把图片下载到 images/")
|
||||
ap.add_argument("--img-dir", default="images")
|
||||
ap.add_argument("--per-dish", type=int, default=3, help="每道菜抓取的图片张数(取搜索前 N 个做法)")
|
||||
ap.add_argument("--proxy", default=os.environ.get("CRAWL_PROXY"),
|
||||
help="如 http://127.0.0.1:7897;默认直连(国内站)")
|
||||
ap.add_argument("--min-delay", type=float, default=4.0)
|
||||
ap.add_argument("--max-delay", type=float, default=9.0)
|
||||
ap.add_argument("--rest-every", type=int, default=40, help="(串行)每抓 N 条额外长歇")
|
||||
ap.add_argument("--workers", type=int, default=1,
|
||||
help="并行线程数;>1 启用并行(自动去掉串行长延迟)")
|
||||
ap.add_argument("--jitter", type=float, default=0.0,
|
||||
help="每任务起点随机错开秒数(并行未设时默认 1.5)")
|
||||
ap.add_argument("--spacing", type=float, default=SEARCH_SPACING,
|
||||
help="全局两次搜索最小间隔秒(控制搜索速率,防限速;默认 2.0)")
|
||||
args = ap.parse_args()
|
||||
|
||||
SEARCH_SPACING = args.spacing
|
||||
|
||||
dishes = list(args.dishes)
|
||||
if args.input:
|
||||
with open(args.input, encoding="utf-8") as f:
|
||||
dishes += [ln.strip() for ln in f if ln.strip()]
|
||||
if not dishes:
|
||||
ap.error("没给菜名")
|
||||
dishes = list(dict.fromkeys(dishes)) # 清单内去重(保序)
|
||||
|
||||
os.makedirs(args.img_dir, exist_ok=True)
|
||||
done = load_done(args.out)
|
||||
todo = [d for d in dishes if d not in done]
|
||||
print(f"共 {len(dishes)} 道,已抓 {len(done)},本次待抓 {len(todo)},并行 {args.workers} 线程",
|
||||
flush=True)
|
||||
if not todo:
|
||||
print("没有待抓的菜。")
|
||||
return
|
||||
|
||||
ok = miss = 0
|
||||
t0 = time.time()
|
||||
|
||||
if args.workers > 1:
|
||||
jitter = args.jitter if args.jitter > 0 else 1.5
|
||||
lock = threading.Lock()
|
||||
with open(args.out, "a", encoding="utf-8") as fout, \
|
||||
ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
futs = {ex.submit(scrape_one, d, args.proxy, args.download,
|
||||
args.img_dir, args.per_dish, jitter): d for d in todo}
|
||||
for i, fut in enumerate(as_completed(futs), 1):
|
||||
dish = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
rec = {"dish": dish, "found": False, "count": 0, "images": [],
|
||||
"reason": f"error:{type(e).__name__}", "fetched_at": now}
|
||||
with lock:
|
||||
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
fout.flush()
|
||||
if rec["found"]:
|
||||
ok += 1
|
||||
else:
|
||||
miss += 1
|
||||
if i % 20 == 0 or i == len(todo):
|
||||
rate = i / max(time.time() - t0, 1e-6)
|
||||
eta = (len(todo) - i) / rate if rate else 0
|
||||
print(f"[{i}/{len(todo)}] ✓{ok} ✗{miss} "
|
||||
f"{rate*60:.0f}道/分 ETA {eta/60:.0f}分", flush=True)
|
||||
else:
|
||||
with open(args.out, "a", encoding="utf-8") as fout:
|
||||
for i, dish in enumerate(todo, 1):
|
||||
print(f"[{i}/{len(todo)}] {dish} ...", flush=True)
|
||||
rec = scrape_one(dish, args.proxy, args.download, args.img_dir, args.per_dish)
|
||||
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
fout.flush()
|
||||
if rec["found"]:
|
||||
ok += 1
|
||||
titles = "、".join(filter(None, (e.get("recipe_title") for e in rec["images"][:3])))
|
||||
print(f" ✓ {rec['count']} 张 {titles}")
|
||||
else:
|
||||
miss += 1
|
||||
print(f" ✗ {rec['reason']}")
|
||||
if i < len(todo):
|
||||
delay = random.uniform(args.min_delay, args.max_delay)
|
||||
if i % args.rest_every == 0:
|
||||
delay += random.uniform(20, 40)
|
||||
print(f" …长歇 {delay:.0f}s")
|
||||
time.sleep(delay)
|
||||
print(f"\n完成:成功 {ok} 缺图 {miss} → {args.out}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""校验 dishes.jsonl 里记录的图是否都在盘上且为有效 JPEG;缺失/损坏的按记录的
|
||||
image_url 重新下载(应对杀软清图 / 下载失败)。
|
||||
|
||||
python verify_repair.py # 只报告(dry-run)
|
||||
python verify_repair.py --repair # 重下缺失/损坏的图
|
||||
"""
|
||||
import argparse, json, os, sys, time, random
|
||||
import requests
|
||||
|
||||
for _s in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_s.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
||||
|
||||
|
||||
def valid_jpeg(path: str) -> bool:
|
||||
"""接受常见图片格式(jpg/png/gif/webp/bmp);过小多半是错误页/截断。"""
|
||||
try:
|
||||
if os.path.getsize(path) < 800:
|
||||
return False
|
||||
with open(path, "rb") as f:
|
||||
b = f.read(12)
|
||||
return (b[:3] == b"\xff\xd8\xff" or b[:8] == b"\x89PNG\r\n\x1a\n"
|
||||
or b[:6] in (b"GIF87a", b"GIF89a")
|
||||
or (b[:4] == b"RIFF" and b[8:12] == b"WEBP") or b[:2] == b"BM")
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def redownload(url: str, path: str) -> bool:
|
||||
h = {"User-Agent": UA, "Referer": "https://www.xiachufang.com/",
|
||||
"Accept": "image/avif,image/webp,image/png,image/*;q=0.8,*/*;q=0.5"}
|
||||
try:
|
||||
r = requests.get(url, headers=h, timeout=25)
|
||||
if r.status_code == 200 and r.content:
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(r.content)
|
||||
return valid_jpeg(path)
|
||||
except requests.RequestException as e:
|
||||
print(f" [!] 重下失败 {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--jsonl", default="dishes.jsonl")
|
||||
ap.add_argument("--repair", action="store_true", help="重下缺失/损坏的图")
|
||||
args = ap.parse_args()
|
||||
|
||||
dishes = found = no_result = 0
|
||||
total = present = missing = corrupt = repaired = failed = 0
|
||||
bad_dishes = []
|
||||
|
||||
with open(args.jsonl, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rec = json.loads(line)
|
||||
dishes += 1
|
||||
if not rec.get("found"):
|
||||
no_result += 1
|
||||
bad_dishes.append(rec["dish"])
|
||||
continue
|
||||
found += 1
|
||||
for e in rec.get("images", []):
|
||||
fp = e.get("image_file")
|
||||
if not fp:
|
||||
continue
|
||||
total += 1
|
||||
ok = valid_jpeg(fp)
|
||||
if ok:
|
||||
present += 1
|
||||
continue
|
||||
if os.path.exists(fp):
|
||||
corrupt += 1
|
||||
else:
|
||||
missing += 1
|
||||
if args.repair and e.get("image_url"):
|
||||
if redownload(e["image_url"], fp):
|
||||
repaired += 1
|
||||
else:
|
||||
failed += 1
|
||||
time.sleep(random.uniform(0.1, 0.3))
|
||||
|
||||
print(f"菜条目: {dishes} (有图 {found} / 无结果 {no_result})")
|
||||
print(f"图片记录: {total} 在盘有效: {present} 缺失: {missing} 损坏: {corrupt}")
|
||||
if args.repair:
|
||||
print(f"重下成功: {repaired} 仍失败: {failed}")
|
||||
if no_result:
|
||||
print(f"无搜索结果的菜({no_result}): {'、'.join(bad_dishes[:40])}{' …' if no_result>40 else ''}")
|
||||
health = present + (repaired if args.repair else 0)
|
||||
print(f"\n图片完好率: {health}/{total} = {100*health/total:.1f}%" if total else "无图片记录")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user