陈世睿
74caa9d112
- crawler/:豆果(主)/下厨房/Bing 爬虫 + dishes.jsonl(3056 菜→9167 图映射)+ verify_repair 按 URL 重下 - retrieval/:三路检索(BM25 + 本地 BGE-M3 向量 + RRF 融合),FastAPI + 前端;写死图片目录(默认 crawler/images,可 IMAGE_DIR 覆盖)、绑 0.0.0.0 局域网访问、启动自动建索引、服务器 serve 图片 - 图片(1.7G)与向量模型(2.3G)不进 git Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
104 lines
3.7 KiB
Python
104 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""校验 dishes.jsonl 里记录的图是否都在盘上且为有效 JPEG;缺失/损坏的按记录的
|
|
image_url 重新下载(应对杀软清图 / 下载失败)。
|
|
|
|
python verify_repair.py # 只报告(dry-run)
|
|
python verify_repair.py --repair # 重下缺失/损坏的图
|
|
"""
|
|
import argparse, json, os, sys, time, random
|
|
import requests
|
|
|
|
for _s in (sys.stdout, sys.stderr):
|
|
try:
|
|
_s.reconfigure(encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
pass
|
|
|
|
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
|
|
|
|
|
def valid_jpeg(path: str) -> bool:
|
|
"""接受常见图片格式(jpg/png/gif/webp/bmp);过小多半是错误页/截断。"""
|
|
try:
|
|
if os.path.getsize(path) < 800:
|
|
return False
|
|
with open(path, "rb") as f:
|
|
b = f.read(12)
|
|
return (b[:3] == b"\xff\xd8\xff" or b[:8] == b"\x89PNG\r\n\x1a\n"
|
|
or b[:6] in (b"GIF87a", b"GIF89a")
|
|
or (b[:4] == b"RIFF" and b[8:12] == b"WEBP") or b[:2] == b"BM")
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def redownload(url: str, path: str) -> bool:
|
|
h = {"User-Agent": UA, "Referer": "https://www.xiachufang.com/",
|
|
"Accept": "image/avif,image/webp,image/png,image/*;q=0.8,*/*;q=0.5"}
|
|
try:
|
|
r = requests.get(url, headers=h, timeout=25)
|
|
if r.status_code == 200 and r.content:
|
|
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
|
with open(path, "wb") as f:
|
|
f.write(r.content)
|
|
return valid_jpeg(path)
|
|
except requests.RequestException as e:
|
|
print(f" [!] 重下失败 {e}", file=sys.stderr)
|
|
return False
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--jsonl", default="dishes.jsonl")
|
|
ap.add_argument("--repair", action="store_true", help="重下缺失/损坏的图")
|
|
args = ap.parse_args()
|
|
|
|
dishes = found = no_result = 0
|
|
total = present = missing = corrupt = repaired = failed = 0
|
|
bad_dishes = []
|
|
|
|
with open(args.jsonl, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
rec = json.loads(line)
|
|
dishes += 1
|
|
if not rec.get("found"):
|
|
no_result += 1
|
|
bad_dishes.append(rec["dish"])
|
|
continue
|
|
found += 1
|
|
for e in rec.get("images", []):
|
|
fp = e.get("image_file")
|
|
if not fp:
|
|
continue
|
|
total += 1
|
|
ok = valid_jpeg(fp)
|
|
if ok:
|
|
present += 1
|
|
continue
|
|
if os.path.exists(fp):
|
|
corrupt += 1
|
|
else:
|
|
missing += 1
|
|
if args.repair and e.get("image_url"):
|
|
if redownload(e["image_url"], fp):
|
|
repaired += 1
|
|
else:
|
|
failed += 1
|
|
time.sleep(random.uniform(0.1, 0.3))
|
|
|
|
print(f"菜条目: {dishes} (有图 {found} / 无结果 {no_result})")
|
|
print(f"图片记录: {total} 在盘有效: {present} 缺失: {missing} 损坏: {corrupt}")
|
|
if args.repair:
|
|
print(f"重下成功: {repaired} 仍失败: {failed}")
|
|
if no_result:
|
|
print(f"无搜索结果的菜({no_result}): {'、'.join(bad_dishes[:40])}{' …' if no_result>40 else ''}")
|
|
health = present + (repaired if args.repair else 0)
|
|
print(f"\n图片完好率: {health}/{total} = {100*health/total:.1f}%" if total else "无图片记录")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|