功能:新手引导视频 + 美团券首页分页索引 (#167)
Co-authored-by: guke <guke@wonderable.ai> Co-authored-by: 左辰勇 <exinglang@gmail.com> Reviewed-on: #167 Co-authored-by: zuochenyong <zuochenyong@wonderable.ai> Co-committed-by: zuochenyong <zuochenyong@wonderable.ai>
This commit was merged in pull request #167.
This commit is contained in:
@@ -11,6 +11,7 @@ import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
@@ -25,6 +26,44 @@ class MeituanCpsError(Exception):
|
||||
"""美团 CPS 接口调用失败。"""
|
||||
|
||||
|
||||
# ────────────────────── 共享 httpx.Client(连接池 + keep-alive) ──────────────────────
|
||||
# 原实现每次 _call 都 `with httpx.Client(...)` 新建再关掉,等于每发一次请求就:
|
||||
# ① 重建一套 SSL 上下文(加载 certifi CA,实测几百 ms) ② 重做一次 TLS 握手 ③ 用完即弃连接。
|
||||
# 首页 feed 一次翻页要向美团发好几次请求(外卖/到店两路,「距离最近」还要续页),这份固定开销被成倍放大,
|
||||
# 是「滑到底部加载很慢」的一大块。改成进程内单例:握手一次、后续走 keep-alive 复用。
|
||||
# httpx.Client 本身线程安全,可被 feed 的 ThreadPoolExecutor 两条抓取线程共用。
|
||||
_client: httpx.Client | None = None
|
||||
_client_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_client() -> httpx.Client:
|
||||
"""取美团 CPS 共享 client(幂等,懒建)。lifespan 启动时预热,把建 SSL 的成本摊到启动。"""
|
||||
global _client
|
||||
if _client is None:
|
||||
with _client_lock:
|
||||
if _client is None:
|
||||
# 走 MT_CPS_PROXY(本机开发直连会 SSL EOF,必须走代理;线上留空=直连)。
|
||||
# trust_env=False:不读进程环境的 HTTP_PROXY,只认配置,避免被错误/失效代理误导。
|
||||
_client = httpx.Client(
|
||||
proxy=settings.MT_CPS_PROXY or None,
|
||||
trust_env=False,
|
||||
timeout=settings.MT_CPS_TIMEOUT_SEC,
|
||||
# 池子留足:feed 每个请求会起 2 条抓取线程,并发用户多时别在池上排队
|
||||
# (排满会等到 MT_CPS_TIMEOUT_SEC 抛 PoolTimeout,表现成"又慢又降级")。
|
||||
limits=httpx.Limits(max_keepalive_connections=16, max_connections=32),
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
def close_client() -> None:
|
||||
"""lifespan 关停时调,优雅关连接池。"""
|
||||
global _client
|
||||
with _client_lock:
|
||||
if _client is not None:
|
||||
_client.close()
|
||||
_client = None
|
||||
|
||||
|
||||
def _content_md5(body: bytes) -> str:
|
||||
return base64.b64encode(hashlib.md5(body).digest()).decode()
|
||||
|
||||
@@ -62,12 +101,8 @@ def _call(path: str, body_obj: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
url = f"{settings.MT_CPS_HOST}{path}"
|
||||
# 美团调用走 MT_CPS_PROXY(本机开发直连会 SSL EOF,必须走代理;线上留空=直连)。
|
||||
# trust_env=False:不读进程环境的 HTTP_PROXY,只认配置,避免被错误/失效代理误导。
|
||||
proxy = settings.MT_CPS_PROXY or None
|
||||
try:
|
||||
with httpx.Client(proxy=proxy, trust_env=False, timeout=settings.MT_CPS_TIMEOUT_SEC) as client:
|
||||
resp = client.post(url, content=body, headers=headers)
|
||||
resp = get_client().post(url, content=body, headers=headers)
|
||||
except httpx.HTTPError as e:
|
||||
logger.exception("[MT] http error calling %s", url)
|
||||
raise MeituanCpsError(f"meituan http error: {e}") from e
|
||||
|
||||
Reference in New Issue
Block a user