shaguabijia-app-server/app/core/cps_reconcile_worker.py

"""美团、京东 CPS 订单每日自动对账任务。

每天北京时间 `CPS_AUTO_RECONCILE_RUN_HOUR`（默认 05:00）后执行一次，按更新时间
回拉最近若干天订单并复用 admin CPS 仓储层的幂等 upsert。手动对账接口保持独立、不受影响。
"""
from __future__ import annotations

import asyncio
import contextlib
import logging
import os
import socket
import time
from collections.abc import Callable, Iterator
from datetime import date, datetime, timedelta
from pathlib import Path
from uuid import uuid4

from app.admin.repositories import cps as cps_repo
from app.core.config import settings
from app.core.rewards import CN_TZ
from app.db.session import SessionLocal, engine

logger = logging.getLogger("shagua.cps_reconcile")
_LOCK_PATH = Path(__file__).resolve().parents[2] / "data" / "cps_reconcile.lock"


def _cn_now() -> datetime:
    return datetime.now(CN_TZ)


def _new_run_id(now: datetime) -> str:
    return f"{now:%Y%m%d-%H%M%S}-{uuid4().hex[:8]}"


def _next_run_at(now: datetime, run_hour: int) -> datetime:
    scheduled = now.replace(hour=run_hour, minute=0, second=0, microsecond=0)
    return scheduled if now < scheduled else scheduled + timedelta(days=1)


def _trigger_for_run(worker_started_at: datetime, now: datetime, run_hour: int) -> str:
    if worker_started_at.date() == now.date() and worker_started_at.hour >= run_hour:
        return "startup_catchup"
    return "scheduled"


def _touch_lock() -> None:
    with contextlib.suppress(FileNotFoundError):
        os.utime(_LOCK_PATH, None)


@contextlib.contextmanager
def _single_instance_lock(stale_after_sec: int) -> Iterator[bool]:
    """同机多进程保护：同一时间只允许一个 CPS 自动对账 worker 运行。"""
    _LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
    fd: int | None = None
    try:
        try:
            fd = os.open(str(_LOCK_PATH), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
        except FileExistsError:
            try:
                age = time.time() - _LOCK_PATH.stat().st_mtime
            except FileNotFoundError:
                age = stale_after_sec + 1
            if age > stale_after_sec:
                with contextlib.suppress(FileNotFoundError):
                    _LOCK_PATH.unlink()
                try:
                    fd = os.open(str(_LOCK_PATH), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
                except FileExistsError:
                    fd = None

        if fd is None:
            yield False
            return

        os.write(fd, f"pid={os.getpid()} started_at={int(time.time())}\n".encode("ascii"))
        yield True
    finally:
        if fd is not None:
            os.close(fd)
            with contextlib.suppress(FileNotFoundError):
                _LOCK_PATH.unlink()


def _empty_result() -> dict:
    return {
        "fetched": 0,
        "inserted": 0,
        "updated": 0,
        "pages": 0,
        "api_requests": 0,
    }


def _platform_log_fields(platform: str, result: dict) -> dict:
    fields = {
        "platform": platform,
        "platform_status": result["status"],
        "duration_ms": result["duration_ms"],
    }
    for key in ("fetched", "inserted", "updated", "pages", "api_requests", "windows"):
        if key in result:
            fields[key] = result[key]
    return fields


def _run_platform(
    *,
    platform: str,
    query_time_type: int,
    common: dict,
    reconcile: Callable[[], dict],
) -> tuple[dict, str | None]:
    started = time.perf_counter()
    logger.info(
        "CPS auto reconcile platform started run_id=%s platform=%s",
        common["run_id"],
        platform,
        extra={
            **common,
            "event": "cps_reconcile.platform_started",
            "platform": platform,
            "query_time_type": query_time_type,
        },
    )
    try:
        raw_result = reconcile()
    except Exception as exc:  # noqa: BLE001 - 单平台失败不能阻断另一平台
        result = {
            "status": "failed",
            "duration_ms": round((time.perf_counter() - started) * 1000, 3),
            "error_type": type(exc).__name__,
            "error_summary": str(exc)[:500],
        }
        logger.exception(
            "CPS auto reconcile platform failed run_id=%s platform=%s error_type=%s",
            common["run_id"],
            platform,
            type(exc).__name__,
            extra={
                **common,
                "event": "cps_reconcile.platform_failed",
                **_platform_log_fields(platform, result),
                "error_type": type(exc).__name__,
                "error_summary": str(exc)[:500],
            },
        )
        return result, str(exc)

    result = {
        **raw_result,
        "status": "success",
        "duration_ms": round((time.perf_counter() - started) * 1000, 3),
    }
    logger.info(
        "CPS auto reconcile platform completed run_id=%s platform=%s fetched=%s inserted=%s updated=%s",
        common["run_id"],
        platform,
        result.get("fetched", 0),
        result.get("inserted", 0),
        result.get("updated", 0),
        extra={
            **common,
            "event": "cps_reconcile.platform_completed",
            **_platform_log_fields(platform, result),
        },
    )
    return result, None


def _skip_platform(platform: str, common: dict) -> dict:
    result = {
        **_empty_result(),
        "status": "skipped",
        "skipped": "not_configured",
        "duration_ms": 0.0,
    }
    logger.warning(
        "CPS auto reconcile platform skipped run_id=%s platform=%s reason=not_configured",
        common["run_id"],
        platform,
        extra={
            **common,
            "event": "cps_reconcile.platform_skipped",
            **_platform_log_fields(platform, result),
            "skip_reason": "not_configured",
        },
    )
    return result


def _reconcile_once(
    now: datetime | None = None,
    *,
    run_id: str | None = None,
    trigger: str = "scheduled",
) -> dict:
    """独立拉取美团和京东；单平台异常只记日志，不影响另一平台。"""
    end = (now or _cn_now()).astimezone(CN_TZ)
    run_id = run_id or _new_run_id(end)
    lookback_days = max(1, int(settings.CPS_AUTO_RECONCILE_LOOKBACK_DAYS))
    start = end - timedelta(days=lookback_days)
    task_started = time.perf_counter()
    common = {
        "run_id": run_id,
        "trigger": trigger,
        "scheduled_date": end.date().isoformat(),
        "app_env": settings.APP_ENV,
        "db_dialect": engine.dialect.name,
        "hostname": socket.gethostname(),
        "pid": os.getpid(),
        "window_start": start.isoformat(),
        "window_end": end.isoformat(),
        "lookback_days": lookback_days,
    }
    result = {
        "run_id": run_id,
        "trigger": trigger,
        "window_start": start.isoformat(),
        "window_end": end.isoformat(),
        "meituan": None,
        "jd": None,
        "errors": {},
    }
    logger.info(
        "CPS auto reconcile started run_id=%s trigger=%s window=%s..%s",
        run_id,
        trigger,
        result["window_start"],
        result["window_end"],
        extra={**common, "event": "cps_reconcile.started"},
    )

    if settings.mt_cps_configured:
        def reconcile_meituan() -> dict:
            with SessionLocal() as db:
                return cps_repo.reconcile_orders(
                    db,
                    start_time=int(start.timestamp()),
                    end_time=int(end.timestamp()),
                    query_time_type=2,
                    audit_context=common,
                )

        result["meituan"], error = _run_platform(
            platform="meituan",
            query_time_type=2,
            common=common,
            reconcile=reconcile_meituan,
        )
        if error is not None:
            result["errors"]["meituan"] = error
    else:
        result["meituan"] = _skip_platform("meituan", common)

    _touch_lock()
    if settings.jd_union_configured:
        def reconcile_jd() -> dict:
            with SessionLocal() as db:
                return cps_repo.reconcile_jd_orders(
                    db,
                    start_time=start,
                    end_time=end,
                    query_time_type=3,
                    audit_context=common,
                )

        result["jd"], error = _run_platform(
            platform="jd",
            query_time_type=3,
            common=common,
            reconcile=reconcile_jd,
        )
        if error is not None:
            result["errors"]["jd"] = error
    else:
        result["jd"] = _skip_platform("jd", common)

    successes = sum(
        platform_result.get("status") == "success"
        for platform_result in (result["meituan"], result["jd"])
    )
    if result["errors"]:
        task_status = "partial_success" if successes else "failed"
    else:
        task_status = "success"
    completed_at = _cn_now()
    duration_ms = round((time.perf_counter() - task_started) * 1000, 3)
    next_run_at = _next_run_at(
        completed_at,
        min(max(int(settings.CPS_AUTO_RECONCILE_RUN_HOUR), 0), 23),
    ).isoformat()
    result.update({
        "status": task_status,
        "duration_ms": duration_ms,
        "manual_retry_required": bool(result["errors"]),
        "next_run_at": next_run_at,
    })
    completion_fields = {
        **common,
        "event": "cps_reconcile.completed",
        "task_status": task_status,
        "duration_ms": duration_ms,
        "manual_retry_required": result["manual_retry_required"],
        "failed_platforms": sorted(result["errors"]),
        "next_run_at": next_run_at,
        "meituan_result": result["meituan"],
        "jd_result": result["jd"],
    }
    log_method = logger.info if task_status == "success" else logger.warning
    log_method(
        "CPS auto reconcile completed run_id=%s status=%s duration_ms=%s failed_platforms=%s next_run_at=%s",
        run_id,
        task_status,
        duration_ms,
        ",".join(sorted(result["errors"])) or "-",
        next_run_at,
        extra=completion_fields,
    )

    return result


def _should_run(last_run: date | None, now: datetime, run_hour: int) -> bool:
    return last_run != now.date() and now.hour >= run_hour


async def _run_loop() -> None:
    interval = max(30, int(settings.CPS_AUTO_RECONCILE_CHECK_INTERVAL_SEC))
    run_hour = min(max(int(settings.CPS_AUTO_RECONCILE_RUN_HOUR), 0), 23)
    lock_stale_after = max(interval * 3, 1800)
    with _single_instance_lock(lock_stale_after) as lock_acquired:
        if not lock_acquired:
            logger.warning(
                "CPS auto reconcile worker skipped: another worker owns lock",
                extra={
                    "event": "cps_reconcile.worker_skipped",
                    "skip_reason": "lock_not_acquired",
                    "pid": os.getpid(),
                },
            )
            return
        await _run_locked_loop(interval, run_hour)


async def _run_locked_loop(interval: int, run_hour: int) -> None:
    worker_started_at = _cn_now()
    logger.info(
        "CPS auto reconcile worker started run_hour=%s interval=%ss lookback_days=%s",
        run_hour,
        interval,
        settings.CPS_AUTO_RECONCILE_LOOKBACK_DAYS,
        extra={
            "event": "cps_reconcile.worker_started",
            "pid": os.getpid(),
            "hostname": socket.gethostname(),
            "app_env": settings.APP_ENV,
            "db_dialect": engine.dialect.name,
            "run_hour": run_hour,
            "interval_sec": interval,
            "lookback_days": settings.CPS_AUTO_RECONCILE_LOOKBACK_DAYS,
            "next_run_at": _next_run_at(worker_started_at, run_hour).isoformat(),
        },
    )
    last_run: date | None = None
    try:
        while True:
            run_id: str | None = None
            try:
                _touch_lock()
                now = _cn_now()
                if _should_run(last_run, now, run_hour):
                    run_id = _new_run_id(now)
                    trigger = _trigger_for_run(worker_started_at, now, run_hour)
                    await asyncio.to_thread(
                        _reconcile_once,
                        now,
                        run_id=run_id,
                        trigger=trigger,
                    )
                    last_run = now.date()
            except Exception:  # noqa: BLE001 - 后台任务不能因单次异常退出
                logger.exception(
                    "CPS auto reconcile unexpected worker error",
                    extra={
                        "event": "cps_reconcile.unexpected_failed",
                        "run_id": run_id or "unassigned",
                        "pid": os.getpid(),
                    },
                )
            await asyncio.sleep(interval)
    except asyncio.CancelledError:
        logger.info(
            "CPS auto reconcile worker stopped",
            extra={"event": "cps_reconcile.worker_stopped", "pid": os.getpid()},
        )
        raise


def start_cps_reconcile_worker() -> asyncio.Task | None:
    if not settings.CPS_AUTO_RECONCILE_ENABLED:
        logger.info(
            "CPS auto reconcile disabled",
            extra={
                "event": "cps_reconcile.worker_skipped",
                "skip_reason": "disabled",
            },
        )
        return None
    if not settings.mt_cps_configured and not settings.jd_union_configured:
        logger.warning(
            "CPS auto reconcile not started: Meituan and JD credentials are missing",
            extra={
                "event": "cps_reconcile.worker_skipped",
                "skip_reason": "all_platform_credentials_missing",
            },
        )
        return None
    return asyncio.create_task(_run_loop(), name="cps-auto-reconcile")


async def stop_cps_reconcile_worker(task: asyncio.Task | None) -> None:
    if task is None:
        return
    task.cancel()
    with contextlib.suppress(asyncio.CancelledError):
        await task