feat(analytics): 4h 调度分析链+CLI 回填+full_sync 夜间自愈步

- scheduler: yunke_analytics_tick job(与消息增量同节奏 IntervalTrigger),
  _analytics_tick 纯 DB——stat_daily_staff 滚动3天刷新 + 敏感词增量扫描,
  不抢 _page_lock 无子进程,异常只记日志不断链;today 用 time.time()
  UTC epoch 喂 day_key,规避服务器非上海时区的 naive now 误判
- cli: staff-daily-stats(全量重建 rollup,幂等)+ sensitive-scan
  (默认增量水位扫描,--full 首跑建水位/新词回填历史;description 明示
  加词不回填历史)+ 幂等补种起步词库
- full_sync_all.sh: daily-stats 后追加 staff-daily-stats(rollup 随
  msg_daily_stats 重建同步自愈)
- 测试: tick 行为测试(今日 seed→stat_daily_staff 出行)+ 异常吞噬
  (monkeypatch 抛错不外抛);同步更新注册断言(3→4 jobs)

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-09-04 21:02:41 +08:00
parent 6a1c904018
commit 6b99202e9b
5 changed files with 212 additions and 4 deletions
+77 -1
View File
@@ -1,6 +1,6 @@
"""Command-line entry point for running syncs + the scheduler (M6, Part B).
Three invocations::
Three invocations (plus analytics backfills, see below)::
# One-shot full sync (every entity) into the configured DB:
python -m app.sync.cli sync
@@ -11,6 +11,11 @@ Three invocations::
# Long-running scheduler (APScheduler; full sync daily + messages every 4h):
python -m app.sync.cli scheduler
# One-shot analytics backfills / self-heal (all idempotent):
python -m app.sync.cli daily-stats
python -m app.sync.cli staff-daily-stats
python -m app.sync.cli sensitive-scan --full
Auth + DB
---------
Auth comes from env / ``.env`` via the same helpers the smoke script uses
@@ -101,6 +106,36 @@ def _build_parser() -> argparse.ArgumentParser:
),
)
# ``staff-daily-stats`` ---------------------------------------------------
p_staff_stats = sub.add_parser(
"staff-daily-stats",
help="Rebuild stat_daily_staff (staff x day rollup).",
description=(
"Full rebuild of the staff daily rollup (stat_daily_staff) from "
"msg_daily_stats. Used as the nightly self-heal step in "
"scripts/full_sync_all.sh and as the one-off deploy backfill. "
"Idempotent."
),
)
# ``sensitive-scan`` ------------------------------------------------------
p_scan = sub.add_parser(
"sensitive-scan",
help="Sensitive-word scan (incremental; --full for backfill).",
description=(
"Incremental sensitive-word scan by fetched_at watermark. "
"First run on a populated DB MUST use --full to build the "
"watermark (run_incremental refuses no-watermark on big tables). "
"Adding a new word does NOT backfill history (watermark only "
"sees new rows) — rerun --full to scan history for it."
),
)
p_scan.add_argument(
"--full",
action="store_true",
help="Full backfill scan (first deploy / new-word history).",
)
# ``scheduler`` -----------------------------------------------------------
p_sched = sub.add_parser(
"scheduler",
@@ -302,6 +337,43 @@ async def _cmd_daily_stats(args: argparse.Namespace) -> int:
await engine.dispose()
async def _cmd_staff_daily_stats(args: argparse.Namespace) -> int:
"""全量重建 stat_daily_staff(夜间自愈 / 部署回填,幂等)。"""
settings = get_settings()
engine, session_maker = await _init_db(settings.db_url)
try:
from app.storage.staff_summary import recompute_all
async with session_maker() as session:
days = await recompute_all(session)
await session.commit()
print(f"[staff-daily-stats] rebuilt {days} days")
return 0
finally:
await engine.dispose()
async def _cmd_sensitive_scan(args: argparse.Namespace) -> int:
"""敏感词扫描:默认增量(水位后);--full 全量回填(首跑/新词历史)。"""
settings = get_settings()
engine, session_maker = await _init_db(settings.db_url)
try:
from app.analytics.sensitive import scanner
from app.analytics.sensitive import words as words_svc
async with session_maker() as session:
seeded = await words_svc.ensure_starter_words(session) # 幂等补种起步词库
if args.full:
out = await scanner.run_full(session)
else:
out = await scanner.run_incremental(session)
await session.commit()
print(f"[sensitive-scan] seeded={seeded} {out}")
return 0
finally:
await engine.dispose()
def main(argv: list[str] | None = None) -> int:
"""CLI entry point. Returns a process exit code."""
# Configure logging so the scheduler's JSON line + the orchestrator's
@@ -322,6 +394,10 @@ def main(argv: list[str] | None = None) -> int:
return asyncio.run(_cmd_scheduler(args))
if args.command == "daily-stats":
return asyncio.run(_cmd_daily_stats(args))
if args.command == "staff-daily-stats":
return asyncio.run(_cmd_staff_daily_stats(args))
if args.command == "sensitive-scan":
return asyncio.run(_cmd_sensitive_scan(args))
except KeyboardInterrupt:
# So `python -m app.sync.cli scheduler` exits cleanly on Ctrl-C.
return 0
+39
View File
@@ -28,6 +28,7 @@ import logging
import os
import shutil
import sys
import time
from collections.abc import Awaitable, Callable
from pathlib import Path
from typing import Any
@@ -535,6 +536,16 @@ class SyncScheduler:
coalesce=True,
max_instances=1,
)
# 4h 分析链:stat_daily_staff 滚动3天刷新 + 敏感词增量扫描(纯 DB,
# 无浏览器子进程;与消息增量同节奏,读滞后语义与「数据截至」一致)
sched.add_job(
self._analytics_tick,
IntervalTrigger(hours=self.message_interval_hours),
id="yunke_analytics_tick",
replace_existing=True,
coalesce=True,
max_instances=1,
)
sched.start()
self._scheduler = sched
logger.info(
@@ -662,6 +673,34 @@ class SyncScheduler:
logger.info(line)
print(line, flush=True)
async def _analytics_tick(self) -> None:
"""4h 分析链(stat_daily_staff 滚动3天 + 敏感词增量)——异常只记日志不断链。
纯 DB 无浏览器子进程,不抢 ``_page_lock``(DB 写毫秒级;
``max_instances=1`` 防自身重叠即可;与夜间 full_sync 的
staff-daily-stats 步骤重叠时 UNIQUE 兜底幂等)。
时区:today 用 ``time.time()``(UTC epoch,无时区歧义)喂
``day_key``——naive ``now().timestamp()`` 按本地时区解释,服务器
时区未必是上海(2026-09-04 部署注意点)。
"""
try:
async with self._session_maker() as session:
from app.analytics.sensitive import scanner
from app.storage.staff_summary import recompute_days, rolling_days
from app.storage.summary import day_key
today = day_key(int(time.time() * 1000))
n = await recompute_days(session, rolling_days(today, 3))
scan = await scanner.run_incremental(session)
await session.commit()
logger.info(
"[analytics-tick] staff_days=%s scanned=%s hits=%s",
n, scan.get("scanned"), scan.get("hits"),
)
except Exception: # noqa: BLE001 — scheduler MUST NOT crash
logger.exception("analytics tick failed")
async def _page_subprocess(self, args: list[str]) -> int:
"""启动页面子进程并等待结束;继承环境 + 补 DB_URL/PYTHONIOENCODING。"""
env = {**os.environ, "PYTHONIOENCODING": "utf-8"}