feat(analytics): 4h 调度分析链+CLI 回填+full_sync 夜间自愈步
- scheduler: yunke_analytics_tick job(与消息增量同节奏 IntervalTrigger), _analytics_tick 纯 DB——stat_daily_staff 滚动3天刷新 + 敏感词增量扫描, 不抢 _page_lock 无子进程,异常只记日志不断链;today 用 time.time() UTC epoch 喂 day_key,规避服务器非上海时区的 naive now 误判 - cli: staff-daily-stats(全量重建 rollup,幂等)+ sensitive-scan (默认增量水位扫描,--full 首跑建水位/新词回填历史;description 明示 加词不回填历史)+ 幂等补种起步词库 - full_sync_all.sh: daily-stats 后追加 staff-daily-stats(rollup 随 msg_daily_stats 重建同步自愈) - 测试: tick 行为测试(今日 seed→stat_daily_staff 出行)+ 异常吞噬 (monkeypatch 抛错不外抛);同步更新注册断言(3→4 jobs) Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
"""Command-line entry point for running syncs + the scheduler (M6, Part B).
|
||||
|
||||
Three invocations::
|
||||
Three invocations (plus analytics backfills, see below)::
|
||||
|
||||
# One-shot full sync (every entity) into the configured DB:
|
||||
python -m app.sync.cli sync
|
||||
@@ -11,6 +11,11 @@ Three invocations::
|
||||
# Long-running scheduler (APScheduler; full sync daily + messages every 4h):
|
||||
python -m app.sync.cli scheduler
|
||||
|
||||
# One-shot analytics backfills / self-heal (all idempotent):
|
||||
python -m app.sync.cli daily-stats
|
||||
python -m app.sync.cli staff-daily-stats
|
||||
python -m app.sync.cli sensitive-scan --full
|
||||
|
||||
Auth + DB
|
||||
---------
|
||||
Auth comes from env / ``.env`` via the same helpers the smoke script uses
|
||||
@@ -101,6 +106,36 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
),
|
||||
)
|
||||
|
||||
# ``staff-daily-stats`` ---------------------------------------------------
|
||||
p_staff_stats = sub.add_parser(
|
||||
"staff-daily-stats",
|
||||
help="Rebuild stat_daily_staff (staff x day rollup).",
|
||||
description=(
|
||||
"Full rebuild of the staff daily rollup (stat_daily_staff) from "
|
||||
"msg_daily_stats. Used as the nightly self-heal step in "
|
||||
"scripts/full_sync_all.sh and as the one-off deploy backfill. "
|
||||
"Idempotent."
|
||||
),
|
||||
)
|
||||
|
||||
# ``sensitive-scan`` ------------------------------------------------------
|
||||
p_scan = sub.add_parser(
|
||||
"sensitive-scan",
|
||||
help="Sensitive-word scan (incremental; --full for backfill).",
|
||||
description=(
|
||||
"Incremental sensitive-word scan by fetched_at watermark. "
|
||||
"First run on a populated DB MUST use --full to build the "
|
||||
"watermark (run_incremental refuses no-watermark on big tables). "
|
||||
"Adding a new word does NOT backfill history (watermark only "
|
||||
"sees new rows) — rerun --full to scan history for it."
|
||||
),
|
||||
)
|
||||
p_scan.add_argument(
|
||||
"--full",
|
||||
action="store_true",
|
||||
help="Full backfill scan (first deploy / new-word history).",
|
||||
)
|
||||
|
||||
# ``scheduler`` -----------------------------------------------------------
|
||||
p_sched = sub.add_parser(
|
||||
"scheduler",
|
||||
@@ -302,6 +337,43 @@ async def _cmd_daily_stats(args: argparse.Namespace) -> int:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _cmd_staff_daily_stats(args: argparse.Namespace) -> int:
|
||||
"""全量重建 stat_daily_staff(夜间自愈 / 部署回填,幂等)。"""
|
||||
settings = get_settings()
|
||||
engine, session_maker = await _init_db(settings.db_url)
|
||||
try:
|
||||
from app.storage.staff_summary import recompute_all
|
||||
|
||||
async with session_maker() as session:
|
||||
days = await recompute_all(session)
|
||||
await session.commit()
|
||||
print(f"[staff-daily-stats] rebuilt {days} days")
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _cmd_sensitive_scan(args: argparse.Namespace) -> int:
|
||||
"""敏感词扫描:默认增量(水位后);--full 全量回填(首跑/新词历史)。"""
|
||||
settings = get_settings()
|
||||
engine, session_maker = await _init_db(settings.db_url)
|
||||
try:
|
||||
from app.analytics.sensitive import scanner
|
||||
from app.analytics.sensitive import words as words_svc
|
||||
|
||||
async with session_maker() as session:
|
||||
seeded = await words_svc.ensure_starter_words(session) # 幂等补种起步词库
|
||||
if args.full:
|
||||
out = await scanner.run_full(session)
|
||||
else:
|
||||
out = await scanner.run_incremental(session)
|
||||
await session.commit()
|
||||
print(f"[sensitive-scan] seeded={seeded} {out}")
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI entry point. Returns a process exit code."""
|
||||
# Configure logging so the scheduler's JSON line + the orchestrator's
|
||||
@@ -322,6 +394,10 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return asyncio.run(_cmd_scheduler(args))
|
||||
if args.command == "daily-stats":
|
||||
return asyncio.run(_cmd_daily_stats(args))
|
||||
if args.command == "staff-daily-stats":
|
||||
return asyncio.run(_cmd_staff_daily_stats(args))
|
||||
if args.command == "sensitive-scan":
|
||||
return asyncio.run(_cmd_sensitive_scan(args))
|
||||
except KeyboardInterrupt:
|
||||
# So `python -m app.sync.cli scheduler` exits cleanly on Ctrl-C.
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user