fix(scheduler): 调度根治——4h 增量只扫活跃对话+账号表进 4h+超时保险丝
用户报「员工列表时间与云客差 2 天」。根因链:4h 增量序列全对话遍历 (WA 9,357 对×~3.5s≈9h,整序列 15h+)超过自身 4h 间隔→与 02:00 全量在 asyncio.Lock 上背靠背互相排队→近两晚全量被挤掉→账号表(仅全量刷)停更 两天,员工列表「数据更新/最新消息时间」显示 09-08 快照。 三层修复: - 活跃窗:_discover_conversations/群发现加 active_since_ms——已扫过的 对话仅保留 last_chat_time 在窗内的(云客 lastChatTime 滞后小时级, 24h 窗足够容错),从未扫过的(新覆盖优先)不受限;陈旧对话无新消息, 每日 full 兜底。CLI --active-window-hours,四消息实体透传 - 4h 序列头部加 accounts/wa-accounts(~2min)——员工列表时间最多滞后 4h,不再依赖每晚全量 - 超时保险丝:_page_subprocess 加 timeout(超时 kill 返回 124,上游 break 下轮补)——messages 各实体 15-80min 预算,单命令拖不死相位; full/full-history 不设限 722 测试绿(序列断言 7→9 命令+窗口参数)。干跑验证:近 24h 活跃私聊/ 群相对全量降一个数量级(本地旧库群 2011→537;私聊依赖 friends 先刷, 序列顺序已保证)。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -602,50 +602,72 @@ class SyncScheduler:
|
||||
if not bash:
|
||||
logger.error("page sync 'full': bash not found on PATH")
|
||||
return
|
||||
commands: list[list[str]] = [[bash, "scripts/full_sync_all.sh"]]
|
||||
commands = [([bash, "scripts/full_sync_all.sh"], None)] # (args, timeout)——full 不设超时(全量本身 10-20h)
|
||||
elif kind == "messages":
|
||||
# 先刷客户列表(最近沟通时间是云客列表的实时排序键,friends 快照
|
||||
# 滞后会让本地排序与云客第一页完全不同,2026-08-20),再抓全部四类
|
||||
# 消息增量(2026-08-21 全接入:水位增量后单对话 1 页,可承受);
|
||||
# 末尾追加员工朋友圈浅扫(2026-08-25,max-pages 3 即每员工最多
|
||||
# 3 页,新帖通常落第一页;全量翻页由每日 full 走 full_sync_all.sh)
|
||||
# 2026-09-10 调度根治(用户报「员工列表时间差 2 天」):
|
||||
# ① 序列头部刷员工账号表(accounts ~1min)——员工列表「数据
|
||||
# 更新时间/最新消息时间」从此最多滞后 4h(此前只在每日
|
||||
# full 刷,长任务挤掉 full 就停更);
|
||||
# ② 消息实体只扫**近 24h 活跃**对话(--active-window-hours;
|
||||
# 从未扫过的不受限)——全对话遍历(WA 9,357 个×~3.5s≈9h)
|
||||
# 曾把序列拖到 15h+,超过 4h 间隔→与 02:00 full 背靠背
|
||||
# 互相挤死;陈旧对话无新消息,每日 full 兜底;
|
||||
# ③ 每命令超时保险丝(超时 kill 下轮补,增量幂等)。
|
||||
# friends 仍先刷(最近沟通时间是排序键+活跃窗口的判定源)。
|
||||
msg_entities = [
|
||||
("accounts", None, 900),
|
||||
("wa-accounts", None, 900),
|
||||
("friends", None, 2400),
|
||||
("wa-friends", None, 2400),
|
||||
("messages", 24, 4800),
|
||||
("wa-messages", 24, 4800),
|
||||
("group-messages", 24, 4800),
|
||||
("wa-group-messages", 24, 2400),
|
||||
("moments", None, 2400),
|
||||
]
|
||||
commands = [
|
||||
[sys.executable, "scripts/scrape_via_page.py", "--entity", "friends"],
|
||||
[sys.executable, "scripts/scrape_via_page.py", "--entity", "wa-friends"],
|
||||
] + [
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/scrape_via_page.py",
|
||||
"--entity",
|
||||
e,
|
||||
"--max-conversations",
|
||||
"99999",
|
||||
]
|
||||
for e in ("messages", "wa-messages", "group-messages", "wa-group-messages")
|
||||
] + [
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/scrape_via_page.py",
|
||||
"--entity",
|
||||
"moments",
|
||||
"--max-pages",
|
||||
"3",
|
||||
]
|
||||
(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/scrape_via_page.py",
|
||||
"--entity",
|
||||
entity,
|
||||
]
|
||||
+ (
|
||||
[
|
||||
"--max-conversations",
|
||||
"99999",
|
||||
"--active-window-hours",
|
||||
str(window),
|
||||
]
|
||||
if window
|
||||
else (
|
||||
["--max-pages", "3"]
|
||||
if entity == "moments"
|
||||
else []
|
||||
)
|
||||
),
|
||||
timeout,
|
||||
)
|
||||
for entity, window, timeout in msg_entities
|
||||
]
|
||||
elif kind == "full-history":
|
||||
# 每周全量对齐(周日 05:00):强制全量翻历史,对齐增量模式抓不到的
|
||||
# 撤回/编辑消息 + 彻底消除 lastChatTime 滞后窗口(2026-08-21)
|
||||
entities = ["messages", "group-messages", "wa-messages", "wa-group-messages"]
|
||||
commands = [
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/scrape_via_page.py",
|
||||
"--entity",
|
||||
e,
|
||||
"--max-conversations",
|
||||
"99999",
|
||||
"--full-history",
|
||||
]
|
||||
(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/scrape_via_page.py",
|
||||
"--entity",
|
||||
e,
|
||||
"--max-conversations",
|
||||
"99999",
|
||||
"--full-history",
|
||||
],
|
||||
None, # 每周对齐不设超时
|
||||
)
|
||||
for e in entities
|
||||
]
|
||||
else:
|
||||
@@ -653,8 +675,8 @@ class SyncScheduler:
|
||||
return
|
||||
|
||||
rc = 0
|
||||
for args in commands:
|
||||
rc = await self._page_subprocess(args) or rc
|
||||
for args, timeout in commands:
|
||||
rc = await self._page_subprocess(args, timeout=timeout) or rc
|
||||
if rc != 0:
|
||||
logger.warning("page sync %s: command failed rc=%s, skipping rest", kind, rc)
|
||||
break
|
||||
@@ -703,8 +725,13 @@ class SyncScheduler:
|
||||
except Exception: # noqa: BLE001 — scheduler MUST NOT crash
|
||||
logger.exception("analytics tick failed")
|
||||
|
||||
async def _page_subprocess(self, args: list[str]) -> int:
|
||||
"""启动页面子进程并等待结束;继承环境 + 补 DB_URL/PYTHONIOENCODING。"""
|
||||
async def _page_subprocess(self, args: list[str], *, timeout: float | None = None) -> int:
|
||||
"""启动页面子进程并等待结束;继承环境 + 补 DB_URL/PYTHONIOENCODING。
|
||||
|
||||
``timeout``(秒,2026-09-10 调度根治保险丝):超时 kill 子进程返回
|
||||
非零——上游 break 跳过本序列剩余命令,增量幂等下轮自动补,防止
|
||||
单命令拖死整个调度相位(4h 序列曾因此 15h+ 与 full 互相挤死)。
|
||||
"""
|
||||
env = {**os.environ, "PYTHONIOENCODING": "utf-8"}
|
||||
settings = get_settings()
|
||||
if settings.db_url and "DB_URL" not in env:
|
||||
@@ -714,7 +741,18 @@ class SyncScheduler:
|
||||
cwd=str(REPO_ROOT),
|
||||
env=env,
|
||||
)
|
||||
return await proc.wait()
|
||||
if timeout is None:
|
||||
return await proc.wait()
|
||||
try:
|
||||
return await asyncio.wait_for(proc.wait(), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(
|
||||
"page sync subprocess timeout after %.0fs (killed): %s",
|
||||
timeout, " ".join(args[-6:]),
|
||||
)
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
return 124 # 超时语义(同 shell timeout 惯例)
|
||||
|
||||
async def _safe_run(
|
||||
self, entity: str, *, log: bool = True
|
||||
|
||||
Reference in New Issue
Block a user