fix(sync): 同步链路审计修复——水位边界白拉/全量无超时/群活跃窗判定源
全项目数据同步审计发现四漏洞(前项为证据): ① 消息增量边界白拉(证据:昨夜 wa-messages fetched=53,682 而真实新消息 仅 321 条,99.4% 是白拉白写的已入库旧消息)——云客 chatRecords/msg-chats 的 start 参数是 inclusive(返回 ts>=start 含边界),四个消息实体(微信 1:1/WA 1:1/微信群/WA 群)统一在归一化后过滤 ts<=watermark 的边界行, 不计数不写库(省 99% 写放大,计数从此反映真实新增) ② full_sync_all.sh 无超时——极端情况(常态 10-20h)拖进次日相位把 02:00 全量又挤掉(前两天事故同型);加 22h 上限 ③ 周日 full-history 无超时——4 实体全历史可能 24h+,必挤周一;单实体 加 6h 上限 ④ 群消息活跃窗判定源是昨晚 full 的 chat_group.last_chat_time 快照—— 「昨天安静、今天突然活跃」的群会被 24h 窗口漏到明晚;4h 序列在 group-messages 前加 chatroom-stats 全窗(~1min)刷新群活跃判定源 722 测试绿(4h 序列断言 9→10 命令+chatroom-stats 位序)。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -763,6 +763,9 @@ async def scrape_messages(cookie_str: str, max_conversations: int = 100, full_hi
|
||||
ts = int(str(ts_raw)) if ts_raw else None
|
||||
except (ValueError, TypeError):
|
||||
ts = None
|
||||
# 增量模式过滤边界旧消息(云客 start inclusive,2026-09-10)
|
||||
if watermark and ts is not None and ts <= watermark:
|
||||
continue
|
||||
records.append({
|
||||
"msg_id": str(row.get("msgId") or row.get("msgSvrId") or row.get("id") or f"{owner_id}:{friend_id}:{i}:{len(records)}"),
|
||||
"channel": "wechat",
|
||||
@@ -965,6 +968,9 @@ async def scrape_group_messages(cookie_str: str, max_groups: int = 500, full_his
|
||||
continue
|
||||
rec = _normalize_group_message(row, owner_id, room_id)
|
||||
if rec is not None:
|
||||
# 增量模式过滤边界旧消息(同 1:1,2026-09-10)
|
||||
if watermark and (rec.get("timestamp") or 0) <= watermark:
|
||||
continue
|
||||
records.append(rec)
|
||||
|
||||
except Exception as e:
|
||||
@@ -2241,7 +2247,14 @@ async def scrape_wa_messages(cookie_str: str, max_conversations: int = 100, full
|
||||
msgs = data.get("messages") or []
|
||||
for row in msgs:
|
||||
if isinstance(row, dict) and not _is_wa_divider(row):
|
||||
records.append(_normalize_whatsapp_message(row))
|
||||
rec = _normalize_whatsapp_message(row)
|
||||
# 增量模式过滤边界旧消息:云客 start 参数是 inclusive
|
||||
# (返回 ts>=start 含边界),不过滤则每对话白拉数条已
|
||||
# 入库旧消息(2026-09-10 实测一夜 fetched 53,682 仅
|
||||
# 321 条真新,99% 白写)。
|
||||
if watermark and (rec.get("timestamp") or 0) <= watermark:
|
||||
continue
|
||||
records.append(rec)
|
||||
if not data.get("hasLast"):
|
||||
break
|
||||
# 增量触底停:翻到已同步区
|
||||
@@ -2385,7 +2398,11 @@ async def scrape_wa_group_messages(cookie_str: str, max_groups: int = 500, full_
|
||||
for row in rows:
|
||||
if not isinstance(row, dict) or _is_wa_divider(row):
|
||||
continue
|
||||
records.append(_normalize_wa_group_message(row))
|
||||
rec = _normalize_wa_group_message(row)
|
||||
# 增量模式过滤边界旧消息(同 1:1,2026-09-10)
|
||||
if watermark and (rec.get("timestamp") or 0) <= watermark:
|
||||
continue
|
||||
records.append(rec)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f" [{i+1}] {str(owner_id)[:18]}:{str(room_id)[:18]} failed: {e}")
|
||||
|
||||
Reference in New Issue
Block a user