feat(scraper): moments 实体(浏览器fetch getFriendsterInfo,秒→ms,hashNext翻页,接入full_sync+4h调度+图片media)

- scrape_via_page:_normalize_moments(createTime秒×1000,缺id丢弃)+
  _iter_moments_pages 翻页内核(fetch回调注入,sleep可注入,hashNext假/
  信封异常/max_pages 上限即止)+ scrape_moments(逐员工遍历wechat_account,
  form小驼峰,首页code≠10000告警cookie失效,换页重试惯例)+CLI注册moments
- media:download_moments_images(image_urls JSON数组展开去重,MomentsPost无
  回链列→用Media.original_url做已下载标记,分片IN防SQLite变量上限,批间commit)
  接入scrape_media第4源
- full_sync_all.sh:9c步moments(media前,同轮图片即可本地化)+汇总加moments计数
- scheduler:4h messages job 末尾追加 moments --max-pages 3 浅扫(全量翻页
  由每日full承担);测试断言7条命令序列
- tests/test_scrape_moments.py:归一化(秒→ms/空安全/坏createTime)+翻页内核
  (两页累积停止/hashNext假单页/max_pages防死循环/丢id行/坏信封)+图片本地化
  (一帖两图/幂等跳过/非http跳过+500容错);全套440测试通过

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-25 14:06:45 +08:00
parent b7669199f8
commit 90c28f4c7b
6 changed files with 549 additions and 15 deletions
+22 -11
View File
@@ -559,13 +559,12 @@ async def test_safe_page_run_full_invokes_bash_full_sync(engine, monkeypatch):
async def test_safe_page_run_messages_builds_scrape_command(engine, monkeypatch):
"""messages job = python scripts/scrape_via_page.py --entity messages。"""
"""messages job = friends→wa-friends→四类消息→moments 浅扫的命令序列。"""
sched = SyncScheduler(engine=engine, max_conversations_per_run=123)
captured: dict = {}
commands: list[list[str]] = []
async def fake_exec(*args, cwd=None, env=None):
captured["args"] = list(args)
captured["env"] = env
commands.append(list(args))
return _FakeProc(0)
monkeypatch.setattr(scheduler_module.asyncio, "create_subprocess_exec", fake_exec)
@@ -573,13 +572,25 @@ async def test_safe_page_run_messages_builds_scrape_command(engine, monkeypatch)
await sched._safe_page_run("messages")
# 2026-08-21: messages job 是 friends→wa-friends→四类消息的命令序列,
# 每条都调 scrape_via_page.py;最后一次执行的命令参数可断言
args = captured["args"]
assert args[1].endswith("scrape_via_page.py")
i = args.index("--entity")
assert args[i + 1] in ("messages", "wa-messages", "group-messages", "wa-group-messages")
j = args.index("--max-conversations")
assert args[j + 1] == "99999"
# 每条都调 scrape_via_page.py;2026-08-25 末尾追加 moments 浅扫
assert len(commands) == 7
for args in commands:
assert args[1].endswith("scrape_via_page.py")
def entity_of(args: list[str]) -> str:
return args[args.index("--entity") + 1]
assert entity_of(commands[0]) == "friends"
assert entity_of(commands[1]) == "wa-friends"
for args in commands[2:6]:
assert entity_of(args) in (
"messages", "wa-messages", "group-messages", "wa-group-messages",
)
assert args[args.index("--max-conversations") + 1] == "99999"
# moments 浅扫在序列末尾:每员工最多 3 页
moments_cmd = commands[-1]
assert entity_of(moments_cmd) == "moments"
assert moments_cmd[moments_cmd.index("--max-pages") + 1] == "3"
async def test_safe_page_run_nonzero_rc_is_logged_not_raised(engine, monkeypatch, capsys):