feat(scraper): moments 实体(浏览器fetch getFriendsterInfo,秒→ms,hashNext翻页,接入full_sync+4h调度+图片media)
- scrape_via_page:_normalize_moments(createTime秒×1000,缺id丢弃)+ _iter_moments_pages 翻页内核(fetch回调注入,sleep可注入,hashNext假/ 信封异常/max_pages 上限即止)+ scrape_moments(逐员工遍历wechat_account, form小驼峰,首页code≠10000告警cookie失效,换页重试惯例)+CLI注册moments - media:download_moments_images(image_urls JSON数组展开去重,MomentsPost无 回链列→用Media.original_url做已下载标记,分片IN防SQLite变量上限,批间commit) 接入scrape_media第4源 - full_sync_all.sh:9c步moments(media前,同轮图片即可本地化)+汇总加moments计数 - scheduler:4h messages job 末尾追加 moments --max-pages 3 浅扫(全量翻页 由每日full承担);测试断言7条命令序列 - tests/test_scrape_moments.py:归一化(秒→ms/空安全/坏createTime)+翻页内核 (两页累积停止/hashNext假单页/max_pages防死循环/丢id行/坏信封)+图片本地化 (一帖两图/幂等跳过/非http跳过+500容错);全套440测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -559,13 +559,12 @@ async def test_safe_page_run_full_invokes_bash_full_sync(engine, monkeypatch):
|
||||
|
||||
|
||||
async def test_safe_page_run_messages_builds_scrape_command(engine, monkeypatch):
|
||||
"""messages job = python scripts/scrape_via_page.py --entity messages。"""
|
||||
"""messages job = friends→wa-friends→四类消息→moments 浅扫的命令序列。"""
|
||||
sched = SyncScheduler(engine=engine, max_conversations_per_run=123)
|
||||
captured: dict = {}
|
||||
commands: list[list[str]] = []
|
||||
|
||||
async def fake_exec(*args, cwd=None, env=None):
|
||||
captured["args"] = list(args)
|
||||
captured["env"] = env
|
||||
commands.append(list(args))
|
||||
return _FakeProc(0)
|
||||
|
||||
monkeypatch.setattr(scheduler_module.asyncio, "create_subprocess_exec", fake_exec)
|
||||
@@ -573,13 +572,25 @@ async def test_safe_page_run_messages_builds_scrape_command(engine, monkeypatch)
|
||||
await sched._safe_page_run("messages")
|
||||
|
||||
# 2026-08-21: messages job 是 friends→wa-friends→四类消息的命令序列,
|
||||
# 每条都调 scrape_via_page.py;最后一次执行的命令参数可断言
|
||||
args = captured["args"]
|
||||
assert args[1].endswith("scrape_via_page.py")
|
||||
i = args.index("--entity")
|
||||
assert args[i + 1] in ("messages", "wa-messages", "group-messages", "wa-group-messages")
|
||||
j = args.index("--max-conversations")
|
||||
assert args[j + 1] == "99999"
|
||||
# 每条都调 scrape_via_page.py;2026-08-25 末尾追加 moments 浅扫
|
||||
assert len(commands) == 7
|
||||
for args in commands:
|
||||
assert args[1].endswith("scrape_via_page.py")
|
||||
|
||||
def entity_of(args: list[str]) -> str:
|
||||
return args[args.index("--entity") + 1]
|
||||
|
||||
assert entity_of(commands[0]) == "friends"
|
||||
assert entity_of(commands[1]) == "wa-friends"
|
||||
for args in commands[2:6]:
|
||||
assert entity_of(args) in (
|
||||
"messages", "wa-messages", "group-messages", "wa-group-messages",
|
||||
)
|
||||
assert args[args.index("--max-conversations") + 1] == "99999"
|
||||
# moments 浅扫在序列末尾:每员工最多 3 页
|
||||
moments_cmd = commands[-1]
|
||||
assert entity_of(moments_cmd) == "moments"
|
||||
assert moments_cmd[moments_cmd.index("--max-pages") + 1] == "3"
|
||||
|
||||
|
||||
async def test_safe_page_run_nonzero_rc_is_logged_not_raised(engine, monkeypatch, capsys):
|
||||
|
||||
Reference in New Issue
Block a user