feat(scraper): moments 实体(浏览器fetch getFriendsterInfo,秒→ms,hashNext翻页,接入full_sync+4h调度+图片media)
- scrape_via_page:_normalize_moments(createTime秒×1000,缺id丢弃)+ _iter_moments_pages 翻页内核(fetch回调注入,sleep可注入,hashNext假/ 信封异常/max_pages 上限即止)+ scrape_moments(逐员工遍历wechat_account, form小驼峰,首页code≠10000告警cookie失效,换页重试惯例)+CLI注册moments - media:download_moments_images(image_urls JSON数组展开去重,MomentsPost无 回链列→用Media.original_url做已下载标记,分片IN防SQLite变量上限,批间commit) 接入scrape_media第4源 - full_sync_all.sh:9c步moments(media前,同轮图片即可本地化)+汇总加moments计数 - scheduler:4h messages job 末尾追加 moments --max-pages 3 浅扫(全量翻页 由每日full承担);测试断言7条命令序列 - tests/test_scrape_moments.py:归一化(秒→ms/空安全/坏createTime)+翻页内核 (两页累积停止/hashNext假单页/max_pages防死循环/丢id行/坏信封)+图片本地化 (一帖两图/幂等跳过/非http跳过+500容错);全套440测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,266 @@
|
||||
"""scrape_moments 归一化 + 翻页内核 + 图片本地化测试（不启浏览器）。
|
||||
|
||||
云客接口事实（2026-08-25 真机抓包，已验证）：
|
||||
``POST /wechatweb/wechatFriendster/getFriendsterInfo``，form 小驼峰字段
|
||||
``startTime``/``endTime``(空串)/``pageNum``(从1)/``pageSize``/``wechatId``；
|
||||
信封 ``data.data[]`` + ``data.hashNext``；行的 ``createTime`` 是**秒**
|
||||
（与消息接口的 ms 不同，归一化 ×1000）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import httpx
|
||||
import respx
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db.base import Base
|
||||
from app.db.models import Media, MomentsPost
|
||||
from app.scraper.media import download_moments_images
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from app.storage.object_store import ObjectStore
|
||||
from scripts.scrape_via_page import _iter_moments_pages, _normalize_moments
|
||||
|
||||
RAW = { # 2026-08-25 真机抓包样本（截取关键字段）
|
||||
"id": "69e082a3c3524a7fe3ac53b4", "wechatId": "wxid_wo97u1u2x11w22",
|
||||
"snsId": "-3546042415332912522", "friendsterName": "Yoyo-HESU",
|
||||
"friendsterWXId": "wxid_wo97u1u2x11w22",
|
||||
"friendsterWXIdImg": "https://wx.qlogo.cn/mmhead/ver_1/vsHn.../0",
|
||||
"content": "Welcome to HESU", "type": "1", "createTime": "1776302051", # 秒！
|
||||
"praise": [{"name": "E**U", "praiseId": "wxid***r822", "praiseImg": "https://wx.qlogo.cn/p/0"}],
|
||||
"comment": [],
|
||||
"imgUrlsList": ["https://wx.oss.yunkecn.com/friendstr/8m0atd/16/b1/ea/xx.jpg"],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _normalize_moments
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_normalize_moments_converts_seconds_to_ms():
|
||||
rec = _normalize_moments(RAW)
|
||||
assert rec["id"] == "69e082a3c3524a7fe3ac53b4"
|
||||
assert rec["create_time"] == 1776302051 * 1000 # 秒→ms
|
||||
assert rec["author_name"] == "Yoyo-HESU"
|
||||
assert rec["praise_json"][0]["name"] == "E**U"
|
||||
assert rec["image_urls"] == RAW["imgUrlsList"]
|
||||
assert rec["raw_json"]["snsId"] == "-3546042415332912522"
|
||||
|
||||
|
||||
def test_normalize_moments_null_safe():
|
||||
rec = _normalize_moments({})
|
||||
assert rec["id"] is None # 缺 id 行由调用方丢弃
|
||||
assert rec["create_time"] is None
|
||||
assert rec["praise_json"] == []
|
||||
assert rec["image_urls"] == []
|
||||
|
||||
|
||||
def test_normalize_moments_bad_create_time_is_none():
|
||||
rec = _normalize_moments({**RAW, "createTime": "not-a-number"})
|
||||
assert rec["create_time"] is None
|
||||
assert rec["id"] == RAW["id"] # 其余字段不受影响
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _iter_moments_pages（翻页内核，fetch 回调注入）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _page(rows: list[dict], hash_next: bool) -> dict:
|
||||
return {"code": "10000", "data": {"hashNext": hash_next, "data": rows}}
|
||||
|
||||
|
||||
async def _noop_sleep(_seconds: float) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def test_iter_moments_pages_accumulates_two_pages_and_stops():
|
||||
"""第一页 hashNext=True → 继续；第二页 hashNext=False → 停（恰好 2 次请求）。"""
|
||||
bodies: list[dict] = []
|
||||
|
||||
async def fetch(body: dict) -> dict:
|
||||
bodies.append(body)
|
||||
return _page([RAW] if body["pageNum"] == 1 else [{**RAW, "id": "second"}], body["pageNum"] == 1)
|
||||
|
||||
records = await _iter_moments_pages(
|
||||
fetch, "wxid_wo97u1u2x11w22", page_size=100, max_pages=50, sleep=_noop_sleep
|
||||
)
|
||||
|
||||
assert [r["id"] for r in records] == [RAW["id"], "second"]
|
||||
assert records[0]["create_time"] == 1776302051 * 1000
|
||||
# 恰好两次请求：hashNext=False 即止
|
||||
assert len(bodies) == 2
|
||||
# form 字段与真机抓包一致（小驼峰，pageNum 从 1 起）
|
||||
assert bodies[0] == {
|
||||
"startTime": "", "endTime": "", "pageNum": 1, "pageSize": 100,
|
||||
"wechatId": "wxid_wo97u1u2x11w22",
|
||||
}
|
||||
assert bodies[1]["pageNum"] == 2
|
||||
|
||||
|
||||
async def test_iter_moments_pages_stops_when_hash_next_falsy():
|
||||
"""单页 hashNext=False → 只请求一次。"""
|
||||
calls: list[int] = []
|
||||
|
||||
async def fetch(body: dict) -> dict:
|
||||
calls.append(body["pageNum"])
|
||||
return _page([RAW], hash_next=False)
|
||||
|
||||
records = await _iter_moments_pages(fetch, "wx", sleep=_noop_sleep)
|
||||
|
||||
assert len(records) == 1
|
||||
assert calls == [1]
|
||||
|
||||
|
||||
async def test_iter_moments_pages_respects_max_pages_cap():
|
||||
"""hashNext 永真 → max_pages 页上限防死循环（4h 调度浅扫 max-pages 3 靠它）。"""
|
||||
calls: list[int] = []
|
||||
|
||||
async def fetch(body: dict) -> dict:
|
||||
calls.append(body["pageNum"])
|
||||
return _page([{**RAW, "id": f"id-{body['pageNum']}"}], hash_next=True)
|
||||
|
||||
records = await _iter_moments_pages(fetch, "wx", max_pages=3, sleep=_noop_sleep)
|
||||
|
||||
assert len(calls) == 3
|
||||
assert [r["id"] for r in records] == ["id-1", "id-2", "id-3"]
|
||||
|
||||
|
||||
async def test_iter_moments_pages_drops_null_id_rows_and_bad_envelope():
|
||||
"""缺 id 行丢弃；data 非 dict / data.data 非列表 → 停止不抛。"""
|
||||
async def fetch(body: dict) -> dict:
|
||||
if body["pageNum"] == 1:
|
||||
return _page([RAW, {"content": "no id"}], hash_next=True)
|
||||
return _page([], hash_next=False)
|
||||
|
||||
records = await _iter_moments_pages(fetch, "wx", sleep=_noop_sleep)
|
||||
assert [r["id"] for r in records] == [RAW["id"]]
|
||||
|
||||
async def fetch_bad(_body: dict) -> dict:
|
||||
return {"code": "10000", "data": None}
|
||||
|
||||
assert await _iter_moments_pages(fetch_bad, "wx", sleep=_noop_sleep) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# download_moments_images（朋友圈图片本地化）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BASE = "https://crm2.yunkecn.com"
|
||||
OSS_A = "https://wx.oss.yunkecn.com/friendstr/8m0atd/16/b1/ea/a.jpg"
|
||||
OSS_B = "https://wx.oss.yunkecn.com/friendstr/8m0atd/16/b1/ea/b.jpg"
|
||||
|
||||
|
||||
async def _make_engine():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
return engine
|
||||
|
||||
|
||||
def _primed_client() -> YunkeClient:
|
||||
return YunkeClient(base_url=BASE, userid="a" * 32, company="8m0atd")
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_download_moments_images_localizes_all_urls(tmp_path):
|
||||
"""一帖两图(image_urls JSON 数组)全部下载入 media 表,media_type='image'。"""
|
||||
store = ObjectStore(tmp_path)
|
||||
engine = await _make_engine()
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
blob_a, blob_b = b"\xff\xd8\xff\xe0jpg-a", b"\xff\xd8\xff\xe0jpg-b"
|
||||
respx.get(OSS_A).mock(return_value=httpx.Response(200, content=blob_a))
|
||||
respx.get(OSS_B).mock(return_value=httpx.Response(200, content=blob_b))
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
session.add(MomentsPost(
|
||||
id="p1", wechat_id="wx1", image_urls=[OSS_A, OSS_B],
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
async with SessionLocal() as session:
|
||||
attempted, localized = await download_moments_images(client, store, session)
|
||||
await session.commit()
|
||||
|
||||
assert (attempted, localized) == (2, 2)
|
||||
for blob, url in ((blob_a, OSS_A), (blob_b, OSS_B)):
|
||||
sha = hashlib.sha256(blob).hexdigest()
|
||||
assert (tmp_path / sha[:2] / f"{sha}.jpg").is_file()
|
||||
async with SessionLocal() as session:
|
||||
media = (await session.execute(
|
||||
sa.select(Media).where(Media.media_hash == sha)
|
||||
)).scalar_one()
|
||||
assert media.media_type == "image"
|
||||
assert media.channel == "wechat"
|
||||
assert media.original_url == url
|
||||
assert media.mime_type == "image/jpeg"
|
||||
finally:
|
||||
await client.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_download_moments_images_idempotent_via_original_url(tmp_path):
|
||||
"""MomentsPost 无 media_hash 回链列 → 已有同 original_url 的 Media 行即跳过
|
||||
(第二轮 attempted=0,不触网)。"""
|
||||
store = ObjectStore(tmp_path)
|
||||
engine = await _make_engine()
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
respx.get(OSS_A).mock(return_value=httpx.Response(200, content=b"blob"))
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
session.add(MomentsPost(
|
||||
id="p1", wechat_id="wx1", image_urls=[OSS_A],
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
async with SessionLocal() as session:
|
||||
first = await download_moments_images(client, store, session)
|
||||
await session.commit()
|
||||
assert first == (1, 1)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
second = await download_moments_images(client, store, session)
|
||||
await session.commit()
|
||||
assert second == (0, 0)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_download_moments_images_skips_non_http_and_survives_failure(tmp_path):
|
||||
"""非 http 条目跳过;下载 500 只计数不抛(下轮重试)。"""
|
||||
store = ObjectStore(tmp_path)
|
||||
engine = await _make_engine()
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
respx.get(OSS_A).mock(return_value=httpx.Response(500))
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
session.add(MomentsPost(
|
||||
id="p1", wechat_id="wx1", image_urls=[OSS_A, "not-a-url"],
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
async with SessionLocal() as session:
|
||||
attempted, localized = await download_moments_images(client, store, session)
|
||||
await session.commit()
|
||||
|
||||
assert (attempted, localized) == (1, 0)
|
||||
async with SessionLocal() as session:
|
||||
assert (await session.execute(sa.select(Media))).scalars().all() == []
|
||||
finally:
|
||||
await client.aclose()
|
||||
await engine.dispose()
|
||||
@@ -559,13 +559,12 @@ async def test_safe_page_run_full_invokes_bash_full_sync(engine, monkeypatch):
|
||||
|
||||
|
||||
async def test_safe_page_run_messages_builds_scrape_command(engine, monkeypatch):
|
||||
"""messages job = python scripts/scrape_via_page.py --entity messages。"""
|
||||
"""messages job = friends→wa-friends→四类消息→moments 浅扫的命令序列。"""
|
||||
sched = SyncScheduler(engine=engine, max_conversations_per_run=123)
|
||||
captured: dict = {}
|
||||
commands: list[list[str]] = []
|
||||
|
||||
async def fake_exec(*args, cwd=None, env=None):
|
||||
captured["args"] = list(args)
|
||||
captured["env"] = env
|
||||
commands.append(list(args))
|
||||
return _FakeProc(0)
|
||||
|
||||
monkeypatch.setattr(scheduler_module.asyncio, "create_subprocess_exec", fake_exec)
|
||||
@@ -573,13 +572,25 @@ async def test_safe_page_run_messages_builds_scrape_command(engine, monkeypatch)
|
||||
await sched._safe_page_run("messages")
|
||||
|
||||
# 2026-08-21: messages job 是 friends→wa-friends→四类消息的命令序列,
|
||||
# 每条都调 scrape_via_page.py;最后一次执行的命令参数可断言
|
||||
args = captured["args"]
|
||||
assert args[1].endswith("scrape_via_page.py")
|
||||
i = args.index("--entity")
|
||||
assert args[i + 1] in ("messages", "wa-messages", "group-messages", "wa-group-messages")
|
||||
j = args.index("--max-conversations")
|
||||
assert args[j + 1] == "99999"
|
||||
# 每条都调 scrape_via_page.py;2026-08-25 末尾追加 moments 浅扫
|
||||
assert len(commands) == 7
|
||||
for args in commands:
|
||||
assert args[1].endswith("scrape_via_page.py")
|
||||
|
||||
def entity_of(args: list[str]) -> str:
|
||||
return args[args.index("--entity") + 1]
|
||||
|
||||
assert entity_of(commands[0]) == "friends"
|
||||
assert entity_of(commands[1]) == "wa-friends"
|
||||
for args in commands[2:6]:
|
||||
assert entity_of(args) in (
|
||||
"messages", "wa-messages", "group-messages", "wa-group-messages",
|
||||
)
|
||||
assert args[args.index("--max-conversations") + 1] == "99999"
|
||||
# moments 浅扫在序列末尾:每员工最多 3 页
|
||||
moments_cmd = commands[-1]
|
||||
assert entity_of(moments_cmd) == "moments"
|
||||
assert moments_cmd[moments_cmd.index("--max-pages") + 1] == "3"
|
||||
|
||||
|
||||
async def test_safe_page_run_nonzero_rc_is_logged_not_raised(engine, monkeypatch, capsys):
|
||||
|
||||
Reference in New Issue
Block a user