267 lines
9.9 KiB
Python
267 lines
9.9 KiB
Python
|
"""scrape_moments 归一化 + 翻页内核 + 图片本地化测试（不启浏览器）。
|
|||
|
|
|
|||
|
|
云客接口事实（2026-08-25 真机抓包，已验证）：
|
|||
|
|
``POST /wechatweb/wechatFriendster/getFriendsterInfo``，form 小驼峰字段
|
|||
|
|
``startTime``/``endTime``(空串)/``pageNum``(从1)/``pageSize``/``wechatId``；
|
|||
|
|
信封 ``data.data[]`` + ``data.hashNext``；行的 ``createTime`` 是**秒**
|
|||
|
|
（与消息接口的 ms 不同，归一化 ×1000）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import hashlib
|
|||
|
|
|
|||
|
|
import httpx
|
|||
|
|
import respx
|
|||
|
|
import sqlalchemy as sa
|
|||
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
|||
|
|
|
|||
|
|
from app.db.base import Base
|
|||
|
|
from app.db.models import Media, MomentsPost
|
|||
|
|
from app.scraper.media import download_moments_images
|
|||
|
|
from app.session.yunke_client import YunkeClient
|
|||
|
|
from app.storage.object_store import ObjectStore
|
|||
|
|
from scripts.scrape_via_page import _iter_moments_pages, _normalize_moments
|
|||
|
|
|
|||
|
|
RAW = { # 2026-08-25 真机抓包样本（截取关键字段）
|
|||
|
|
"id": "69e082a3c3524a7fe3ac53b4", "wechatId": "wxid_wo97u1u2x11w22",
|
|||
|
|
"snsId": "-3546042415332912522", "friendsterName": "Yoyo-HESU",
|
|||
|
|
"friendsterWXId": "wxid_wo97u1u2x11w22",
|
|||
|
|
"friendsterWXIdImg": "https://wx.qlogo.cn/mmhead/ver_1/vsHn.../0",
|
|||
|
|
"content": "Welcome to HESU", "type": "1", "createTime": "1776302051", # 秒！
|
|||
|
|
"praise": [{"name": "E**U", "praiseId": "wxid***r822", "praiseImg": "https://wx.qlogo.cn/p/0"}],
|
|||
|
|
"comment": [],
|
|||
|
|
"imgUrlsList": ["https://wx.oss.yunkecn.com/friendstr/8m0atd/16/b1/ea/xx.jpg"],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# _normalize_moments
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_normalize_moments_converts_seconds_to_ms():
|
|||
|
|
rec = _normalize_moments(RAW)
|
|||
|
|
assert rec["id"] == "69e082a3c3524a7fe3ac53b4"
|
|||
|
|
assert rec["create_time"] == 1776302051 * 1000 # 秒→ms
|
|||
|
|
assert rec["author_name"] == "Yoyo-HESU"
|
|||
|
|
assert rec["praise_json"][0]["name"] == "E**U"
|
|||
|
|
assert rec["image_urls"] == RAW["imgUrlsList"]
|
|||
|
|
assert rec["raw_json"]["snsId"] == "-3546042415332912522"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_normalize_moments_null_safe():
|
|||
|
|
rec = _normalize_moments({})
|
|||
|
|
assert rec["id"] is None # 缺 id 行由调用方丢弃
|
|||
|
|
assert rec["create_time"] is None
|
|||
|
|
assert rec["praise_json"] == []
|
|||
|
|
assert rec["image_urls"] == []
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_normalize_moments_bad_create_time_is_none():
|
|||
|
|
rec = _normalize_moments({**RAW, "createTime": "not-a-number"})
|
|||
|
|
assert rec["create_time"] is None
|
|||
|
|
assert rec["id"] == RAW["id"] # 其余字段不受影响
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# _iter_moments_pages（翻页内核，fetch 回调注入）
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _page(rows: list[dict], hash_next: bool) -> dict:
|
|||
|
|
return {"code": "10000", "data": {"hashNext": hash_next, "data": rows}}
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def _noop_sleep(_seconds: float) -> None:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_iter_moments_pages_accumulates_two_pages_and_stops():
|
|||
|
|
"""第一页 hashNext=True → 继续；第二页 hashNext=False → 停（恰好 2 次请求）。"""
|
|||
|
|
bodies: list[dict] = []
|
|||
|
|
|
|||
|
|
async def fetch(body: dict) -> dict:
|
|||
|
|
bodies.append(body)
|
|||
|
|
return _page([RAW] if body["pageNum"] == 1 else [{**RAW, "id": "second"}], body["pageNum"] == 1)
|
|||
|
|
|
|||
|
|
records = await _iter_moments_pages(
|
|||
|
|
fetch, "wxid_wo97u1u2x11w22", page_size=100, max_pages=50, sleep=_noop_sleep
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
assert [r["id"] for r in records] == [RAW["id"], "second"]
|
|||
|
|
assert records[0]["create_time"] == 1776302051 * 1000
|
|||
|
|
# 恰好两次请求：hashNext=False 即止
|
|||
|
|
assert len(bodies) == 2
|
|||
|
|
# form 字段与真机抓包一致（小驼峰，pageNum 从 1 起）
|
|||
|
|
assert bodies[0] == {
|
|||
|
|
"startTime": "", "endTime": "", "pageNum": 1, "pageSize": 100,
|
|||
|
|
"wechatId": "wxid_wo97u1u2x11w22",
|
|||
|
|
}
|
|||
|
|
assert bodies[1]["pageNum"] == 2
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_iter_moments_pages_stops_when_hash_next_falsy():
|
|||
|
|
"""单页 hashNext=False → 只请求一次。"""
|
|||
|
|
calls: list[int] = []
|
|||
|
|
|
|||
|
|
async def fetch(body: dict) -> dict:
|
|||
|
|
calls.append(body["pageNum"])
|
|||
|
|
return _page([RAW], hash_next=False)
|
|||
|
|
|
|||
|
|
records = await _iter_moments_pages(fetch, "wx", sleep=_noop_sleep)
|
|||
|
|
|
|||
|
|
assert len(records) == 1
|
|||
|
|
assert calls == [1]
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_iter_moments_pages_respects_max_pages_cap():
|
|||
|
|
"""hashNext 永真 → max_pages 页上限防死循环（4h 调度浅扫 max-pages 3 靠它）。"""
|
|||
|
|
calls: list[int] = []
|
|||
|
|
|
|||
|
|
async def fetch(body: dict) -> dict:
|
|||
|
|
calls.append(body["pageNum"])
|
|||
|
|
return _page([{**RAW, "id": f"id-{body['pageNum']}"}], hash_next=True)
|
|||
|
|
|
|||
|
|
records = await _iter_moments_pages(fetch, "wx", max_pages=3, sleep=_noop_sleep)
|
|||
|
|
|
|||
|
|
assert len(calls) == 3
|
|||
|
|
assert [r["id"] for r in records] == ["id-1", "id-2", "id-3"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_iter_moments_pages_drops_null_id_rows_and_bad_envelope():
|
|||
|
|
"""缺 id 行丢弃；data 非 dict / data.data 非列表 → 停止不抛。"""
|
|||
|
|
async def fetch(body: dict) -> dict:
|
|||
|
|
if body["pageNum"] == 1:
|
|||
|
|
return _page([RAW, {"content": "no id"}], hash_next=True)
|
|||
|
|
return _page([], hash_next=False)
|
|||
|
|
|
|||
|
|
records = await _iter_moments_pages(fetch, "wx", sleep=_noop_sleep)
|
|||
|
|
assert [r["id"] for r in records] == [RAW["id"]]
|
|||
|
|
|
|||
|
|
async def fetch_bad(_body: dict) -> dict:
|
|||
|
|
return {"code": "10000", "data": None}
|
|||
|
|
|
|||
|
|
assert await _iter_moments_pages(fetch_bad, "wx", sleep=_noop_sleep) == []
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# download_moments_images（朋友圈图片本地化）
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
BASE = "https://crm2.yunkecn.com"
|
|||
|
|
OSS_A = "https://wx.oss.yunkecn.com/friendstr/8m0atd/16/b1/ea/a.jpg"
|
|||
|
|
OSS_B = "https://wx.oss.yunkecn.com/friendstr/8m0atd/16/b1/ea/b.jpg"
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def _make_engine():
|
|||
|
|
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
|||
|
|
async with engine.begin() as conn:
|
|||
|
|
await conn.run_sync(Base.metadata.create_all)
|
|||
|
|
return engine
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _primed_client() -> YunkeClient:
|
|||
|
|
return YunkeClient(base_url=BASE, userid="a" * 32, company="8m0atd")
|
|||
|
|
|
|||
|
|
|
|||
|
|
@respx.mock
|
|||
|
|
async def test_download_moments_images_localizes_all_urls(tmp_path):
|
|||
|
|
"""一帖两图(image_urls JSON 数组)全部下载入 media 表,media_type='image'。"""
|
|||
|
|
store = ObjectStore(tmp_path)
|
|||
|
|
engine = await _make_engine()
|
|||
|
|
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
|||
|
|
|
|||
|
|
blob_a, blob_b = b"\xff\xd8\xff\xe0jpg-a", b"\xff\xd8\xff\xe0jpg-b"
|
|||
|
|
respx.get(OSS_A).mock(return_value=httpx.Response(200, content=blob_a))
|
|||
|
|
respx.get(OSS_B).mock(return_value=httpx.Response(200, content=blob_b))
|
|||
|
|
|
|||
|
|
client = _primed_client()
|
|||
|
|
try:
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
session.add(MomentsPost(
|
|||
|
|
id="p1", wechat_id="wx1", image_urls=[OSS_A, OSS_B],
|
|||
|
|
))
|
|||
|
|
await session.commit()
|
|||
|
|
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
attempted, localized = await download_moments_images(client, store, session)
|
|||
|
|
await session.commit()
|
|||
|
|
|
|||
|
|
assert (attempted, localized) == (2, 2)
|
|||
|
|
for blob, url in ((blob_a, OSS_A), (blob_b, OSS_B)):
|
|||
|
|
sha = hashlib.sha256(blob).hexdigest()
|
|||
|
|
assert (tmp_path / sha[:2] / f"{sha}.jpg").is_file()
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
media = (await session.execute(
|
|||
|
|
sa.select(Media).where(Media.media_hash == sha)
|
|||
|
|
)).scalar_one()
|
|||
|
|
assert media.media_type == "image"
|
|||
|
|
assert media.channel == "wechat"
|
|||
|
|
assert media.original_url == url
|
|||
|
|
assert media.mime_type == "image/jpeg"
|
|||
|
|
finally:
|
|||
|
|
await client.aclose()
|
|||
|
|
await engine.dispose()
|
|||
|
|
|
|||
|
|
|
|||
|
|
@respx.mock
|
|||
|
|
async def test_download_moments_images_idempotent_via_original_url(tmp_path):
|
|||
|
|
"""MomentsPost 无 media_hash 回链列 → 已有同 original_url 的 Media 行即跳过
|
|||
|
|
(第二轮 attempted=0,不触网)。"""
|
|||
|
|
store = ObjectStore(tmp_path)
|
|||
|
|
engine = await _make_engine()
|
|||
|
|
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
|||
|
|
|
|||
|
|
respx.get(OSS_A).mock(return_value=httpx.Response(200, content=b"blob"))
|
|||
|
|
|
|||
|
|
client = _primed_client()
|
|||
|
|
try:
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
session.add(MomentsPost(
|
|||
|
|
id="p1", wechat_id="wx1", image_urls=[OSS_A],
|
|||
|
|
))
|
|||
|
|
await session.commit()
|
|||
|
|
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
first = await download_moments_images(client, store, session)
|
|||
|
|
await session.commit()
|
|||
|
|
assert first == (1, 1)
|
|||
|
|
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
second = await download_moments_images(client, store, session)
|
|||
|
|
await session.commit()
|
|||
|
|
assert second == (0, 0)
|
|||
|
|
finally:
|
|||
|
|
await client.aclose()
|
|||
|
|
await engine.dispose()
|
|||
|
|
|
|||
|
|
|
|||
|
|
@respx.mock
|
|||
|
|
async def test_download_moments_images_skips_non_http_and_survives_failure(tmp_path):
|
|||
|
|
"""非 http 条目跳过;下载 500 只计数不抛(下轮重试)。"""
|
|||
|
|
store = ObjectStore(tmp_path)
|
|||
|
|
engine = await _make_engine()
|
|||
|
|
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
|||
|
|
|
|||
|
|
respx.get(OSS_A).mock(return_value=httpx.Response(500))
|
|||
|
|
|
|||
|
|
client = _primed_client()
|
|||
|
|
try:
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
session.add(MomentsPost(
|
|||
|
|
id="p1", wechat_id="wx1", image_urls=[OSS_A, "not-a-url"],
|
|||
|
|
))
|
|||
|
|
await session.commit()
|
|||
|
|
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
attempted, localized = await download_moments_images(client, store, session)
|
|||
|
|
await session.commit()
|
|||
|
|
|
|||
|
|
assert (attempted, localized) == (1, 0)
|
|||
|
|
async with SessionLocal() as session:
|
|||
|
|
assert (await session.execute(sa.select(Media))).scalars().all() == []
|
|||
|
|
finally:
|
|||
|
|
await client.aclose()
|
|||
|
|
await engine.dispose()
|