浅念
5a2ff58af3
The scraper was writing a normalized type label ("text"/"unknown"/...) into
Message.type, but the column was Integer, so the storage layer coerced it to
NULL and the raw code was only recoverable from raw_json. Align with design
§4.3, which specifies two fields: `type` (normalized enum) and `type_raw`
(original code).
- Model (Message): type -> String(32) normalized label; add type_raw Integer
for the raw upstream type code.
- Scraper: _normalize_message sets both `type` (label) and `type_raw`
(coerced int from messageType/type); _is_divider now checks raw -2.
- Repos: type_raw joins _INT_FIELDS; Message.type stored as-is (string).
- Migration 96975050fa79: add type_raw + alter type Integer->String(32),
wrapped in batch_alter_table so it runs on SQLite and MySQL.
- env.py: enable render_as_batch on SQLite (dev/test) for portable batch
migrations; MySQL still uses direct ALTER.
- Tests updated end-to-end (scraper, repos, models).
Co-Authored-By: Claude <noreply@anthropic.com>
221 lines
7.6 KiB
Python
221 lines
7.6 KiB
Python
"""Tests for the WeChat message scraper (Task 1.6).
|
|
|
|
Drives the real backward-cursor pagination + normalization logic through
|
|
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
|
|
login flow is skipped) against a mocked transport. Two pages are served:
|
|
|
|
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
|
|
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
|
|
|
|
The scraper must yield exactly ``[m1, m2, m3]`` (the ``-2`` divider dropped),
|
|
map fields per the live-capture names, AND the 2nd request body's ``end`` must
|
|
equal ``E1 - 1`` (proving the backward cursor advances by one ms).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import urllib.parse
|
|
|
|
import httpx
|
|
import respx
|
|
|
|
from app.scraper.wechat import scrape_wechat_messages
|
|
from app.session.yunke_client import YunkeClient
|
|
from tests.cassettes.msg_chats_fixture import (
|
|
E1,
|
|
E2,
|
|
MESSAGE_RECORDS_PAGE1,
|
|
MESSAGE_RECORDS_PAGE2,
|
|
make_msg_chats_response,
|
|
)
|
|
|
|
BASE = "https://crm2.yunkecn.com"
|
|
MSG_CHATS_URL = f"{BASE}/wechatweb/msg/chats"
|
|
|
|
USERID = "a" * 32
|
|
COMPANY = "8m0atd"
|
|
|
|
WECHAT_ID = "wxid_staff_alice"
|
|
FRIEND_ID = "wxid_friend_0001"
|
|
START = 1
|
|
END = 1784518000000 # the initial upper-bound cursor
|
|
|
|
|
|
def _primed_client() -> YunkeClient:
|
|
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
|
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
|
|
|
|
|
def _form(sent: httpx.Request) -> dict[str, str]:
|
|
"""Parse the form-encoded body of an outgoing request into a flat dict.
|
|
|
|
``keep_blank_values=True`` so an empty form field (e.g. ``types=``) is
|
|
preserved — the real API contract treats the empty ``types`` field as
|
|
"no filter", so its presence in the body is part of what we assert.
|
|
"""
|
|
parsed = urllib.parse.parse_qs(sent.content.decode(), keep_blank_values=True)
|
|
return {k: v[0] for k, v in parsed.items()}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test A: end-to-end backward-cursor pagination + divider drop + field mapping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_messages_drops_divider_and_paginates_backward():
|
|
page1 = make_msg_chats_response(
|
|
MESSAGE_RECORDS_PAGE1, end=E1, has_last=True, trace_id="trace-1"
|
|
)
|
|
page2 = make_msg_chats_response(
|
|
MESSAGE_RECORDS_PAGE2, end=E2, has_last=False, trace_id="trace-2"
|
|
)
|
|
|
|
route = respx.post(MSG_CHATS_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_messages(
|
|
client, WECHAT_ID, FRIEND_ID, START, END
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# --- Exactly 2 calls: hasLast=false on page 2 means no 3rd fetch. ---
|
|
assert route.call_count == 2
|
|
|
|
# --- Exactly [m1, m2, m3] yielded (the -2 divider dropped). ---
|
|
assert [r["msg_id"] for r in results] == ["1000001", "1000002", "999999"]
|
|
assert len(results) == 3
|
|
|
|
r1, r2, r3 = results
|
|
|
|
# m1 — text / direct / incoming.
|
|
assert r1["msg_id"] == "1000001"
|
|
assert r1["channel"] == "wechat"
|
|
assert r1["owner_account_id"] == WECHAT_ID
|
|
assert r1["talker_id"] == "wxid_friend_0001"
|
|
assert r1["ori_talker"] == "wxid_friend_0001"
|
|
assert r1["room_id"] == "" # raw value carried through
|
|
assert r1["is_send"] == 0 # pulled from `mine`
|
|
assert r1["type"] == "text" # normalized label (design §4.3)
|
|
assert r1["type_raw"] == 1 # original raw code preserved
|
|
assert r1["text"] == "hi alice"
|
|
assert r1["timestamp"] == E1 + 600000
|
|
assert r1["sender_name"] == "Alice Wang"
|
|
assert r1["sender_head_url"] == "https://cdn.example.com/alice.png"
|
|
assert r1["raw_json"] == MESSAGE_RECORDS_PAGE1[0]
|
|
|
|
# m2 — image / chatroom / outgoing.
|
|
assert r2["msg_id"] == "1000002"
|
|
assert r2["room_id"] == "room12345@chatroom"
|
|
assert r2["ori_talker"] == "room12345@chatroom"
|
|
assert r2["is_send"] == 1
|
|
assert r2["type"] == "unknown" # unmapped code passes through
|
|
assert r2["type_raw"] == 3 # raw image-type code preserved
|
|
assert r2["file_url"] == "https://cdn.example.com/img2.jpg"
|
|
assert r2["file_th_url"] == "https://cdn.example.com/img2_th.jpg"
|
|
assert r2["file_hd_url"] == "https://cdn.example.com/img2_hd.jpg"
|
|
assert r2["md5"] == "md5img2"
|
|
assert r2["md5_th"] == "md5img2th"
|
|
assert r2["md5_hd"] == "md5img2hd"
|
|
assert r2["img_path"] == "/tmp/img2.jpg"
|
|
assert r2["original_image"] == "/tmp/img2_orig.jpg"
|
|
assert r2["pic_type"] == 1
|
|
assert r2["raw_json"] == MESSAGE_RECORDS_PAGE1[2]
|
|
|
|
# m3 — voice / transcription.
|
|
assert r3["msg_id"] == "999999"
|
|
assert r3["type"] == "unknown"
|
|
assert r3["type_raw"] == 34 # raw voice-type code preserved
|
|
assert r3["duration"] == 5
|
|
assert r3["transcription_text"] == "hello from yesterday"
|
|
assert r3["pay_msg_id"] == "pay-abc-123"
|
|
assert r3["refer_msg"] == {"text": "earlier msg", "msgSvrId": "888888"}
|
|
assert r3["raw_json"] == MESSAGE_RECORDS_PAGE2[0]
|
|
|
|
# --- Backward cursor advance: 2nd request body's end == E1 - 1. ---
|
|
first_req = route.calls[0].request
|
|
second_req = route.calls[1].request
|
|
|
|
assert _form(first_req)["end"] == str(END)
|
|
assert _form(second_req)["end"] == str(E1 - 1)
|
|
|
|
# --- Form-encoding is actually sent (NOT JSON). Content-type overridden. ---
|
|
assert (
|
|
first_req.headers["content-type"]
|
|
== "application/x-www-form-urlencoded"
|
|
)
|
|
form_first = _form(first_req)
|
|
assert form_first["wechatId"] == WECHAT_ID
|
|
assert form_first["friendId"] == FRIEND_ID
|
|
assert form_first["types"] == ""
|
|
assert form_first["start"] == str(START)
|
|
assert form_first["end"] == str(END)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test B: hasLast=false on first page -> stop after one call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_messages_stops_when_has_last_false_on_first_page():
|
|
"""A single page with hasLast=false: scraper yields the records and stops."""
|
|
page = make_msg_chats_response(
|
|
[MESSAGE_RECORDS_PAGE2[0]], end=E2, has_last=False
|
|
)
|
|
route = respx.post(MSG_CHATS_URL).mock(
|
|
return_value=httpx.Response(200, json=page)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_messages(
|
|
client, WECHAT_ID, FRIEND_ID, START, END
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert [r["msg_id"] for r in results] == ["999999"]
|
|
assert route.call_count == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test C: empty messages array on the first page -> stop without a 3rd fetch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_messages_empty_first_page_yields_nothing():
|
|
"""An empty messages array is a defensive stop signal — no rows, one call."""
|
|
page = make_msg_chats_response([], end=E1, has_last=True)
|
|
route = respx.post(MSG_CHATS_URL).mock(
|
|
return_value=httpx.Response(200, json=page)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_messages(
|
|
client, WECHAT_ID, FRIEND_ID, START, END
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert results == []
|
|
assert route.call_count == 1
|