"""Tests for the WeChat message scraper (Task 1.6). Drives the real backward-cursor pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. Two pages are served: * page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true`` * page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false`` The scraper must yield exactly ``[m1, m2, m3]`` (the ``-2`` divider dropped), map fields per the live-capture names, AND the 2nd request body's ``end`` must equal ``E1 - 1`` (proving the backward cursor advances by one ms). """ from __future__ import annotations import urllib.parse import httpx import respx from app.scraper.wechat import scrape_wechat_messages from app.session.yunke_client import YunkeClient from tests.cassettes.msg_chats_fixture import ( E1, E2, MESSAGE_RECORDS_PAGE1, MESSAGE_RECORDS_PAGE2, make_msg_chats_response, ) BASE = "https://crm2.yunkecn.com" MSG_CHATS_URL = f"{BASE}/wechatweb/msg/chats" USERID = "a" * 32 COMPANY = "8m0atd" WECHAT_ID = "wxid_staff_alice" FRIEND_ID = "wxid_friend_0001" START = 1 END = 1784518000000 # the initial upper-bound cursor def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY) def _form(sent: httpx.Request) -> dict[str, str]: """Parse the form-encoded body of an outgoing request into a flat dict. ``keep_blank_values=True`` so an empty form field (e.g. ``types=``) is preserved — the real API contract treats the empty ``types`` field as "no filter", so its presence in the body is part of what we assert. """ parsed = urllib.parse.parse_qs(sent.content.decode(), keep_blank_values=True) return {k: v[0] for k, v in parsed.items()} # --------------------------------------------------------------------------- # Test A: end-to-end backward-cursor pagination + divider drop + field mapping # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_messages_drops_divider_and_paginates_backward(): page1 = make_msg_chats_response( MESSAGE_RECORDS_PAGE1, end=E1, has_last=True, trace_id="trace-1" ) page2 = make_msg_chats_response( MESSAGE_RECORDS_PAGE2, end=E2, has_last=False, trace_id="trace-2" ) route = respx.post(MSG_CHATS_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), ] ) client = _primed_client() try: results = [ r async for r in scrape_wechat_messages( client, WECHAT_ID, FRIEND_ID, START, END ) ] finally: await client.aclose() # --- Exactly 2 calls: hasLast=false on page 2 means no 3rd fetch. --- assert route.call_count == 2 # --- Exactly [m1, m2, m3] yielded (the -2 divider dropped). --- assert [r["msg_id"] for r in results] == ["1000001", "1000002", "999999"] assert len(results) == 3 r1, r2, r3 = results # m1 — text / direct / incoming. assert r1["msg_id"] == "1000001" assert r1["channel"] == "wechat" assert r1["owner_account_id"] == WECHAT_ID assert r1["talker_id"] == "wxid_friend_0001" assert r1["ori_talker"] == "wxid_friend_0001" assert r1["room_id"] == "" # raw value carried through assert r1["is_send"] == 0 # pulled from `mine` assert r1["type"] == "text" # normalized label (design §4.3) assert r1["type_raw"] == 1 # original raw code preserved assert r1["text"] == "hi alice" assert r1["timestamp"] == E1 + 600000 assert r1["sender_name"] == "Alice Wang" assert r1["sender_head_url"] == "https://cdn.example.com/alice.png" assert r1["raw_json"] == MESSAGE_RECORDS_PAGE1[0] # m2 — image (legacy) / chatroom / outgoing. assert r2["msg_id"] == "1000002" assert r2["room_id"] == "room12345@chatroom" assert r2["ori_talker"] == "room12345@chatroom" assert r2["is_send"] == 1 assert r2["type"] == "image_legacy" # code 3 -> legacy image (phase0 §2) assert r2["type_raw"] == 3 # raw image-type code preserved assert r2["file_url"] == "https://cdn.example.com/img2.jpg" assert r2["file_th_url"] == "https://cdn.example.com/img2_th.jpg" assert r2["file_hd_url"] == "https://cdn.example.com/img2_hd.jpg" assert r2["md5"] == "md5img2" assert r2["md5_th"] == "md5img2th" assert r2["md5_hd"] == "md5img2hd" assert r2["img_path"] == "/tmp/img2.jpg" assert r2["original_image"] == "/tmp/img2_orig.jpg" assert r2["pic_type"] == 1 assert r2["raw_json"] == MESSAGE_RECORDS_PAGE1[2] # m3 — voice / transcription. assert r3["msg_id"] == "999999" assert r3["type"] == "unknown" assert r3["type_raw"] == 34 # raw voice-type code preserved assert r3["duration"] == 5 assert r3["transcription_text"] == "hello from yesterday" assert r3["pay_msg_id"] == "pay-abc-123" assert r3["refer_msg"] == {"text": "earlier msg", "msgSvrId": "888888"} assert r3["raw_json"] == MESSAGE_RECORDS_PAGE2[0] # --- Backward cursor advance: 2nd request body's end == E1 - 1. --- first_req = route.calls[0].request second_req = route.calls[1].request assert _form(first_req)["end"] == str(END) assert _form(second_req)["end"] == str(E1 - 1) # --- Form-encoding is actually sent (NOT JSON). Content-type overridden. --- assert ( first_req.headers["content-type"] == "application/x-www-form-urlencoded" ) form_first = _form(first_req) assert form_first["wechatId"] == WECHAT_ID assert form_first["friendId"] == FRIEND_ID assert form_first["types"] == "" assert form_first["start"] == str(START) assert form_first["end"] == str(END) # --------------------------------------------------------------------------- # Test B: hasLast=false on first page -> stop after one call # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_messages_stops_when_has_last_false_on_first_page(): """A single page with hasLast=false: scraper yields the records and stops.""" page = make_msg_chats_response( [MESSAGE_RECORDS_PAGE2[0]], end=E2, has_last=False ) route = respx.post(MSG_CHATS_URL).mock( return_value=httpx.Response(200, json=page) ) client = _primed_client() try: results = [ r async for r in scrape_wechat_messages( client, WECHAT_ID, FRIEND_ID, START, END ) ] finally: await client.aclose() assert [r["msg_id"] for r in results] == ["999999"] assert route.call_count == 1 # --------------------------------------------------------------------------- # Test C: empty messages array on the first page -> stop without a 3rd fetch # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_messages_empty_first_page_yields_nothing(): """An empty messages array is a defensive stop signal — no rows, one call.""" page = make_msg_chats_response([], end=E1, has_last=True) route = respx.post(MSG_CHATS_URL).mock( return_value=httpx.Response(200, json=page) ) client = _primed_client() try: results = [ r async for r in scrape_wechat_messages( client, WECHAT_ID, FRIEND_ID, START, END ) ] finally: await client.aclose() assert results == [] assert route.call_count == 1