"""Tests for the WeChat chat-group scraper (M2). Drives the real page-number pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. The fixture ships 3 records across 2 pages of size 2, plus a terminating empty page (the documented primary stop signal for page-number endpoints): * page 1 -> 2 records, ``total=3`` * page 2 -> 1 record, ``total=3`` * page 3 -> empty ``data`` array (no more rows) The scraper must yield exactly the 3 records (normalized to ``ChatGroup`` snake_case columns + ``raw_json``), parse ``memberIds`` into ``member_ids_list`` (dropping empty segments), make exactly 3 calls, and stop without a 4th. """ from __future__ import annotations import json as _json import httpx import respx from app.scraper.wechat import scrape_wechat_groups from app.session.yunke_client import YunkeClient from tests.cassettes.chatroom_fixture import ( CHATROOM_RECORDS, DEFAULT_PAGE_SIZE, TOTAL, make_chatroom_list_response, ) BASE = "https://crm2.yunkecn.com" CHATROOM_LIST_URL = f"{BASE}/wechatweb/chatroom/getChatroomList" USERID = "a" * 32 COMPANY = "8m0atd" def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY) # --------------------------------------------------------------------------- # End-to-end: pagination + field mapping + member_ids_list parsing # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_groups_yields_records_and_paginates(): """Two full pages + a terminating empty page -> 3 records, 3 calls.""" page1 = make_chatroom_list_response( CHATROOM_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_chatroom_list_response( CHATROOM_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE ) page3 = make_chatroom_list_response( [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE ) route = respx.post(CHATROOM_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), httpx.Response(200, json=page3), ] ) client = _primed_client() try: results = [ r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE) ] finally: await client.aclose() # Pagination: exactly 3 calls (page1, page2, empty-page stop). assert route.call_count == 3 # Exactly the 3 fixture records (not duplicated, not dropped). assert len(results) == 3 assert {r["room_id"] for r in results} == { rec["chatRoomId"] for rec in CHATROOM_RECORDS } # Field mapping for each record (compare against the source record). for result, record in zip(results, CHATROOM_RECORDS, strict=True): # Identity + channel. assert result["room_id"] == record["chatRoomId"] assert result["channel"] == "wechat" # Naming + ownership. assert result["room_name"] == record["chatRoomName"] assert result["owner_account_id"] == record["wechatId"] assert result["owner_account_nickname"] == record["wechatNickName"] assert result["owner_user_id"] == record["userId"] assert result["owner_user_phone"] == record["userPhone"] # Counts. assert result["member_count"] == record["memberCount"] assert result["message_count"] == record["messageCount"] assert result["own_message_count"] == record["ownMessageCount"] assert result["loss_members"] == record["lossMembers"] assert result["add_members"] == record["addMembers"] # Avatar + timestamps. assert result["head_image_url"] == record["headImageUrl"] assert result["last_chat_time"] == record["lastChatTime"] assert result["update_time"] == record["updateTime"] # member_ids carried raw; member_ids_list parsed (empties dropped). assert result["member_ids"] == record["memberIds"] assert result["member_ids_list"] == [ seg for seg in record["memberIds"].split(";") if seg ] # Version carried verbatim; raw_json preserves the original record. assert result["version"] == record["version"] assert result["raw_json"] == record # --------------------------------------------------------------------------- # member_ids_list parsing edge cases # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_groups_drops_empty_member_id_segments(): """Trailing/empty ``memberIds`` segments are dropped from member_ids_list.""" # Record 2 has a trailing ";" -> one empty segment that must be dropped. page1 = make_chatroom_list_response( [CHATROOM_RECORDS[1]], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_chatroom_list_response([], total=1, page_num=2) respx.post(CHATROOM_LIST_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) client = _primed_client() try: results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)] finally: await client.aclose() assert len(results) == 1 # "wxid_member_b1;wxid_member_b2;" -> 2 ids (trailing empty dropped). assert results[0]["member_ids_list"] == ["wxid_member_b1", "wxid_member_b2"] @respx.mock async def test_scrape_wechat_groups_handles_missing_member_ids(): """A group with no ``memberIds`` field yields an empty member_ids_list.""" record = {"chatRoomId": "x@chatroom", "chatRoomName": "Empty"} page1 = make_chatroom_list_response([record], total=1, page_num=1) page2 = make_chatroom_list_response([], total=1, page_num=2) respx.post(CHATROOM_LIST_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) client = _primed_client() try: results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)] finally: await client.aclose() assert len(results) == 1 assert results[0]["member_ids"] is None assert results[0]["member_ids_list"] == [] # --------------------------------------------------------------------------- # Pagination body: pageNum/pageSize advanced correctly # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_groups_sends_pagination_body(): """Outgoing request body carries pageNum/pageSize; advances per page.""" page1 = make_chatroom_list_response( [CHATROOM_RECORDS[0]], total=TOTAL, page_num=1, page_size=1 ) page2 = make_chatroom_list_response([], total=TOTAL, page_num=2, page_size=1) route = respx.post(CHATROOM_LIST_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) client = _primed_client() try: results = [r async for r in scrape_wechat_groups(client, page_size=1)] finally: await client.aclose() # One record on page 1, empty page 2 stops the loop. assert len(results) == 1 assert route.call_count == 2 body1 = _json.loads(route.calls[0].request.content.decode()) assert body1["pageNum"] == 1 assert body1["pageSize"] == 1 body2 = _json.loads(route.calls[1].request.content.decode()) assert body2["pageNum"] == 2 assert body2["pageSize"] == 1 # --------------------------------------------------------------------------- # Empty first page -> no yields, single call # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_groups_stops_on_empty_first_page(): """If the very first page is empty, scraper stops without over-fetching.""" empty = make_chatroom_list_response([], total=0, page_num=1) route = respx.post(CHATROOM_LIST_URL).mock( return_value=httpx.Response(200, json=empty) ) client = _primed_client() try: results = [r async for r in scrape_wechat_groups(client)] finally: await client.aclose() assert results == [] assert route.call_count == 1