"""Tests for the WhatsApp chat-group scraper. Drives the real page-number pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. WA groups reuse the SAME upstream endpoint as WA friends (``POST /qqweb/WhatsAppUser/businessFriendList``); the ``type="1"`` body parameter selects groups (``"0"`` selects friends). The response envelope is identical to the WA friends scraper's (``data.overviews`` + ``data.totalCount``), so this test reuses :func:`tests.cassettes.wa_friends_fixture.make_wa_friends_response` to build capture-shaped envelopes. The records mirror the field names returned by the WeChat ``friend_list/es`` group endpoint (``friendWxId`` / ``friendWxNickname`` / ``friendNum`` / ...) — the WA endpoint is assumed to return the same shape for groups because both channels share the yunke CRM's friend-list contract. ``_normalize_whatsapp_group`` reuses the WeChat :func:`app.scraper.wechat._normalize_chat_group` field mapping and overrides ``channel`` to ``"whatsapp"``. NOTE: the group-type switch ``type="1"`` is UNVERIFIED (云客 risk-control blocking live API calls as of 2026-08-11) — see the scraper docstring. These tests pin the STRUCTURE (body shape, normalization, channel override, pagination); only the param NAME may need a post-risk-control adjustment. The fixture ships 3 records across 2 pages of size 2, plus a terminating empty page (the documented primary stop signal for page-number endpoints): * page 1 -> 2 records, ``totalCount=3`` * page 2 -> 1 record, ``totalCount=3`` * page 3 -> empty ``overviews`` array (no more rows) The scraper must yield exactly the 3 records (normalized to ``ChatGroup`` snake_case columns + ``raw_json``, ``channel="whatsapp"``), make exactly 3 calls, and stop without a 4th. Member details are NOT surfaced by this endpoint — ``member_ids`` is ``None`` and ``member_ids_list`` is ``[]`` on every yielded record. """ from __future__ import annotations import json as _json from typing import Any import httpx import respx from app.scraper.whatsapp import scrape_whatsapp_groups from app.session.yunke_client import YunkeClient from tests.cassettes.wa_friends_fixture import make_wa_friends_response BASE = "https://crm2.yunkecn.com" WA_FRIENDS_URL = f"{BASE}/qqweb/WhatsAppUser/businessFriendList" USERID = "a" * 32 COMPANY = "8m0atd" # Realistic, capture-shaped WA group records. Field names mirror the WeChat # ``friend_list/es`` group payload (verified 2026-08-10 Playwright capture) — # the WA ``businessFriendList`` endpoint is assumed to return the SAME shape # for the group-type switch because both channels share the yunke CRM's # friend-list contract. Every field with a target ``ChatGroup`` column is # populated so the test asserts a full mapping end-to-end. (Member ids are # NOT in this payload — the endpoint surfaces only the ``friendNum`` count — # so ``member_ids`` / ``member_ids_list`` are exercised as None / [] on every # record.) WA_GROUP_RECORDS: list[dict[str, Any]] = [ { "friendWxId": "120363000000000000@g.us", # WA group id (not @chatroom) "friendWxNickname": "VIP WA Customer Group", "friendNum": 5, "friendHeadImageUrl": "https://wx.qlogo.cn/wa_group_001.png", "wxId": "8613800000001@s.whatsapp.net", # owning WA staff account "ownerName": "Alice Wang", "userId": "wa_staff_001", "phone": "8613800000001", "wechatPhone": "8613900000001", "messageCount": 4521, "sendNum": 1204, "lastChatTime": 1700000000123, "version": "v1.20260811.waroom1", }, { "friendWxId": "120363111111111111@g.us", "friendWxNickname": "East Region WA Sales", "friendNum": 8, "friendHeadImageUrl": "https://wx.qlogo.cn/wa_group_002.png", "wxId": "8613800000002@s.whatsapp.net", "ownerName": "Bob Chen", "userId": "wa_staff_002", "phone": "8613800000002", "wechatPhone": "8613900000002", "messageCount": 9821, "sendNum": 211, "lastChatTime": 1700000000456, "version": "v1.20260811.waroom2", }, { # A group whose owner has no resolved staff id (userId None) — the # scraper must still yield it; None-safe fields stay None. "friendWxId": "120363222222222222@g.us", "friendWxNickname": "External WA Coord", "friendNum": 3, "friendHeadImageUrl": "https://wx.qlogo.cn/wa_group_003.png", "wxId": "8613800000004@s.whatsapp.net", "ownerName": "Orphan WA", "userId": None, "phone": None, "wechatPhone": None, "messageCount": 12, "sendNum": 0, "lastChatTime": 1700000000789, "version": "v1.20260811.waroom3", }, ] TOTAL = len(WA_GROUP_RECORDS) DEFAULT_PAGE_SIZE = 2 def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY) # --------------------------------------------------------------------------- # End-to-end: pagination + field mapping + channel override # --------------------------------------------------------------------------- @respx.mock async def test_scrape_whatsapp_groups_yields_records_and_paginates(): """Two full pages + a terminating empty page -> 3 records, 3 calls.""" page1 = make_wa_friends_response( WA_GROUP_RECORDS[:2], total_count=TOTAL, page=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_wa_friends_response( WA_GROUP_RECORDS[2:], total_count=TOTAL, page=2, page_size=DEFAULT_PAGE_SIZE ) page3 = make_wa_friends_response( [], total_count=TOTAL, page=3, page_size=DEFAULT_PAGE_SIZE ) route = respx.post(WA_FRIENDS_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), httpx.Response(200, json=page3), ] ) client = _primed_client() try: results = [ r async for r in scrape_whatsapp_groups(client, page_size=DEFAULT_PAGE_SIZE) ] finally: await client.aclose() # Pagination: exactly 3 calls (page1, page2, empty-page stop). assert route.call_count == 3 # Exactly the 3 fixture records (not duplicated, not dropped). assert len(results) == 3 assert {r["room_id"] for r in results} == { rec["friendWxId"] for rec in WA_GROUP_RECORDS } # Field mapping for each record (compare against the source record). for result, record in zip(results, WA_GROUP_RECORDS, strict=True): # Identity + channel override (whatsapp, NOT wechat). assert result["room_id"] == record["friendWxId"] assert result["channel"] == "whatsapp" # Naming + ownership. assert result["room_name"] == record["friendWxNickname"] assert result["owner_account_id"] == record["wxId"] assert result["owner_account_nickname"] == record["ownerName"] assert result["owner_user_id"] == record["userId"] # phone preferred, wechatPhone fallback. assert result["owner_user_phone"] == ( record["phone"] or record["wechatPhone"] ) # Counts. assert result["member_count"] == record["friendNum"] assert result["message_count"] == record["messageCount"] assert result["own_message_count"] == record["sendNum"] # Avatar + timestamps. assert result["head_image_url"] == record["friendHeadImageUrl"] assert result["last_chat_time"] == record["lastChatTime"] # Member details NOT surfaced by this endpoint -> None / []. assert result["member_ids"] is None assert result["member_ids_list"] == [] # Fields the endpoint does not carry. assert result["loss_members"] is None assert result["add_members"] is None assert result["update_time"] is None # Version carried verbatim; raw_json preserves the original record. assert result["version"] == record["version"] assert result["raw_json"] == record # --------------------------------------------------------------------------- # member_ids behaviour: always None / [] (not surfaced by this endpoint) # --------------------------------------------------------------------------- @respx.mock async def test_scrape_whatsapp_groups_emits_empty_member_ids(): """The endpoint does not surface member ids; they are always ``None`` / ``[]`` on yielded records (member mirroring deferred).""" record = dict(WA_GROUP_RECORDS[0]) page1 = make_wa_friends_response([record], total_count=1, page=1) page2 = make_wa_friends_response([], total_count=1, page=2) respx.post(WA_FRIENDS_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) client = _primed_client() try: results = [ r async for r in scrape_whatsapp_groups(client, page_size=DEFAULT_PAGE_SIZE) ] finally: await client.aclose() assert len(results) == 1 assert results[0]["member_ids"] is None assert results[0]["member_ids_list"] == [] # --------------------------------------------------------------------------- # Pagination body: page/pageSize advanced + type="1" (UNVERIFIED group switch) # --------------------------------------------------------------------------- @respx.mock async def test_scrape_whatsapp_groups_sends_pagination_body(): """Outgoing body carries page/pageSize, departmentId, and ``type="1"``. The ``type="1"`` group switch is UNVERIFIED (see scraper docstring); this test pins the current choice so a future post-risk-control adjustment is surfaced as a deliberate test update, not a silent regression. """ page1 = make_wa_friends_response( [WA_GROUP_RECORDS[0]], total_count=TOTAL, page=1, page_size=1 ) page2 = make_wa_friends_response([], total_count=TOTAL, page=2, page_size=1) route = respx.post(WA_FRIENDS_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) client = _primed_client() try: results = [ r async for r in scrape_whatsapp_groups( client, department_id="ROOT_DEPT", page_size=1 ) ] finally: await client.aclose() # One record on page 1, empty page 2 stops the loop. assert len(results) == 1 assert route.call_count == 2 body1 = _json.loads(route.calls[0].request.content.decode()) # businessFriendList uses `page` (not pageNum) + departmentId + type. assert body1["page"] == 1 assert body1["pageSize"] == 1 assert body1["departmentId"] == "ROOT_DEPT" # The crucial (UNVERIFIED) switch: type="1" selects GROUPS ("0" = friends). assert body1["type"] == "1" body2 = _json.loads(route.calls[1].request.content.decode()) assert body2["page"] == 2 assert body2["pageSize"] == 1 assert body2["type"] == "1" # --------------------------------------------------------------------------- # Empty first page -> no yields, single call # --------------------------------------------------------------------------- @respx.mock async def test_scrape_whatsapp_groups_stops_on_empty_first_page(): """If the very first page is empty, scraper stops without over-fetching.""" empty = make_wa_friends_response([], total_count=0, page=1) route = respx.post(WA_FRIENDS_URL).mock( return_value=httpx.Response(200, json=empty) ) client = _primed_client() try: results = [r async for r in scrape_whatsapp_groups(client)] finally: await client.aclose() assert results == [] assert route.call_count == 1