"""Tests for the WeChat staff-account scraper (Task 2.1). Drives the real page-number pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. The fixture ships 4 records across 2 pages of size 2, plus a terminating empty page (the documented primary stop signal for page-number endpoints): * page 1 -> 2 records, ``total=4`` * page 2 -> 2 records, ``total=4`` * page 3 -> empty ``data`` array (no more rows) The scraper must yield exactly the 4 records (normalized to ``WechatAccount`` snake_case columns + ``raw_json``), make exactly 3 calls, and stop without a 4th. The empty-page stop is the primary signal; ``total=4`` is only a defensive over-fetch ceiling that must NOT trigger early (we yield exactly 4 == total, so the strict-``>`` ceiling never fires here). The REQUEST body is form-urlencoded (NOT JSON) and must carry the required ``isOnJob`` / ``departmentId`` / ``startdate`` / ``enddate`` filters; the "send pagination body" test asserts those keys land on the wire. """ from __future__ import annotations from urllib.parse import parse_qs import httpx import respx from app.scraper.base import FORM_CONTENT_TYPE from app.scraper.wechat import scrape_wechat_accounts from app.session.yunke_client import YunkeClient from tests.cassettes.wxUser_fixture import ( DEFAULT_PAGE_SIZE, TOTAL, WXUSER_RECORDS, make_wxuser_list_response, ) BASE = "https://crm2.yunkecn.com" WXUSER_LIST_URL = f"{BASE}/wechatweb/wxUser/list" USERID = "a" * 32 COMPANY = "8m0atd" # The company-root department id used as the required ``departmentId`` filter. # Mirrors the shape of the real root dept id (32 hex chars) without hardcoding # the live value in the scraper itself. ROOT_DEPARTMENT_ID = "0E9C4B803E134C7687CBA824886319DA" def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY) def _parse_form(request: httpx.Request) -> dict[str, str]: """Decode a form-urlencoded request body into a flat ``{key: value}`` dict. ``parse_qs`` returns lists per key; the wxUser body has no repeated keys, so we collapse each to its single value for ergonomic assertions. ``keep_blank_values=True`` so the always-empty ``userId`` / ``salesname`` filters (sent as ``userId=``) survive decoding and can be asserted on. """ body = request.content.decode() parsed = parse_qs(body, keep_blank_values=True) return {k: v[0] for k, v in parsed.items()} # --------------------------------------------------------------------------- # End-to-end: pagination + field mapping + raw_json # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_accounts_yields_records_and_paginates(): """Two full pages + a terminating empty page -> 4 records, 3 calls.""" page1 = make_wxuser_list_response( WXUSER_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_wxuser_list_response( WXUSER_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE ) page3 = make_wxuser_list_response( [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE ) route = respx.post(WXUSER_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), httpx.Response(200, json=page3), ] ) client = _primed_client() try: results = [ r async for r in scrape_wechat_accounts( client, department_id=ROOT_DEPARTMENT_ID, page_size=DEFAULT_PAGE_SIZE, ) ] finally: await client.aclose() # Pagination: exactly 3 calls (page1, page2, empty-page stop). assert route.call_count == 3 # The required form filters ride on every request (the live endpoint # answers code -90000 without them). for call in route.calls: body = _parse_form(call.request) assert body["departmentId"] == ROOT_DEPARTMENT_ID assert body["isOnJob"] == "1" assert body["startdate"] assert body["enddate"] assert call.request.headers["content-type"] == FORM_CONTENT_TYPE # Exactly the 4 fixture records (not duplicated, not dropped). assert len(results) == 4 assert {r["wx_id"] for r in results} == { rec["wechatId"] for rec in WXUSER_RECORDS } # Field mapping for each record (compare against the source record). for result, record in zip(results, WXUSER_RECORDS, strict=True): # Identity. assert result["wx_id"] == record["wechatId"] # Naming. assert result["nick_name"] == record["nickName"] assert result["alias"] == record["alias"] # Phones. assert result["phone"] == record["phone"] # Embedded staff fields (the owning employee). assert result["user_id"] == record["userId"] assert result["user_name"] == record["userName"] # Counts. assert result["contacts_num"] == record["contactsNum"] assert result["group_num"] == record["groupNum"] assert result["send_num"] == record["sendNum"] assert result["receive_num"] == record["receiveNum"] # Status / sizing / timestamps. assert result["is_on_job"] == record["isOnJob"] assert result["db_size"] == record["dbSize"] assert result["online_status"] == record["onlineStatus"] assert result["last_update"] == record["lastUpdate"] # Version carried verbatim; raw_json preserves the original record. assert result["version"] == record["version"] assert result["raw_json"] == record # --------------------------------------------------------------------------- # Pagination body: pageNum/pageSize advanced correctly # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_accounts_sends_pagination_body(): """Outgoing form body carries pageNum/pageSize + the required filters; the page number advances per request.""" page1 = make_wxuser_list_response( WXUSER_RECORDS[:1], total=TOTAL, page_num=1, page_size=1 ) page2 = make_wxuser_list_response( [], total=TOTAL, page_num=2, page_size=1 ) route = respx.post(WXUSER_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), ] ) client = _primed_client() try: results = [ r async for r in scrape_wechat_accounts( client, department_id=ROOT_DEPARTMENT_ID, page_size=1, start="2026-07-01", end="2026-07-08", ) ] finally: await client.aclose() # One record on page 1, empty page 2 stops the loop. assert len(results) == 1 assert route.call_count == 2 # Every request is form-urlencoded (NOT JSON) and carries the required # filters the live endpoint demands (isOnJob / departmentId / startdate / # enddate), plus the always-empty free-text filters from the real client. for call in route.calls: assert call.request.headers["content-type"] == FORM_CONTENT_TYPE body = _parse_form(call.request) assert body["isOnJob"] == "1" assert body["departmentId"] == ROOT_DEPARTMENT_ID assert body["startdate"] == "2026-07-01" assert body["enddate"] == "2026-07-08" assert body["userId"] == "" assert body["salesname"] == "" assert body["weChatNickName"] == "" # pageNum/pageSize advance correctly across the form body. body1 = _parse_form(route.calls[0].request) assert body1["pageNum"] == "1" assert body1["pageSize"] == "1" body2 = _parse_form(route.calls[1].request) assert body2["pageNum"] == "2" assert body2["pageSize"] == "1" # --------------------------------------------------------------------------- # Empty first page -> no yields, single call # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_accounts_stops_on_empty_first_page(): """If the very first page is empty, scraper stops without over-fetching.""" empty = make_wxuser_list_response([], total=0, page_num=1) route = respx.post(WXUSER_LIST_URL).mock( return_value=httpx.Response(200, json=empty) ) client = _primed_client() try: results = [ r async for r in scrape_wechat_accounts( client, department_id=ROOT_DEPARTMENT_ID ) ] finally: await client.aclose() assert results == [] assert route.call_count == 1