"""Tests for the WhatsApp voice-call scraper (Plan 4a Task 1). Drives the real page-number pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. The WA voice endpoint reuses the **unified** ``queryWeChatVoiceList`` interface (the same one the WeChat voice scraper hits) with the WA-specific ``callSource="4"`` flag — same envelope (``data.data`` / ``data.total``), same record schema, just a different ``channel`` label (``"wa_voice_call"``). The fixture ships 3 records across 2 pages of size 2, plus a terminating empty page (the page-number paginator's primary stop signal): * page 1 -> 2 records, ``total=3`` * page 2 -> 1 record, ``total=3`` * page 3 -> empty ``data`` array (no more rows) The scraper must: * make exactly 3 calls and stop without a 4th, * send ``callSource=4`` + advancing ``pageNum``/``pageSize`` in the body, * yield exactly the 3 records normalized to the ``PhoneCall`` column shape with ``channel="wa_voice_call"``, * preserve the WA-specific fields (``weChatId`` / ``callFileId`` / ``nextId`` / ``isRoom`` / ``from``) via ``raw_json`` (no PhoneCall columns exist for them). Field mapping asserted end-to-end (mirrors the WeChat voice test, ``test_scrape_wechat_voice_calls``): * ``id`` -> ``call_id`` (PK) * ``talker`` -> ``call_number`` (the customer WA id) * ``talkerNickName`` -> ``plan_customer_name`` * ``ossFileName``-> ``record_file`` (may be null) * ``startTime`` -> ``start_call_time`` (``YYYY-MM-DD HH:MM:SS`` -> ms epoch) * ``duration`` -> ``call_duration`` (string seconds coerced to int) * ``isSend`` -> ``incoming_call`` with INVERTED polarity (0=incoming -> ``incoming_call=1``; 1=outgoing -> ``incoming_call=0``). """ from __future__ import annotations import datetime as dt import json as _json import httpx import respx from app.scraper.whatsapp import scrape_wa_voice_calls from app.session.yunke_client import YunkeClient from tests.cassettes.wa_voice_fixture import ( DEFAULT_PAGE_SIZE, TOTAL, WA_VOICE_RECORDS, make_wa_voice_list_response, ) BASE = "https://crm2.yunkecn.com" VOICE_URL = f"{BASE}/wechatweb/callRecord/queryWeChatVoiceList" USERID = "a" * 32 COMPANY = "8m0atd" def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY) def _ms(date_str: str) -> int: """Expected ms-epoch for a ``YYYY-MM-DD HH:MM:SS`` string parsed as UTC. Mirrors the scraper's documented parsing rule so the assertion is an independent recomputation (not a self-reference). """ return int( dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") .replace(tzinfo=dt.UTC) .timestamp() * 1000 ) # --------------------------------------------------------------------------- # End-to-end: pagination + body + field mapping + raw_json # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wa_voice_calls_yields_records_and_paginates(): """Two pages + a terminating empty page -> 3 records, 3 calls. Body must carry the full filter set the live endpoint REQUIRES (``callSource=4`` + ``departmentId`` + ``isRoom`` + ``startTime`` + ``endTime`` — omit any and the endpoint returns 200 with empty ``data.data``, 0 rows) plus the standard ``pageNum``/``pageSize`` pagination pair. Each yielded record is normalized to the ``PhoneCall`` shape with ``channel="wa_voice_call"`` and the WA-specific fields preserved via ``raw_json``. """ page1 = make_wa_voice_list_response( WA_VOICE_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_wa_voice_list_response( WA_VOICE_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE ) page3 = make_wa_voice_list_response( [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE ) route = respx.post(VOICE_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), httpx.Response(200, json=page3), ] ) client = _primed_client() try: results = [ r async for r in scrape_wa_voice_calls( client, department_id="DEPT_TEST", page_size=DEFAULT_PAGE_SIZE ) ] finally: await client.aclose() # Pagination: exactly 3 calls (page1, page2, empty-page stop). assert route.call_count == 3 # Exactly the 3 fixture records (not duplicated, not dropped). assert len(results) == len(WA_VOICE_RECORDS) assert {r["call_id"] for r in results} == {rec["id"] for rec in WA_VOICE_RECORDS} # First request body: full required filter set the live endpoint demands. # callSource=4 (WA) + departmentId + isRoom="0" + wide time window + # empty userId/sort/ids (the live endpoint accepts these blank). body1 = _json.loads(route.calls[0].request.content.decode()) assert body1["callSource"] == "4" assert body1["departmentId"] == "DEPT_TEST" assert body1["isRoom"] == "0" assert body1["startTime"] == "2000-01-01 00:00:00" assert body1["endTime"] == "2099-12-31 23:59:59" assert body1["pageNum"] == 1 assert body1["pageSize"] == DEFAULT_PAGE_SIZE # The blank-but-required keys are present (the live endpoint expects # them in the body even when empty). assert body1["userId"] == "" assert body1["sort"] == "" assert body1["ids"] == [] # Second page advances pageNum and keeps the same filter set verbatim. body2 = _json.loads(route.calls[1].request.content.decode()) assert body2["pageNum"] == 2 assert body2["pageSize"] == DEFAULT_PAGE_SIZE assert body2["callSource"] == "4" assert body2["departmentId"] == "DEPT_TEST" # Field mapping for each record (compare against the source record). for result, record in zip(results, WA_VOICE_RECORDS, strict=True): # Identity + channel. assert result["call_id"] == record["id"] assert result["channel"] == "wa_voice_call" # Staff side. assert result["user_id"] == record["userId"] assert result["user_id_name"] == record["userName"] # Customer side: talker -> call_number, talkerNickName -> plan_customer_name. assert result["call_number"] == record["talker"] assert result["plan_customer_name"] == record["talkerNickName"] # Recording URL (may be None). assert result["record_file"] == record["ossFileName"] # Status + duration + start time coercions. assert result["call_status"] == record["callStatus"] assert result["call_duration"] == int(record["duration"]) assert result["start_call_time"] == _ms(record["startTime"]) # incoming_call polarity: isSend 0=incoming -> 1; 1=outgoing -> 0. assert result["incoming_call"] == 1 - record["isSend"] # Version carried verbatim; raw_json preserves the original record # (so WA-specific fields like weChatId / callFileId / nextId / isRoom # / from survive end-to-end despite having no PhoneCall column). assert result["version"] == record["version"] assert result["raw_json"] == record # --------------------------------------------------------------------------- # Empty first page -> no yields, single call # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wa_voice_calls_stops_on_empty_first_page(): """If the very first page is empty, scraper stops without over-fetching.""" empty = make_wa_voice_list_response([], total=0, page_num=1) route = respx.post(VOICE_URL).mock( return_value=httpx.Response(200, json=empty) ) client = _primed_client() try: results = [r async for r in scrape_wa_voice_calls(client)] finally: await client.aclose() assert results == [] assert route.call_count == 1