"""Tests for the voice-clip (长语音) scraper. Drives the real page-number pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. The fixture ships 4 records across 2 pages of size 2, plus a terminating empty page (the page-number paginator's primary stop signal): * page 1 -> 2 records, ``total=4`` * page 2 -> 2 records, ``total=4`` * page 3 -> empty ``data`` array (no more rows) The scraper must yield exactly the 4 records normalized to the ``VoiceClip`` column shape, make exactly 3 calls, and stop without a 4th. Field mapping asserted end-to-end incl.: * ``id`` -> ``voice_clip_id`` (PK, int) * ``userId`` -> ``user_id`` * ``userRealName`` -> ``user_real_name`` * ``startTime``/``endTime`` -> ``start_time``/``end_time`` (ms-epoch STRING coerced to int) * ``ossFileName`` -> ``oss_file_name`` (may be null) * ``mp3Duration`` -> ``mp3_duration`` (int seconds) * ``channel`` -> stamped locally from the ``channel`` arg (``wechat`` / ``whatsapp``), NOT from the API response """ from __future__ import annotations import json as _json import httpx import respx from app.scraper.voice_clip import scrape_voice_clips from app.session.yunke_client import YunkeClient from tests.cassettes.voiceClip_fixture import ( DEFAULT_PAGE_SIZE, TOTAL, VOICE_CLIP_RECORDS, make_voice_clip_list_response, ) BASE = "https://crm2.yunkecn.com" VOICE_CLIP_LIST_URL = f"{BASE}/wechatweb/call/list" USERID = "a" * 32 COMPANY = "8m0atd" def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY) # --------------------------------------------------------------------------- # End-to-end: pagination + field mapping + raw_json # --------------------------------------------------------------------------- @respx.mock async def test_scrape_voice_clips_yields_records_and_paginates(): """Two full pages + a terminating empty page -> 4 records, 3 calls.""" page1 = make_voice_clip_list_response( VOICE_CLIP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_voice_clip_list_response( VOICE_CLIP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE ) page3 = make_voice_clip_list_response( [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE ) route = respx.post(VOICE_CLIP_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), httpx.Response(200, json=page3), ] ) client = _primed_client() try: results = [ r async for r in scrape_voice_clips( client, "832EC001_wechat_staff", channel="wechat", page_size=DEFAULT_PAGE_SIZE, ) ] finally: await client.aclose() # Pagination: exactly 3 calls (page1, page2, empty-page stop). assert route.call_count == 3 # Exactly the 4 fixture records (not duplicated, not dropped). assert len(results) == 4 assert {r["voice_clip_id"] for r in results} == { rec["id"] for rec in VOICE_CLIP_RECORDS } # Field mapping for each record (compare against the source record). for result, record in zip(results, VOICE_CLIP_RECORDS, strict=True): # Identity + locally-stamped channel. assert result["voice_clip_id"] == record["id"] assert result["channel"] == "wechat" # Staff side. assert result["user_id"] == record["userId"] assert result["user_real_name"] == record["userRealName"] assert result["company_code"] == record["companyCode"] # ms-epoch strings coerced to int. assert result["start_time"] == int(record["startTime"]) assert result["end_time"] == int(record["endTime"]) # Recording URL (may be None). assert result["oss_file_name"] == record["ossFileName"] # Duration fields. assert result["mp3_duration"] == record["mp3Duration"] assert result["duration_format"] == record["durationFormat"] # raw_json preserves the original record verbatim. assert result["raw_json"] == record # --------------------------------------------------------------------------- # Channel is stamped locally (NOT from the API response) # --------------------------------------------------------------------------- @respx.mock async def test_scrape_voice_clips_stamps_whatsapp_channel(): """``channel="whatsapp"`` is stamped on every record regardless of payload.""" page1 = make_voice_clip_list_response( VOICE_CLIP_RECORDS[:1], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_voice_clip_list_response( [], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE ) respx.post(VOICE_CLIP_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), ] ) client = _primed_client() try: results = [ r async for r in scrape_voice_clips( client, "F33DA003_wa_staff", channel="whatsapp" ) ] finally: await client.aclose() assert len(results) == 1 assert results[0]["channel"] == "whatsapp" # --------------------------------------------------------------------------- # Null ossFileName -> oss_file_name None # --------------------------------------------------------------------------- @respx.mock async def test_scrape_voice_clips_null_oss_file_name_yields_none(): """A record with ``ossFileName=null`` -> ``oss_file_name`` None.""" rec = { "id": 66010, "userId": "staff_x", "userRealName": "X", "wxId": None, "talker": None, "status": None, "isSend": None, "companyCode": "8m0atd", "startTime": "1786072733000", "endTime": "1786072733000", "ossFileName": None, "mp3Duration": 0, "durationFormat": "00:00:00", } page1 = make_voice_clip_list_response( [rec], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE ) page2 = make_voice_clip_list_response( [], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE ) respx.post(VOICE_CLIP_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), ] ) client = _primed_client() try: results = [ r async for r in scrape_voice_clips( client, "staff_x", channel="wechat", page_size=DEFAULT_PAGE_SIZE ) ] finally: await client.aclose() assert len(results) == 1 assert results[0]["oss_file_name"] is None assert results[0]["voice_clip_id"] == 66010 # --------------------------------------------------------------------------- # Pagination body: pageNum/pageSize/userId # --------------------------------------------------------------------------- @respx.mock async def test_scrape_voice_clips_sends_pagination_body(): """Body carries ``userId`` + advancing ``pageNum``/``pageSize`` only. The ``call/list`` endpoint takes ``{pageNum, pageSize, userId}``. The optional ``startTime``/``endTime`` filters are omitted for full-history sync (the scraper pulls the employee's whole voice-clip set in one pass). """ page1 = make_voice_clip_list_response( VOICE_CLIP_RECORDS[:1], total=TOTAL, page_num=1, page_size=1 ) page2 = make_voice_clip_list_response( [], total=TOTAL, page_num=2, page_size=1 ) route = respx.post(VOICE_CLIP_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), ] ) client = _primed_client() try: results = [ r async for r in scrape_voice_clips( client, "EMP_USER_001", channel="wechat", page_size=1 ) ] finally: await client.aclose() # One record on page 1, empty page 2 stops the loop. assert len(results) == 1 assert route.call_count == 2 # Page 1 body: userId + standard pagination pair, no time window. body1 = _json.loads(route.calls[0].request.content.decode()) assert body1["userId"] == "EMP_USER_001" assert body1["pageNum"] == 1 assert body1["pageSize"] == 1 # No startTime/endTime (full-history scrape). assert "startTime" not in body1 assert "endTime" not in body1 # Page 2 advances pageNum and keeps userId verbatim. body2 = _json.loads(route.calls[1].request.content.decode()) assert body2["pageNum"] == 2 assert body2["pageSize"] == 1 assert body2["userId"] == "EMP_USER_001" # --------------------------------------------------------------------------- # Empty first page -> no yields, single call # --------------------------------------------------------------------------- @respx.mock async def test_scrape_voice_clips_stops_on_empty_first_page(): """If the very first page is empty, scraper stops without over-fetching.""" empty = make_voice_clip_list_response([], total=0, page_num=1) route = respx.post(VOICE_CLIP_LIST_URL).mock( return_value=httpx.Response(200, json=empty) ) client = _primed_client() try: results = [r async for r in scrape_voice_clips(client, "any_user")] finally: await client.aclose() assert results == [] assert route.call_count == 1