feat(api): 长语音 voice_clip 后端(model+migration+scraper+sync+endpoint)

- VoiceClip model (BigInteger PK voice_clip_id, structured cols + raw_json) - Alembic migration a1b2c3d4e5f6 (chains from 8c4f1d2a9e6b) - scrape_voice_clips(client, user_id, *, channel, page_size) per-employee scraper over POST /wechatweb/call/list {pageNum, pageSize, userId}; channel stamped locally (wechat/whatsapp), startTime/endTime ms-epoch string -> int - upsert_voice_clip repo + start_time/end_time/mp3_duration int coercion - sync_voice_clips orchestrator: iterates distinct user_id across WechatAccount + WhatsappAccount, stamps channel per account table - scheduler ENTITY_FUNCS['voice_clips'] + added to daily 'all' job - GET /api/voice-clips (user_id exact beats user_real_name fuzzy; channel; company_code; start/end window on end_time; ordered end_time desc) - VoiceClipOut schema + router registered in main.py - tests: scraper (5) + API (11) = 16 new; full suite 370 green Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-10 18:12:28 +08:00
parent a0db4535f1
commit d9060b6920
12 changed files with 1271 additions and 1 deletions
@@ -0,0 +1,297 @@
+"""Tests for the voice-clip (长语音) scraper.
+
+Drives the real page-number pagination + normalization logic through
+``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
+login flow is skipped) against a mocked transport.
+
+The fixture ships 4 records across 2 pages of size 2, plus a terminating
+empty page (the page-number paginator's primary stop signal):
+
+* page 1 -> 2 records, ``total=4``
+* page 2 -> 2 records, ``total=4``
+* page 3 -> empty ``data`` array (no more rows)
+
+The scraper must yield exactly the 4 records normalized to the ``VoiceClip``
+column shape, make exactly 3 calls, and stop without a 4th. Field mapping
+asserted end-to-end incl.:
+
+* ``id``           -> ``voice_clip_id`` (PK, int)
+* ``userId``       -> ``user_id``
+* ``userRealName`` -> ``user_real_name``
+* ``startTime``/``endTime`` -> ``start_time``/``end_time`` (ms-epoch STRING
+  coerced to int)
+* ``ossFileName``  -> ``oss_file_name`` (may be null)
+* ``mp3Duration``  -> ``mp3_duration`` (int seconds)
+* ``channel``      -> stamped locally from the ``channel`` arg (``wechat`` /
+  ``whatsapp``), NOT from the API response
+"""
+
+from __future__ import annotations
+
+import json as _json
+
+import httpx
+import respx
+
+from app.scraper.voice_clip import scrape_voice_clips
+from app.session.yunke_client import YunkeClient
+from tests.cassettes.voiceClip_fixture import (
+    DEFAULT_PAGE_SIZE,
+    TOTAL,
+    VOICE_CLIP_RECORDS,
+    make_voice_clip_list_response,
+)
+
+BASE = "https://crm2.yunkecn.com"
+VOICE_CLIP_LIST_URL = f"{BASE}/wechatweb/call/list"
+
+USERID = "a" * 32
+COMPANY = "8m0atd"
+
+
+def _primed_client() -> YunkeClient:
+    """A YunkeClient already in a logged-in state (login flow skipped)."""
+    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
+
+
+# ---------------------------------------------------------------------------
+# End-to-end: pagination + field mapping + raw_json
+# ---------------------------------------------------------------------------
+
+
+@respx.mock
+async def test_scrape_voice_clips_yields_records_and_paginates():
+    """Two full pages + a terminating empty page -> 4 records, 3 calls."""
+    page1 = make_voice_clip_list_response(
+        VOICE_CLIP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
+    )
+    page2 = make_voice_clip_list_response(
+        VOICE_CLIP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
+    )
+    page3 = make_voice_clip_list_response(
+        [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
+    )
+
+    route = respx.post(VOICE_CLIP_LIST_URL).mock(
+        side_effect=[
+            httpx.Response(200, json=page1),
+            httpx.Response(200, json=page2),
+            httpx.Response(200, json=page3),
+        ]
+    )
+
+    client = _primed_client()
+    try:
+        results = [
+            r
+            async for r in scrape_voice_clips(
+                client,
+                "832EC001_wechat_staff",
+                channel="wechat",
+                page_size=DEFAULT_PAGE_SIZE,
+            )
+        ]
+    finally:
+        await client.aclose()
+
+    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
+    assert route.call_count == 3
+
+    # Exactly the 4 fixture records (not duplicated, not dropped).
+    assert len(results) == 4
+    assert {r["voice_clip_id"] for r in results} == {
+        rec["id"] for rec in VOICE_CLIP_RECORDS
+    }
+
+    # Field mapping for each record (compare against the source record).
+    for result, record in zip(results, VOICE_CLIP_RECORDS, strict=True):
+        # Identity + locally-stamped channel.
+        assert result["voice_clip_id"] == record["id"]
+        assert result["channel"] == "wechat"
+
+        # Staff side.
+        assert result["user_id"] == record["userId"]
+        assert result["user_real_name"] == record["userRealName"]
+        assert result["company_code"] == record["companyCode"]
+
+        # ms-epoch strings coerced to int.
+        assert result["start_time"] == int(record["startTime"])
+        assert result["end_time"] == int(record["endTime"])
+
+        # Recording URL (may be None).
+        assert result["oss_file_name"] == record["ossFileName"]
+
+        # Duration fields.
+        assert result["mp3_duration"] == record["mp3Duration"]
+        assert result["duration_format"] == record["durationFormat"]
+
+        # raw_json preserves the original record verbatim.
+        assert result["raw_json"] == record
+
+
+# ---------------------------------------------------------------------------
+# Channel is stamped locally (NOT from the API response)
+# ---------------------------------------------------------------------------
+
+
+@respx.mock
+async def test_scrape_voice_clips_stamps_whatsapp_channel():
+    """``channel="whatsapp"`` is stamped on every record regardless of payload."""
+    page1 = make_voice_clip_list_response(
+        VOICE_CLIP_RECORDS[:1], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
+    )
+    page2 = make_voice_clip_list_response(
+        [], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
+    )
+    respx.post(VOICE_CLIP_LIST_URL).mock(
+        side_effect=[
+            httpx.Response(200, json=page1),
+            httpx.Response(200, json=page2),
+        ]
+    )
+
+    client = _primed_client()
+    try:
+        results = [
+            r
+            async for r in scrape_voice_clips(
+                client, "F33DA003_wa_staff", channel="whatsapp"
+            )
+        ]
+    finally:
+        await client.aclose()
+
+    assert len(results) == 1
+    assert results[0]["channel"] == "whatsapp"
+
+
+# ---------------------------------------------------------------------------
+# Null ossFileName -> oss_file_name None
+# ---------------------------------------------------------------------------
+
+
+@respx.mock
+async def test_scrape_voice_clips_null_oss_file_name_yields_none():
+    """A record with ``ossFileName=null`` -> ``oss_file_name`` None."""
+    rec = {
+        "id": 66010,
+        "userId": "staff_x",
+        "userRealName": "X",
+        "wxId": None,
+        "talker": None,
+        "status": None,
+        "isSend": None,
+        "companyCode": "8m0atd",
+        "startTime": "1786072733000",
+        "endTime": "1786072733000",
+        "ossFileName": None,
+        "mp3Duration": 0,
+        "durationFormat": "00:00:00",
+    }
+    page1 = make_voice_clip_list_response(
+        [rec], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
+    )
+    page2 = make_voice_clip_list_response(
+        [], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
+    )
+    respx.post(VOICE_CLIP_LIST_URL).mock(
+        side_effect=[
+            httpx.Response(200, json=page1),
+            httpx.Response(200, json=page2),
+        ]
+    )
+
+    client = _primed_client()
+    try:
+        results = [
+            r
+            async for r in scrape_voice_clips(
+                client, "staff_x", channel="wechat", page_size=DEFAULT_PAGE_SIZE
+            )
+        ]
+    finally:
+        await client.aclose()
+
+    assert len(results) == 1
+    assert results[0]["oss_file_name"] is None
+    assert results[0]["voice_clip_id"] == 66010
+
+
+# ---------------------------------------------------------------------------
+# Pagination body: pageNum/pageSize/userId
+# ---------------------------------------------------------------------------
+
+
+@respx.mock
+async def test_scrape_voice_clips_sends_pagination_body():
+    """Body carries ``userId`` + advancing ``pageNum``/``pageSize`` only.
+
+    The ``call/list`` endpoint takes ``{pageNum, pageSize, userId}``. The
+    optional ``startTime``/``endTime`` filters are omitted for full-history
+    sync (the scraper pulls the employee's whole voice-clip set in one pass).
+    """
+    page1 = make_voice_clip_list_response(
+        VOICE_CLIP_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
+    )
+    page2 = make_voice_clip_list_response(
+        [], total=TOTAL, page_num=2, page_size=1
+    )
+    route = respx.post(VOICE_CLIP_LIST_URL).mock(
+        side_effect=[
+            httpx.Response(200, json=page1),
+            httpx.Response(200, json=page2),
+        ]
+    )
+
+    client = _primed_client()
+    try:
+        results = [
+            r
+            async for r in scrape_voice_clips(
+                client, "EMP_USER_001", channel="wechat", page_size=1
+            )
+        ]
+    finally:
+        await client.aclose()
+
+    # One record on page 1, empty page 2 stops the loop.
+    assert len(results) == 1
+    assert route.call_count == 2
+
+    # Page 1 body: userId + standard pagination pair, no time window.
+    body1 = _json.loads(route.calls[0].request.content.decode())
+    assert body1["userId"] == "EMP_USER_001"
+    assert body1["pageNum"] == 1
+    assert body1["pageSize"] == 1
+    # No startTime/endTime (full-history scrape).
+    assert "startTime" not in body1
+    assert "endTime" not in body1
+
+    # Page 2 advances pageNum and keeps userId verbatim.
+    body2 = _json.loads(route.calls[1].request.content.decode())
+    assert body2["pageNum"] == 2
+    assert body2["pageSize"] == 1
+    assert body2["userId"] == "EMP_USER_001"
+
+
+# ---------------------------------------------------------------------------
+# Empty first page -> no yields, single call
+# ---------------------------------------------------------------------------
+
+
+@respx.mock
+async def test_scrape_voice_clips_stops_on_empty_first_page():
+    """If the very first page is empty, scraper stops without over-fetching."""
+    empty = make_voice_clip_list_response([], total=0, page_num=1)
+    route = respx.post(VOICE_CLIP_LIST_URL).mock(
+        return_value=httpx.Response(200, json=empty)
+    )
+
+    client = _primed_client()
+    try:
+        results = [r async for r in scrape_voice_clips(client, "any_user")]
+    finally:
+        await client.aclose()
+
+    assert results == []
+    assert route.call_count == 1