浅念
d9060b6920
- VoiceClip model (BigInteger PK voice_clip_id, structured cols + raw_json)
- Alembic migration a1b2c3d4e5f6 (chains from 8c4f1d2a9e6b)
- scrape_voice_clips(client, user_id, *, channel, page_size) per-employee scraper
over POST /wechatweb/call/list {pageNum, pageSize, userId}; channel stamped
locally (wechat/whatsapp), startTime/endTime ms-epoch string -> int
- upsert_voice_clip repo + start_time/end_time/mp3_duration int coercion
- sync_voice_clips orchestrator: iterates distinct user_id across
WechatAccount + WhatsappAccount, stamps channel per account table
- scheduler ENTITY_FUNCS['voice_clips'] + added to daily 'all' job
- GET /api/voice-clips (user_id exact beats user_real_name fuzzy; channel;
company_code; start/end window on end_time; ordered end_time desc)
- VoiceClipOut schema + router registered in main.py
- tests: scraper (5) + API (11) = 16 new; full suite 370 green
Co-Authored-By: Claude <noreply@anthropic.com>
298 lines
9.5 KiB
Python
298 lines
9.5 KiB
Python
"""Tests for the voice-clip (长语音) scraper.
|
|
|
|
Drives the real page-number pagination + normalization logic through
|
|
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
|
|
login flow is skipped) against a mocked transport.
|
|
|
|
The fixture ships 4 records across 2 pages of size 2, plus a terminating
|
|
empty page (the page-number paginator's primary stop signal):
|
|
|
|
* page 1 -> 2 records, ``total=4``
|
|
* page 2 -> 2 records, ``total=4``
|
|
* page 3 -> empty ``data`` array (no more rows)
|
|
|
|
The scraper must yield exactly the 4 records normalized to the ``VoiceClip``
|
|
column shape, make exactly 3 calls, and stop without a 4th. Field mapping
|
|
asserted end-to-end incl.:
|
|
|
|
* ``id`` -> ``voice_clip_id`` (PK, int)
|
|
* ``userId`` -> ``user_id``
|
|
* ``userRealName`` -> ``user_real_name``
|
|
* ``startTime``/``endTime`` -> ``start_time``/``end_time`` (ms-epoch STRING
|
|
coerced to int)
|
|
* ``ossFileName`` -> ``oss_file_name`` (may be null)
|
|
* ``mp3Duration`` -> ``mp3_duration`` (int seconds)
|
|
* ``channel`` -> stamped locally from the ``channel`` arg (``wechat`` /
|
|
``whatsapp``), NOT from the API response
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json as _json
|
|
|
|
import httpx
|
|
import respx
|
|
|
|
from app.scraper.voice_clip import scrape_voice_clips
|
|
from app.session.yunke_client import YunkeClient
|
|
from tests.cassettes.voiceClip_fixture import (
|
|
DEFAULT_PAGE_SIZE,
|
|
TOTAL,
|
|
VOICE_CLIP_RECORDS,
|
|
make_voice_clip_list_response,
|
|
)
|
|
|
|
BASE = "https://crm2.yunkecn.com"
|
|
VOICE_CLIP_LIST_URL = f"{BASE}/wechatweb/call/list"
|
|
|
|
USERID = "a" * 32
|
|
COMPANY = "8m0atd"
|
|
|
|
|
|
def _primed_client() -> YunkeClient:
|
|
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
|
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# End-to-end: pagination + field mapping + raw_json
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_voice_clips_yields_records_and_paginates():
|
|
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
|
|
page1 = make_voice_clip_list_response(
|
|
VOICE_CLIP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_voice_clip_list_response(
|
|
VOICE_CLIP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page3 = make_voice_clip_list_response(
|
|
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
|
|
route = respx.post(VOICE_CLIP_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
httpx.Response(200, json=page3),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_voice_clips(
|
|
client,
|
|
"832EC001_wechat_staff",
|
|
channel="wechat",
|
|
page_size=DEFAULT_PAGE_SIZE,
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
|
|
assert route.call_count == 3
|
|
|
|
# Exactly the 4 fixture records (not duplicated, not dropped).
|
|
assert len(results) == 4
|
|
assert {r["voice_clip_id"] for r in results} == {
|
|
rec["id"] for rec in VOICE_CLIP_RECORDS
|
|
}
|
|
|
|
# Field mapping for each record (compare against the source record).
|
|
for result, record in zip(results, VOICE_CLIP_RECORDS, strict=True):
|
|
# Identity + locally-stamped channel.
|
|
assert result["voice_clip_id"] == record["id"]
|
|
assert result["channel"] == "wechat"
|
|
|
|
# Staff side.
|
|
assert result["user_id"] == record["userId"]
|
|
assert result["user_real_name"] == record["userRealName"]
|
|
assert result["company_code"] == record["companyCode"]
|
|
|
|
# ms-epoch strings coerced to int.
|
|
assert result["start_time"] == int(record["startTime"])
|
|
assert result["end_time"] == int(record["endTime"])
|
|
|
|
# Recording URL (may be None).
|
|
assert result["oss_file_name"] == record["ossFileName"]
|
|
|
|
# Duration fields.
|
|
assert result["mp3_duration"] == record["mp3Duration"]
|
|
assert result["duration_format"] == record["durationFormat"]
|
|
|
|
# raw_json preserves the original record verbatim.
|
|
assert result["raw_json"] == record
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Channel is stamped locally (NOT from the API response)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_voice_clips_stamps_whatsapp_channel():
|
|
"""``channel="whatsapp"`` is stamped on every record regardless of payload."""
|
|
page1 = make_voice_clip_list_response(
|
|
VOICE_CLIP_RECORDS[:1], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_voice_clip_list_response(
|
|
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
respx.post(VOICE_CLIP_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_voice_clips(
|
|
client, "F33DA003_wa_staff", channel="whatsapp"
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert len(results) == 1
|
|
assert results[0]["channel"] == "whatsapp"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Null ossFileName -> oss_file_name None
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_voice_clips_null_oss_file_name_yields_none():
|
|
"""A record with ``ossFileName=null`` -> ``oss_file_name`` None."""
|
|
rec = {
|
|
"id": 66010,
|
|
"userId": "staff_x",
|
|
"userRealName": "X",
|
|
"wxId": None,
|
|
"talker": None,
|
|
"status": None,
|
|
"isSend": None,
|
|
"companyCode": "8m0atd",
|
|
"startTime": "1786072733000",
|
|
"endTime": "1786072733000",
|
|
"ossFileName": None,
|
|
"mp3Duration": 0,
|
|
"durationFormat": "00:00:00",
|
|
}
|
|
page1 = make_voice_clip_list_response(
|
|
[rec], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_voice_clip_list_response(
|
|
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
respx.post(VOICE_CLIP_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_voice_clips(
|
|
client, "staff_x", channel="wechat", page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert len(results) == 1
|
|
assert results[0]["oss_file_name"] is None
|
|
assert results[0]["voice_clip_id"] == 66010
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pagination body: pageNum/pageSize/userId
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_voice_clips_sends_pagination_body():
|
|
"""Body carries ``userId`` + advancing ``pageNum``/``pageSize`` only.
|
|
|
|
The ``call/list`` endpoint takes ``{pageNum, pageSize, userId}``. The
|
|
optional ``startTime``/``endTime`` filters are omitted for full-history
|
|
sync (the scraper pulls the employee's whole voice-clip set in one pass).
|
|
"""
|
|
page1 = make_voice_clip_list_response(
|
|
VOICE_CLIP_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
|
|
)
|
|
page2 = make_voice_clip_list_response(
|
|
[], total=TOTAL, page_num=2, page_size=1
|
|
)
|
|
route = respx.post(VOICE_CLIP_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_voice_clips(
|
|
client, "EMP_USER_001", channel="wechat", page_size=1
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# One record on page 1, empty page 2 stops the loop.
|
|
assert len(results) == 1
|
|
assert route.call_count == 2
|
|
|
|
# Page 1 body: userId + standard pagination pair, no time window.
|
|
body1 = _json.loads(route.calls[0].request.content.decode())
|
|
assert body1["userId"] == "EMP_USER_001"
|
|
assert body1["pageNum"] == 1
|
|
assert body1["pageSize"] == 1
|
|
# No startTime/endTime (full-history scrape).
|
|
assert "startTime" not in body1
|
|
assert "endTime" not in body1
|
|
|
|
# Page 2 advances pageNum and keeps userId verbatim.
|
|
body2 = _json.loads(route.calls[1].request.content.decode())
|
|
assert body2["pageNum"] == 2
|
|
assert body2["pageSize"] == 1
|
|
assert body2["userId"] == "EMP_USER_001"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Empty first page -> no yields, single call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_voice_clips_stops_on_empty_first_page():
|
|
"""If the very first page is empty, scraper stops without over-fetching."""
|
|
empty = make_voice_clip_list_response([], total=0, page_num=1)
|
|
route = respx.post(VOICE_CLIP_LIST_URL).mock(
|
|
return_value=httpx.Response(200, json=empty)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_voice_clips(client, "any_user")]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert results == []
|
|
assert route.call_count == 1
|