xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api): 长语音 voice_clip 后端(model+migration+scraper+sync+endpoint)

Browse Source 
- VoiceClip model (BigInteger PK voice_clip_id, structured cols + raw_json)
- Alembic migration a1b2c3d4e5f6 (chains from 8c4f1d2a9e6b)
- scrape_voice_clips(client, user_id, *, channel, page_size) per-employee scraper
  over POST /wechatweb/call/list {pageNum, pageSize, userId}; channel stamped
  locally (wechat/whatsapp), startTime/endTime ms-epoch string -> int
- upsert_voice_clip repo + start_time/end_time/mp3_duration int coercion
- sync_voice_clips orchestrator: iterates distinct user_id across
  WechatAccount + WhatsappAccount, stamps channel per account table
- scheduler ENTITY_FUNCS['voice_clips'] + added to daily 'all' job
- GET /api/voice-clips (user_id exact beats user_real_name fuzzy; channel;
  company_code; start/end window on end_time; ordered end_time desc)
- VoiceClipOut schema + router registered in main.py
- tests: scraper (5) + API (11) = 16 new; full suite 370 green

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-10 18:12:28 +08:00
parent a0db4535f1
commit d9060b6920
12 changed files with 1271 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/app/api/main.py
+3
View File
@@ -15,6 +15,7 @@ Includes:
(see :mod:`app.api.routers.groups`).
* ``GET /api/calls`` (see :mod:`app.api.routers.calls`).
* ``GET /api/sms`` (see :mod:`app.api.routers.sms`).
* ``GET /api/voice-clips`` (see :mod:`app.api.routers.voice_clips`).
* ``GET /api/sync/status`` (see :mod:`app.api.routers.sync`).
* ``GET /api/search`` (see :mod:`app.api.routers.search`).
* ``GET /api/export/chat-html`` (see :mod:`app.api.routers.export`).
@@ -49,6 +50,7 @@ from app.api.routers import (
sms,
staff_chat_stats,
sync,
voice_clips,
)
app = FastAPI(title="yunke data mirror", dependencies=[Depends(require_auth)])
@@ -72,6 +74,7 @@ app.include_router(sync.router)
app.include_router(search.router)
app.include_router(sessions.router)
app.include_router(export.router)
app.include_router(voice_clips.router)
@app.get("/api/health")
yunke-server/app/api/routers/voice_clips.py
+89
View File
@@ -0,0 +1,89 @@
"""GET /api/voice-clips — 长语音 (voice_clip) list.
Read-only query over the mirrored ``voice_clip`` table. Powers the
``VoiceClipModal`` (微信长语音 / WA wavoicecalllist 共用的只读展示 modal).
Filters (all optional):
* ``user_id`` — exact employee-id match (modal opens from a CallList
row keyed on the employee). Takes PRECEDENCE over ``user_real_name`` when
both are given (the id is more precise).
* ``user_real_name`` — employee-name ILIKE / contains match (the 云客 modal's
员工姓名 fuzzy filter). Ignored when ``user_id`` is present.
* ``channel`` — messaging channel (``wechat`` / ``whatsapp``).
* ``company_code`` — exact 企业串码 match.
* ``start`` / ``end`` — ms-epoch window compared against ``end_time`` (录音结束时间,
inclusive) — mirrors the 云客 modal's 录音结束时间 range filter.
Ordered by ``end_time desc`` (nulls last) — most recent clips first, matching
the 云客 modal's default ordering.
"""
from __future__ import annotations
from typing import Annotated
import sqlalchemy as sa
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_session
from app.api.schemas import Page, VoiceClipOut
from app.db.models import VoiceClip
router = APIRouter(prefix="/api", tags=["voice_clips"])
SessionDep = Annotated[AsyncSession, Depends(get_session)]
PageParam = Annotated[int, Query(ge=1)]
PageSizeParam = Annotated[int, Query(ge=1, le=100)]
@router.get("/voice-clips")
async def list_voice_clips(
session: SessionDep,
page: PageParam = 1,
page_size: PageSizeParam = 20,
user_id: Annotated[str | None, Query()] = None,
user_real_name: Annotated[str | None, Query()] = None,
channel: Annotated[str | None, Query()] = None,
company_code: Annotated[str | None, Query()] = None,
start: Annotated[int | None, Query(ge=0)] = None,
end: Annotated[int | None, Query(ge=0)] = None,
) -> Page[VoiceClipOut]:
"""List voice clips with optional filters, newest by recording end first."""
conditions = []
# user_id takes precedence over user_real_name (id is the more precise
# filter; the 云客 modal switches between them based on whether the user
# edited the staff-name box).
if user_id is not None:
conditions.append(VoiceClip.user_id == user_id)
elif user_real_name is not None:
conditions.append(VoiceClip.user_real_name.ilike(f"%{user_real_name}%"))
if channel is not None:
conditions.append(VoiceClip.channel == channel)
if company_code is not None:
conditions.append(VoiceClip.company_code == company_code)
if start is not None:
conditions.append(VoiceClip.end_time >= start)
if end is not None:
conditions.append(VoiceClip.end_time <= end)
total_stmt = sa.select(sa.func.count()).select_from(VoiceClip)
if conditions:
total_stmt = total_stmt.where(*conditions)
total = (await session.execute(total_stmt)).scalar_one()
list_stmt = sa.select(VoiceClip)
if conditions:
list_stmt = list_stmt.where(*conditions)
list_stmt = (
list_stmt.order_by(
sa.nullslast(VoiceClip.end_time.desc()),
VoiceClip.voice_clip_id.asc(),
)
.limit(page_size)
.offset((page - 1) * page_size)
)
rows = (await session.execute(list_stmt)).scalars().all()
items = [VoiceClipOut.model_validate(row) for row in rows]
return Page(items=items, total=total, page=page, page_size=page_size)
yunke-server/app/api/schemas.py
+31
View File
@@ -249,6 +249,37 @@ class SmsOut(BaseModel):
raw_json: dict[str, Any] | None = None
class VoiceClipOut(BaseModel):
"""长语音 (voice_clip) row — a long audio segment from ``/wechatweb/call/list``.
The structured columns power the VoiceClipModal table (员工姓名 /
企业串码 / 文件地址 / 录音开始时间 / 录音结束时间 / 录音时长); the verbatim
upstream record is exposed as ``raw_json`` so the frontend can render any
extra fields without a second endpoint. Picked up automatically via
``from_attributes``.
"""
model_config = ConfigDict(from_attributes=True)
voice_clip_id: int
user_id: str
channel: str
user_real_name: str | None = None
wx_id: str | None = None
talker: str | None = None
status: str | None = None
is_send: bool | None = None
company_code: str | None = None
start_time: int | None = None
end_time: int | None = None
# OSS recording URL — surfaced so the modal can render an inline
# ``<audio controls>`` player (see spec §6).
oss_file_name: str | None = None
mp3_duration: int | None = None
duration_format: str | None = None
raw_json: dict[str, Any] | None = None
# ---------------------------------------------------------------------------
# Sync monitor
# ---------------------------------------------------------------------------
yunke-server/app/db/models.py
+47
View File
@@ -477,6 +477,52 @@ class PhoneCall(Base):
)
class VoiceClip(Base):
"""长语音片段 — source: ``POST /wechatweb/call/list`` (WeChat + WhatsApp
share the same endpoint, distinguished by the employee's channel).
Distinct from :class:`PhoneCall` (which holds ``callLog/list`` phone calls
and ``queryWeChatVoiceList`` voice-call records): a voice_clip is one of the
long audio segments listed in the 云客 微信长语音 / WA wavoicecalllist modals.
PK is the source ``id`` (a cross-employee-globally-unique bigint).
``channel`` here is the **messaging channel** in {``wechat``, ``whatsapp``}
— stamped LOCALLY by the sync orchestrator based on which staff-account
table owns the scraped ``user_id`` (WechatAccount -> ``wechat``,
WhatsappAccount -> ``whatsapp``). The upstream endpoint is shared and does
NOT distinguish the channel in its response, so the channel must come from
the employee mapping, not the record.
``start_time`` / ``end_time`` arrive as ms-epoch **strings** (e.g.
``"1786072733378"``); the scraper normalizes them to ``int``.
"""
__tablename__ = "voice_clip"
voice_clip_id: Mapped[int] = mapped_column(
BigInteger, primary_key=True, autoincrement=False
)
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
user_real_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
wx_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
talker: Mapped[str | None] = mapped_column(String(128), nullable=True)
status: Mapped[str | None] = mapped_column(String(32), nullable=True)
is_send: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
company_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
start_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
end_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
oss_file_name: Mapped[str | None] = mapped_column(String(1024), nullable=True)
mp3_duration: Mapped[int | None] = mapped_column(Integer, nullable=True)
duration_format: Mapped[str | None] = mapped_column(String(32), nullable=True)
channel: Mapped[str] = mapped_column(String(16), nullable=False)
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
fetched_at: Mapped[dt.datetime] = _fetched_at()
updated_at: Mapped[dt.datetime] = _updated_at()
__table_args__ = (Index("ix_voice_clip_user_time", "user_id", "start_time"),)
class Sms(Base):
"""短信记录 — source: ``POST /pc/customer/smsLogsList``."""
@@ -617,6 +663,7 @@ __all__ = [
"Staff",
"SyncRun",
"SyncState",
"VoiceClip",
"WechatAccount",
"WhatsappAccount",
"YkAccount",
yunke-server/app/scraper/voice_clip.py
+183
View File
@@ -0,0 +1,183 @@
"""Voice-clip (长语音) scraper.
Wraps ``POST /wechatweb/call/list`` — the endpoint behind the 云客 微信长语音 /
WA wavoicecalllist modals. Unlike :func:`app.scraper.wechat.scrape_wechat_voice_calls`
(which is scoped by ``departmentId`` + ``callSource``), this endpoint is scoped
**per employee**: the request body takes a required ``userId`` (employee id) and
returns that employee's long audio segments (the modal opens from a CallList
row keyed on the employee).
Endpoint contract (Playwright-confirmed 2026-08-07, see spec §2):
* Request body ``{pageNum, pageSize, userId}`` (JSON). ``startTime`` / ``endTime``
ms-epoch string filters are OPTIONAL — omitted for full-history sync.
* Response envelope ``{code, message, success, data: {pageNum, pageSize, total,
data: [...]}}`` — the SAME shape as ``queryWeChatVoiceList`` (rows under
``data.data``, declared total on ``data.total``), so the pagination helpers
mirror the voice-call scraper.
* Row fields: ``id`` (bigint PK) / ``userId`` / ``userRealName`` / ``wxId`` /
``talker`` / ``status`` / ``isSend`` / ``companyCode`` / ``startTime`` /
``endTime`` (both ms-epoch STRINGS — normalized to int) / ``ossFileName``
(recording URL) / ``mp3Duration`` (seconds) / ``durationFormat`` (HH:MM:SS).
Channel handling: the endpoint is SHARED by WeChat and WhatsApp employees
(distinct ``userId`` ranges, same path). The response does NOT carry the
channel, so :func:`scrape_voice_clips` takes a ``channel`` argument and stamps
it locally on every yielded record — the caller (sync orchestrator) resolves
it from the owning staff-account table (WechatAccount -> ``wechat``,
WhatsappAccount -> ``whatsapp``).
Reuses :func:`app.scraper.base.paginate_by_page` and emits snake_case dicts
plus a verbatim ``raw_json`` copy, mirroring :mod:`app.scraper.wechat`.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from app.scraper.base import paginate_by_page
from app.scraper.parsers import coerce_int_or_none
from app.session.yunke_client import YunkeClient
__all__ = ["scrape_voice_clips"]
VOICE_CLIP_LIST_PATH = "/wechatweb/call/list"
def _parse_voice_clip_total(resp: Any) -> int | None:
"""Read the declared row count from the ``call/list`` envelope.
Shape: ``data.total`` (an int) — the same location as
``queryWeChatVoiceList``. Returns ``None`` if the path is absent so the
paginator falls back to the empty-page stop signal.
"""
try:
body = resp.json()
except (AttributeError, ValueError):
return None
data = body.get("data") if isinstance(body, dict) else None
if not isinstance(data, dict):
return None
total = data.get("total")
return int(total) if isinstance(total, (int, float)) and not isinstance(
total, bool
) else None
def _parse_voice_clip_rows(resp: Any) -> list[dict[str, Any]]:
"""Extract the voice-clip list from the ``call/list`` envelope.
Shape: ``data.data`` (a list of records). Missing / non-list values
collapse to an empty list so the paginator sees "empty page".
"""
try:
body = resp.json()
except (AttributeError, ValueError):
return []
data = body.get("data") if isinstance(body, dict) else None
if not isinstance(data, dict):
return []
rows = data.get("data")
return rows if isinstance(rows, list) else []
def _build_voice_clip_body(
page_num: int,
page_size: int,
*,
user_id: str,
) -> dict[str, Any]:
"""Build the ``call/list`` request body for a given page.
``userId`` is the REQUIRED employee filter (the endpoint returns nothing
without it). The optional ``startTime`` / ``endTime`` ms-epoch filters are
omitted so the scrape pulls the employee's full history in one pass (the
sync orchestrator wants all segments, not a time window).
"""
return {
"pageNum": page_num,
"pageSize": page_size,
"userId": user_id,
}
def _normalize_voice_clip(
record: dict[str, Any], *, channel: str
) -> dict[str, Any]:
"""Map one upstream ``call/list`` record to the ``VoiceClip`` column shape.
Stamps the locally-resolved ``channel`` (``wechat`` / ``whatsapp``) — the
API response does not carry it. ``startTime`` / ``endTime`` arrive as
ms-epoch STRINGS (e.g. ``"1786072733378"``); they are coerced to ``int``
via :func:`coerce_int_or_none` so the BigInteger columns land clean.
``mp3Duration`` (seconds, already int in the capture) is coerced to int
defensively. ``isSend`` is carried through verbatim — the repo coerces it
to bool. ``None``-safe on every field: a missing field becomes ``None``
rather than raising, so a partial payload does not break the scrape.
Preserves the original record under ``raw_json``. See spec §3 for the
column-by-column mapping.
"""
return {
"voice_clip_id": record.get("id"),
"user_id": record.get("userId"),
"user_real_name": record.get("userRealName"),
"wx_id": record.get("wxId"),
"talker": record.get("talker"),
"status": record.get("status"),
"is_send": record.get("isSend"),
"company_code": record.get("companyCode"),
# ms-epoch strings -> int (the BigInteger column type).
"start_time": coerce_int_or_none(record.get("startTime")),
"end_time": coerce_int_or_none(record.get("endTime")),
"oss_file_name": record.get("ossFileName"),
"mp3_duration": coerce_int_or_none(record.get("mp3Duration")),
"duration_format": record.get("durationFormat"),
# Local label — resolved by the orchestrator from the staff-account
# table that owns this user_id (NOT from the API response).
"channel": channel,
"raw_json": record,
}
async def scrape_voice_clips(
client: YunkeClient,
user_id: str,
*,
channel: str = "wechat",
page_size: int = 50,
) -> AsyncIterator[dict[str, Any]]:
"""Yield normalized voice-clip records for one employee from ``call/list``.
Walks ``POST /wechatweb/call/list`` page by page (body
``{pageNum, pageSize, userId}``) until the upstream reports no more rows
(empty ``data.data`` array — the primary stop signal), mapping each record
to the ``VoiceClip`` column shape plus a verbatim ``raw_json`` copy.
Parameters
----------
client:
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
handled by the client.
user_id:
Employee id (the REQUIRED ``userId`` request field). The sync
orchestrator resolves this from the staff-account tables.
channel:
Local messaging-channel label (``"wechat"`` / ``"whatsapp"``) stamped
on every yielded record. The endpoint is shared and does NOT carry the
channel; the caller resolves it from the owning account table.
page_size:
Page size to request. Defaults to 50 (a sane middle ground between
request count and per-response payload size for this endpoint).
"""
async for record in paginate_by_page(
client,
VOICE_CLIP_LIST_PATH,
page_size=page_size,
parse_total=_parse_voice_clip_total,
parse_rows=_parse_voice_clip_rows,
build_body=_build_voice_clip_body,
user_id=user_id,
):
yield _normalize_voice_clip(record, channel=channel)
yunke-server/app/storage/repos.py
+30
View File
@@ -63,6 +63,7 @@ from app.db.models import (
PhoneCall,
Sms,
Staff,
VoiceClip,
WechatAccount,
WhatsappAccount,
)
@@ -79,6 +80,7 @@ __all__ = [
"upsert_staff",
"upsert_staff_from_account",
"upsert_voice_call",
"upsert_voice_clip",
"upsert_wechat_account",
"upsert_whatsapp_account",
]
@@ -108,6 +110,12 @@ _INT_FIELDS: frozenset[str] = frozenset(
"start_call_time",
"qc_status",
"create_time",
# VoiceClip (voice_clip): ms-epoch + duration fields arrive as strings
# or ints depending on the capture; coerce to int defensively. The
# scraper already emits ints, but a hand-built record may not.
"start_time",
"end_time",
"mp3_duration",
}
)
@@ -565,6 +573,28 @@ async def upsert_sms(session: AsyncSession, rec: dict[str, Any]) -> None:
await _upsert_one(session, Sms, rec)
# ---------------------------------------------------------------------------
# VoiceClip (长语音) upsert
# ---------------------------------------------------------------------------
async def upsert_voice_clip(session: AsyncSession, rec: dict[str, Any]) -> None:
"""Idempotently upsert one :class:`VoiceClip` (PK = ``voice_clip_id``).
Same ``voice_clip_id`` twice -> one row, with non-PK fields refreshed from
the later record. Applies the documented int coercions (``start_time`` /
``end_time`` / ``mp3_duration`` -> int when they arrive as numeric strings;
the voice-clip scraper already emits ints via :func:`coerce_int_or_none`,
but a hand-built record may not). ``is_send`` is a bool field and goes
through :func:`_coerce_bool` like the other bool columns.
``channel`` is stamped locally by the sync orchestrator (``wechat`` /
``whatsapp``) based on which staff-account table owns the scraped
``user_id``; the endpoint does not carry it.
"""
await _upsert_one(session, VoiceClip, rec)
# ---------------------------------------------------------------------------
# Media upsert (M2, Task 2.6)
# ---------------------------------------------------------------------------
yunke-server/app/sync/orchestrator.py
+138 -1
View File
@@ -49,9 +49,10 @@ from typing import Any
import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Department, SyncRun
from app.db.models import Department, SyncRun, WechatAccount, WhatsappAccount
from app.scraper.call import scrape_phone_calls, scrape_sms
from app.scraper.media import download_call_recordings, download_message_media
from app.scraper.voice_clip import scrape_voice_clips
from app.scraper.wechat import (
scrape_departments,
scrape_wechat_accounts,
@@ -76,6 +77,7 @@ from app.storage.repos import (
upsert_phone_call,
upsert_sms,
upsert_voice_call,
upsert_voice_clip,
upsert_wechat_account,
upsert_whatsapp_account,
)
@@ -91,6 +93,7 @@ __all__ = [
"sync_departments",
"sync_phone_calls",
"sync_sms",
"sync_voice_clips",
"sync_wechat_accounts",
"sync_wechat_friends",
"sync_wechat_groups",
@@ -131,6 +134,10 @@ ENTITY_SMS = "sms"
# object store (records a SyncRun with this entity_type; no SyncState row
# because the phone_call table itself is the cursor).
ENTITY_CALL_RECORDINGS = "call_recordings"
# 长语音 (voice_clip) — per-employee scrape of ``/wechatweb/call/list``. The
# sync walks every distinct user_id across both staff-account tables and stamps
# the channel locally (WechatAccount -> wechat, WhatsappAccount -> whatsapp).
ENTITY_VOICE_CLIP = "voice_clip"
CHANNEL_WECHAT = "wechat"
CHANNEL_WHATSAPP = "whatsapp"
@@ -889,6 +896,136 @@ async def sync_sms(
)
# ---------------------------------------------------------------------------
# VoiceClip (长语音) — per-employee scrape across both staff-account tables
# ---------------------------------------------------------------------------
async def _collect_voice_clip_staff(
session: AsyncSession,
) -> list[tuple[str, str]]:
"""Return ``(user_id, channel)`` pairs for voice-clip scraping.
One pair per DISTINCT ``user_id`` across both staff-account tables. The
``call/list`` endpoint is scoped per employee (``userId`` request field)
and is shared by WeChat + WhatsApp staff (distinct ``userId`` ranges);
each employee is scraped exactly once. WeChat accounts take precedence on
collision (deterministic — in practice the two ``userId`` ranges are
disjoint, so collisions are not expected).
The returned ``channel`` is the LOCAL messaging-channel label stamped on
every scraped record: ``wechat`` for ids found in :class:`WechatAccount`,
``whatsapp`` for ids found only in :class:`WhatsappAccount`. The endpoint
response does NOT carry the channel, so it must be resolved here from the
owning account table.
"""
pairs: list[tuple[str, str]] = []
seen: set[str] = set()
wechat_rows = (
await session.execute(
sa.select(WechatAccount.user_id)
.where(WechatAccount.user_id.is_not(None))
.where(WechatAccount.user_id != "")
.distinct()
)
).all()
for (uid,) in wechat_rows:
if uid and uid not in seen:
seen.add(uid)
pairs.append((uid, CHANNEL_WECHAT))
wa_rows = (
await session.execute(
sa.select(WhatsappAccount.user_id)
.where(WhatsappAccount.user_id.is_not(None))
.where(WhatsappAccount.user_id != "")
.distinct()
)
).all()
for (uid,) in wa_rows:
if uid and uid not in seen:
seen.add(uid)
pairs.append((uid, CHANNEL_WHATSAPP))
return pairs
async def sync_voice_clips(
session: AsyncSession, client: YunkeClient
) -> SyncResult:
"""Full re-scrape of all 长语音 (voice_clip) records across all staff.
Walks every distinct ``user_id`` in the local staff-account tables
(:class:`WechatAccount` + :class:`WhatsappAccount`) and, per employee,
scrapes ``POST /wechatweb/call/list`` (body ``{pageNum, pageSize,
userId}``), upserting each record into :class:`VoiceClip`. The channel is
stamped locally (``wechat`` / ``whatsapp``) based on which account table
owns the ``user_id`` — the shared endpoint does not carry it.
Mirrors :func:`sync_wechat_voice_calls`'s failure policy + SyncRun
bookkeeping, but iterates MULTIPLE per-employee scrape streams (rather than
one department-scoped stream). A scraper failure on one employee is caught,
recorded as ``status='failed'`` for the whole entity, and a failed
:class:`SyncResult` is returned (no re-raise) — the caller decides whether
to surface / retry.
When no staff ``user_id`` is present (e.g. accounts not yet synced), the
scrape is SKIPPED with a logged warning and a ``status='skipped'``
:class:`SyncResult` (no ``SyncRun`` recorded), so a fresh DB does not abort
a larger batch.
"""
pairs = await _collect_voice_clip_staff(session)
if not pairs:
logger.warning(
"sync_voice_clips: no staff user_ids found in wechat_account /"
" whatsapp_account (sync accounts first?); skipping call/list scrape"
)
return SyncResult(
entity=ENTITY_VOICE_CLIP,
status="skipped",
fetched=0,
upserted=0,
error="no staff user_ids available",
)
run = await _start_run(session, ENTITY_VOICE_CLIP)
fetched = 0
upserted = 0
try:
for uid, channel in pairs:
async for rec in scrape_voice_clips(client, uid, channel=channel):
await upsert_voice_clip(session, rec)
fetched += 1
upserted += 1
except Exception as exc:
logger.exception("scrape_voice_clips failed")
error = repr(exc)
await _finish_run(
session, run, status="failed",
fetched=fetched, upserted=upserted, error=error,
)
return SyncResult(
entity=ENTITY_VOICE_CLIP,
status="failed",
fetched=fetched,
upserted=upserted,
error=error,
)
await set_watermark(session, ENTITY_VOICE_CLIP)
await _finish_run(
session, run, status="success", fetched=fetched, upserted=upserted
)
return SyncResult(
entity=ENTITY_VOICE_CLIP,
status="success",
fetched=fetched,
upserted=upserted,
)
# ---------------------------------------------------------------------------
# Coordinator
# ---------------------------------------------------------------------------
yunke-server/app/sync/scheduler.py
+13
View File
@@ -59,6 +59,7 @@ from app.sync.orchestrator import (
sync_departments,
sync_phone_calls,
sync_sms,
sync_voice_clips,
sync_wa_voice_calls,
sync_wechat_accounts,
sync_wechat_friends,
@@ -188,6 +189,13 @@ async def _run_sms(
return [await sync_sms(session, client)]
async def _run_voice_clips(
session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> list[SyncResult]:
"""Per-employee scrape of 长语音 (voice_clip) across all staff."""
return [await sync_voice_clips(session, client)]
async def _run_call_recordings(
session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> list[SyncResult]:
@@ -264,6 +272,10 @@ async def _run_all(
results.extend(await _run_wa_messages(session, client, store))
results.append(await sync_phone_calls(session, client))
results.append(await sync_sms(session, client))
# 长语音 (voice_clip): per-employee scrape across both staff-account
# tables. Must run AFTER wechat_accounts + wa_accounts (which seed the
# user_id -> channel mapping the voice-clip sync reads).
results.append(await sync_voice_clips(session, client))
return results
@@ -283,6 +295,7 @@ ENTITY_FUNCS: dict[
"wa_friends": _run_wa_friends,
"phone_calls": _run_phone_calls,
"sms": _run_sms,
"voice_clips": _run_voice_clips,
"call_recordings": _run_call_recordings,
"messages": _run_messages,
"wa_messages": _run_wa_messages,
yunke-server/migrations/versions/a1b2c3d4e5f6_voice_clip_table.py
+49
View File
@@ -0,0 +1,49 @@
"""voice_clip table
Revision ID: a1b2c3d4e5f6
Revises: 8c4f1d2a9e6b
Create Date: 2026-08-10 00:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'a1b2c3d4e5f6'
down_revision: str | Sequence[str] | None = '8c4f1d2a9e6b'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema."""
op.create_table('voice_clip',
sa.Column('voice_clip_id', sa.BigInteger(), autoincrement=False, nullable=False),
sa.Column('user_id', sa.String(length=64), nullable=False),
sa.Column('user_real_name', sa.String(length=255), nullable=True),
sa.Column('wx_id', sa.String(length=128), nullable=True),
sa.Column('talker', sa.String(length=128), nullable=True),
sa.Column('status', sa.String(length=32), nullable=True),
sa.Column('is_send', sa.Boolean(), nullable=True),
sa.Column('company_code', sa.String(length=64), nullable=True),
sa.Column('start_time', sa.BigInteger(), nullable=True),
sa.Column('end_time', sa.BigInteger(), nullable=True),
sa.Column('oss_file_name', sa.String(length=1024), nullable=True),
sa.Column('mp3_duration', sa.Integer(), nullable=True),
sa.Column('duration_format', sa.String(length=32), nullable=True),
sa.Column('channel', sa.String(length=16), nullable=False),
sa.Column('raw_json', sa.JSON(), nullable=True),
sa.Column('fetched_at', sa.DateTime(), server_default=sa.func.now(), nullable=False),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), nullable=False),
sa.PrimaryKeyConstraint('voice_clip_id')
)
op.create_index('ix_voice_clip_user_time', 'voice_clip', ['user_id', 'start_time'], unique=False)
def downgrade() -> None:
"""Downgrade schema."""
op.drop_index('ix_voice_clip_user_time', table_name='voice_clip')
op.drop_table('voice_clip')
yunke-server/tests/cassettes/voiceClip_fixture.py
+152
View File
@@ -0,0 +1,152 @@
"""Fixture payloads for ``POST /wechatweb/call/list`` (长语音 voice_clip).
These records mirror the real field names confirmed by the 2026-08-07
Playwright capture (see spec §2): ``id`` (bigint PK) / ``userId`` /
``userRealName`` / ``wxId`` (WA=null) / ``talker`` (WA=null) / ``status`` /
``isSend`` / ``companyCode`` (企业串码) / ``startTime`` / ``endTime`` (both
ms-epoch STRINGS) / ``ossFileName`` (recording URL) / ``mp3Duration`` (秒) /
``durationFormat`` (HH:MM:SS).
The envelope is the SAME shape as ``queryWeChatVoiceList`` (rows under
``data.data``, declared total on ``data.total``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "pageNum":.., "pageSize":..}}
The cassette ships four records across two pages so a test can drive the
page-number paginator across the empty-page stop signal:
* page 1 -> 2 of the 4 fixture records, ``total=4``
* page 2 -> the other 2 records, ``total=4``
* (page 3 -> empty ``data`` array, served inline by the test)
Records deliberately exercise the documented mapping edges:
* ``vc_66001`` — wechat clip with start/end as ms-epoch strings, non-null
``ossFileName``, ``isSend=null`` (the WA/wechat long-clip payload carries
``isSend`` as null in the capture).
* ``vc_66002`` — clip with ``ossFileName=null`` (no recording URL).
* ``vc_66003`` — clip with ``mp3Duration=0`` + ``durationFormat="00:00:00"``.
* ``vc_66004`` — clip with a very long ``ossFileName`` URL.
"""
# Module name mirrors the upstream feature (长语音 voice_clip list).
# ruff: noqa: N999
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped voice-clip records (4 total across 2 pages).
# ---------------------------------------------------------------------------
VOICE_CLIP_RECORDS: list[dict[str, Any]] = [
{
# Standard wechat clip: start/end as ms-epoch strings, recording URL.
"id": 66001,
"userId": "832EC001_wechat_staff",
"userRealName": "聂玉婷",
"wxId": "wxid_staff_alice01",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072733378", # ms-epoch string -> int
"endTime": "1786072926741",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/imei001/1786072733378_1786072926741.mp3"
),
"mp3Duration": 193,
"durationFormat": "00:03:13",
},
{
# Clip with no recording URL.
"id": 66002,
"userId": "832EC001_wechat_staff",
"userRealName": "聂玉婷",
"wxId": "wxid_staff_alice01",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072926742",
"endTime": "1786072926742",
"ossFileName": None,
"mp3Duration": 0,
"durationFormat": "00:00:00",
},
{
# Clip with zero duration.
"id": 66003,
"userId": "832EC002_wechat_staff",
"userRealName": "李四",
"wxId": "wxid_staff_bob02",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786000000000",
"endTime": "1786000000000",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/imei002/1786000000000_1786000000000.mp3"
),
"mp3Duration": 0,
"durationFormat": "00:00:00",
},
{
# WA employee (userId range F33DA..., wxId=null) with a long URL.
"id": 66004,
"userId": "F33DA003_wa_staff",
"userRealName": "王五",
"wxId": None,
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786100000000",
"endTime": "1786100123000",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/wa/F33DA003/1786100000000_1786100123000.mp3"
),
"mp3Duration": 123,
"durationFormat": "00:02:03",
},
]
# Default page size for the fixture (and the scraper). total=4 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(VOICE_CLIP_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_voice_clip_list_response(
records: list[dict[str, Any]],
*,
total: int,
page_num: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``call/list`` response envelope.
Shape (rows + pagination fields directly under ``data``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "pageNum":.., "pageSize":..}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-{page_num}",
"data": {
"data": list(records),
"total": total,
"pageNum": page_num,
"pageSize": page_size,
},
}
yunke-server/tests/test_api_voice_clips.py
+239
View File
@@ -0,0 +1,239 @@
"""Tests for GET /api/voice-clips — 长语音 (voice_clip) list.
Seeds 3 clips (2 wechat for user u1 at end_time 1000/3000, 1 whatsapp for u2
at end_time 2000) and asserts filters + ordering (``end_time desc``) +
pagination + the ``user_id``-beats-``user_real_name`` precedence rule.
"""
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import VoiceClip
async def _seed(session: AsyncSession) -> None:
session.add_all(
[
VoiceClip(
voice_clip_id=1,
user_id="u1",
channel="wechat",
user_real_name="Alice",
company_code="8m0atd",
start_time=900,
end_time=1000,
oss_file_name="https://oss/clip1.mp3",
mp3_duration=100,
duration_format="00:01:40",
),
VoiceClip(
voice_clip_id=2,
user_id="u2",
channel="whatsapp",
user_real_name="Bob",
company_code="8m0atd",
start_time=1900,
end_time=2000,
oss_file_name="https://oss/clip2.mp3",
mp3_duration=60,
duration_format="00:01:00",
),
VoiceClip(
voice_clip_id=3,
user_id="u1",
channel="wechat",
user_real_name="Alice",
company_code="other",
start_time=2900,
end_time=3000,
oss_file_name=None,
mp3_duration=0,
duration_format="00:00:00",
),
]
)
await session.commit()
async def test_default_list_returns_all_ordered_desc(make_client):
"""No filters -> 3 rows, ordered by ``end_time desc``."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips")
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert [it["voice_clip_id"] for it in body["items"]] == [3, 2, 1]
for key in (
"voice_clip_id",
"user_id",
"channel",
"user_real_name",
"wx_id",
"talker",
"status",
"is_send",
"company_code",
"start_time",
"end_time",
"oss_file_name",
"mp3_duration",
"duration_format",
"raw_json",
):
assert key in body["items"][0]
async def test_filter_by_user_id(make_client):
"""``?user_id=u1`` -> 2 clips for Alice."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"user_id": "u1"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 3}
async def test_user_id_takes_precedence_over_user_real_name(make_client):
"""When both ``user_id`` and ``user_real_name`` are given, user_id wins.
The 云客 modal switches between the two based on whether the user edited
the staff-name box; the id is the more precise filter and wins on conflict.
Here ``user_id=u1`` (Alice) + ``user_real_name=Bob`` should still return
Alice's clips (user_id path), NOT Bob's.
"""
client = await make_client(_seed)
r = await client.get(
"/api/voice-clips", params={"user_id": "u1", "user_real_name": "Bob"}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 3}
async def test_filter_by_user_real_name_fuzzy(make_client):
"""``?user_real_name=ob`` -> Bob's clip only (ILIKE contains)."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"user_real_name": "ob"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filter_by_channel(make_client):
"""``?channel=whatsapp`` -> the single WA clip."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"channel": "whatsapp"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filter_by_company_code(make_client):
"""``?company_code=8m0atd`` -> 2 clips (clip 3 has 'other')."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"company_code": "8m0atd"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 2}
async def test_time_window_filter(make_client):
"""``?start=1500&end=2500`` -> only clip 2 (end_time 2000)."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"start": 1500, "end": 2500})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filters_compose(make_client):
"""``user_id=u1 & start=1500`` -> only clip 3 (u1's later clip)."""
client = await make_client(_seed)
r = await client.get(
"/api/voice-clips", params={"user_id": "u1", "start": 1500}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 3
async def test_pagination(make_client):
"""``page_size=1`` -> 1 item, total 3."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"page": 1, "page_size": 1})
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert len(body["items"]) == 1
assert body["items"][0]["voice_clip_id"] == 3
async def test_oss_file_name_serializes(make_client):
"""``oss_file_name`` (OSS recording URL) round-trips so the modal can
render an inline ``<audio>`` player — present when set, None when unset.
"""
client = await make_client(_seed)
r = await client.get("/api/voice-clips")
assert r.status_code == 200
by_id = {it["voice_clip_id"]: it for it in r.json()["items"]}
assert by_id[1]["oss_file_name"] == "https://oss/clip1.mp3"
assert by_id[3]["oss_file_name"] is None
async def test_raw_json_serializes_verbatim(make_client):
"""``raw_json`` round-trips the verbatim upstream record when set."""
async def seed(session: AsyncSession) -> None:
await _seed(session)
session.add(
VoiceClip(
voice_clip_id=99,
user_id="u1",
channel="wechat",
user_real_name="Alice",
start_time=5000,
end_time=5100,
raw_json={
"id": 99,
"userId": "u1",
"userRealName": "Alice",
"startTime": "5000",
"endTime": "5100",
"ossFileName": "https://oss/clip99.mp3",
"mp3Duration": 100,
"durationFormat": "00:01:40",
"companyCode": "8m0atd",
},
)
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/voice-clips", params={"user_id": "u1", "start": 5000})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 99
assert body["items"][0]["raw_json"] == {
"id": 99,
"userId": "u1",
"userRealName": "Alice",
"startTime": "5000",
"endTime": "5100",
"ossFileName": "https://oss/clip99.mp3",
"mp3Duration": 100,
"durationFormat": "00:01:40",
"companyCode": "8m0atd",
}
# Seeded without an explicit raw_json -> JSON null.
r_all = await client.get("/api/voice-clips")
by_id = {it["voice_clip_id"]: it for it in r_all.json()["items"]}
assert by_id[1]["raw_json"] is None
yunke-server/tests/test_scrape_voice_clips.py
+297
View File
@@ -0,0 +1,297 @@
"""Tests for the voice-clip (长语音) scraper.
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the page-number paginator's primary stop signal):
* page 1 -> 2 records, ``total=4``
* page 2 -> 2 records, ``total=4``
* page 3 -> empty ``data`` array (no more rows)
The scraper must yield exactly the 4 records normalized to the ``VoiceClip``
column shape, make exactly 3 calls, and stop without a 4th. Field mapping
asserted end-to-end incl.:
* ``id`` -> ``voice_clip_id`` (PK, int)
* ``userId`` -> ``user_id``
* ``userRealName`` -> ``user_real_name``
* ``startTime``/``endTime`` -> ``start_time``/``end_time`` (ms-epoch STRING
coerced to int)
* ``ossFileName`` -> ``oss_file_name`` (may be null)
* ``mp3Duration`` -> ``mp3_duration`` (int seconds)
* ``channel`` -> stamped locally from the ``channel`` arg (``wechat`` /
``whatsapp``), NOT from the API response
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.voice_clip import scrape_voice_clips
from app.session.yunke_client import YunkeClient
from tests.cassettes.voiceClip_fixture import (
DEFAULT_PAGE_SIZE,
TOTAL,
VOICE_CLIP_RECORDS,
make_voice_clip_list_response,
)
BASE = "https://crm2.yunkecn.com"
VOICE_CLIP_LIST_URL = f"{BASE}/wechatweb/call/list"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + raw_json
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_voice_clip_list_response(
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client,
"832EC001_wechat_staff",
channel="wechat",
page_size=DEFAULT_PAGE_SIZE,
)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 4 fixture records (not duplicated, not dropped).
assert len(results) == 4
assert {r["voice_clip_id"] for r in results} == {
rec["id"] for rec in VOICE_CLIP_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, VOICE_CLIP_RECORDS, strict=True):
# Identity + locally-stamped channel.
assert result["voice_clip_id"] == record["id"]
assert result["channel"] == "wechat"
# Staff side.
assert result["user_id"] == record["userId"]
assert result["user_real_name"] == record["userRealName"]
assert result["company_code"] == record["companyCode"]
# ms-epoch strings coerced to int.
assert result["start_time"] == int(record["startTime"])
assert result["end_time"] == int(record["endTime"])
# Recording URL (may be None).
assert result["oss_file_name"] == record["ossFileName"]
# Duration fields.
assert result["mp3_duration"] == record["mp3Duration"]
assert result["duration_format"] == record["durationFormat"]
# raw_json preserves the original record verbatim.
assert result["raw_json"] == record
# ---------------------------------------------------------------------------
# Channel is stamped locally (NOT from the API response)
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_stamps_whatsapp_channel():
"""``channel="whatsapp"`` is stamped on every record regardless of payload."""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:1], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "F33DA003_wa_staff", channel="whatsapp"
)
]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["channel"] == "whatsapp"
# ---------------------------------------------------------------------------
# Null ossFileName -> oss_file_name None
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_null_oss_file_name_yields_none():
"""A record with ``ossFileName=null`` -> ``oss_file_name`` None."""
rec = {
"id": 66010,
"userId": "staff_x",
"userRealName": "X",
"wxId": None,
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072733000",
"endTime": "1786072733000",
"ossFileName": None,
"mp3Duration": 0,
"durationFormat": "00:00:00",
}
page1 = make_voice_clip_list_response(
[rec], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "staff_x", channel="wechat", page_size=DEFAULT_PAGE_SIZE
)
]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["oss_file_name"] is None
assert results[0]["voice_clip_id"] == 66010
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize/userId
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_sends_pagination_body():
"""Body carries ``userId`` + advancing ``pageNum``/``pageSize`` only.
The ``call/list`` endpoint takes ``{pageNum, pageSize, userId}``. The
optional ``startTime``/``endTime`` filters are omitted for full-history
sync (the scraper pulls the employee's whole voice-clip set in one pass).
"""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
)
page2 = make_voice_clip_list_response(
[], total=TOTAL, page_num=2, page_size=1
)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "EMP_USER_001", channel="wechat", page_size=1
)
]
finally:
await client.aclose()
# One record on page 1, empty page 2 stops the loop.
assert len(results) == 1
assert route.call_count == 2
# Page 1 body: userId + standard pagination pair, no time window.
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["userId"] == "EMP_USER_001"
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
# No startTime/endTime (full-history scrape).
assert "startTime" not in body1
assert "endTime" not in body1
# Page 2 advances pageNum and keeps userId verbatim.
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
assert body2["userId"] == "EMP_USER_001"
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_voice_clip_list_response([], total=0, page_num=1)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_voice_clips(client, "any_user")]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1