feat(api): 长语音 voice_clip 后端(model+migration+scraper+sync+endpoint)

- VoiceClip model (BigInteger PK voice_clip_id, structured cols + raw_json)
- Alembic migration a1b2c3d4e5f6 (chains from 8c4f1d2a9e6b)
- scrape_voice_clips(client, user_id, *, channel, page_size) per-employee scraper
  over POST /wechatweb/call/list {pageNum, pageSize, userId}; channel stamped
  locally (wechat/whatsapp), startTime/endTime ms-epoch string -> int
- upsert_voice_clip repo + start_time/end_time/mp3_duration int coercion
- sync_voice_clips orchestrator: iterates distinct user_id across
  WechatAccount + WhatsappAccount, stamps channel per account table
- scheduler ENTITY_FUNCS['voice_clips'] + added to daily 'all' job
- GET /api/voice-clips (user_id exact beats user_real_name fuzzy; channel;
  company_code; start/end window on end_time; ordered end_time desc)
- VoiceClipOut schema + router registered in main.py
- tests: scraper (5) + API (11) = 16 new; full suite 370 green

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-10 18:12:28 +08:00
parent a0db4535f1
commit d9060b6920
12 changed files with 1271 additions and 1 deletions
+3
View File
@@ -15,6 +15,7 @@ Includes:
(see :mod:`app.api.routers.groups`). (see :mod:`app.api.routers.groups`).
* ``GET /api/calls`` (see :mod:`app.api.routers.calls`). * ``GET /api/calls`` (see :mod:`app.api.routers.calls`).
* ``GET /api/sms`` (see :mod:`app.api.routers.sms`). * ``GET /api/sms`` (see :mod:`app.api.routers.sms`).
* ``GET /api/voice-clips`` (see :mod:`app.api.routers.voice_clips`).
* ``GET /api/sync/status`` (see :mod:`app.api.routers.sync`). * ``GET /api/sync/status`` (see :mod:`app.api.routers.sync`).
* ``GET /api/search`` (see :mod:`app.api.routers.search`). * ``GET /api/search`` (see :mod:`app.api.routers.search`).
* ``GET /api/export/chat-html`` (see :mod:`app.api.routers.export`). * ``GET /api/export/chat-html`` (see :mod:`app.api.routers.export`).
@@ -49,6 +50,7 @@ from app.api.routers import (
sms, sms,
staff_chat_stats, staff_chat_stats,
sync, sync,
voice_clips,
) )
app = FastAPI(title="yunke data mirror", dependencies=[Depends(require_auth)]) app = FastAPI(title="yunke data mirror", dependencies=[Depends(require_auth)])
@@ -72,6 +74,7 @@ app.include_router(sync.router)
app.include_router(search.router) app.include_router(search.router)
app.include_router(sessions.router) app.include_router(sessions.router)
app.include_router(export.router) app.include_router(export.router)
app.include_router(voice_clips.router)
@app.get("/api/health") @app.get("/api/health")
@@ -0,0 +1,89 @@
"""GET /api/voice-clips — 长语音 (voice_clip) list.
Read-only query over the mirrored ``voice_clip`` table. Powers the
``VoiceClipModal`` (微信长语音 / WA wavoicecalllist 共用的只读展示 modal).
Filters (all optional):
* ``user_id`` — exact employee-id match (modal opens from a CallList
row keyed on the employee). Takes PRECEDENCE over ``user_real_name`` when
both are given (the id is more precise).
* ``user_real_name`` — employee-name ILIKE / contains match (the 云客 modal's
员工姓名 fuzzy filter). Ignored when ``user_id`` is present.
* ``channel`` — messaging channel (``wechat`` / ``whatsapp``).
* ``company_code`` — exact 企业串码 match.
* ``start`` / ``end`` — ms-epoch window compared against ``end_time`` (录音结束时间,
inclusive) — mirrors the 云客 modal's 录音结束时间 range filter.
Ordered by ``end_time desc`` (nulls last) — most recent clips first, matching
the 云客 modal's default ordering.
"""
from __future__ import annotations
from typing import Annotated
import sqlalchemy as sa
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_session
from app.api.schemas import Page, VoiceClipOut
from app.db.models import VoiceClip
router = APIRouter(prefix="/api", tags=["voice_clips"])
SessionDep = Annotated[AsyncSession, Depends(get_session)]
PageParam = Annotated[int, Query(ge=1)]
PageSizeParam = Annotated[int, Query(ge=1, le=100)]
@router.get("/voice-clips")
async def list_voice_clips(
session: SessionDep,
page: PageParam = 1,
page_size: PageSizeParam = 20,
user_id: Annotated[str | None, Query()] = None,
user_real_name: Annotated[str | None, Query()] = None,
channel: Annotated[str | None, Query()] = None,
company_code: Annotated[str | None, Query()] = None,
start: Annotated[int | None, Query(ge=0)] = None,
end: Annotated[int | None, Query(ge=0)] = None,
) -> Page[VoiceClipOut]:
"""List voice clips with optional filters, newest by recording end first."""
conditions = []
# user_id takes precedence over user_real_name (id is the more precise
# filter; the 云客 modal switches between them based on whether the user
# edited the staff-name box).
if user_id is not None:
conditions.append(VoiceClip.user_id == user_id)
elif user_real_name is not None:
conditions.append(VoiceClip.user_real_name.ilike(f"%{user_real_name}%"))
if channel is not None:
conditions.append(VoiceClip.channel == channel)
if company_code is not None:
conditions.append(VoiceClip.company_code == company_code)
if start is not None:
conditions.append(VoiceClip.end_time >= start)
if end is not None:
conditions.append(VoiceClip.end_time <= end)
total_stmt = sa.select(sa.func.count()).select_from(VoiceClip)
if conditions:
total_stmt = total_stmt.where(*conditions)
total = (await session.execute(total_stmt)).scalar_one()
list_stmt = sa.select(VoiceClip)
if conditions:
list_stmt = list_stmt.where(*conditions)
list_stmt = (
list_stmt.order_by(
sa.nullslast(VoiceClip.end_time.desc()),
VoiceClip.voice_clip_id.asc(),
)
.limit(page_size)
.offset((page - 1) * page_size)
)
rows = (await session.execute(list_stmt)).scalars().all()
items = [VoiceClipOut.model_validate(row) for row in rows]
return Page(items=items, total=total, page=page, page_size=page_size)
+31
View File
@@ -249,6 +249,37 @@ class SmsOut(BaseModel):
raw_json: dict[str, Any] | None = None raw_json: dict[str, Any] | None = None
class VoiceClipOut(BaseModel):
"""长语音 (voice_clip) row — a long audio segment from ``/wechatweb/call/list``.
The structured columns power the VoiceClipModal table (员工姓名 /
企业串码 / 文件地址 / 录音开始时间 / 录音结束时间 / 录音时长); the verbatim
upstream record is exposed as ``raw_json`` so the frontend can render any
extra fields without a second endpoint. Picked up automatically via
``from_attributes``.
"""
model_config = ConfigDict(from_attributes=True)
voice_clip_id: int
user_id: str
channel: str
user_real_name: str | None = None
wx_id: str | None = None
talker: str | None = None
status: str | None = None
is_send: bool | None = None
company_code: str | None = None
start_time: int | None = None
end_time: int | None = None
# OSS recording URL — surfaced so the modal can render an inline
# ``<audio controls>`` player (see spec §6).
oss_file_name: str | None = None
mp3_duration: int | None = None
duration_format: str | None = None
raw_json: dict[str, Any] | None = None
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Sync monitor # Sync monitor
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+47
View File
@@ -477,6 +477,52 @@ class PhoneCall(Base):
) )
class VoiceClip(Base):
"""长语音片段 — source: ``POST /wechatweb/call/list`` (WeChat + WhatsApp
share the same endpoint, distinguished by the employee's channel).
Distinct from :class:`PhoneCall` (which holds ``callLog/list`` phone calls
and ``queryWeChatVoiceList`` voice-call records): a voice_clip is one of the
long audio segments listed in the 云客 微信长语音 / WA wavoicecalllist modals.
PK is the source ``id`` (a cross-employee-globally-unique bigint).
``channel`` here is the **messaging channel** in {``wechat``, ``whatsapp``}
— stamped LOCALLY by the sync orchestrator based on which staff-account
table owns the scraped ``user_id`` (WechatAccount -> ``wechat``,
WhatsappAccount -> ``whatsapp``). The upstream endpoint is shared and does
NOT distinguish the channel in its response, so the channel must come from
the employee mapping, not the record.
``start_time`` / ``end_time`` arrive as ms-epoch **strings** (e.g.
``"1786072733378"``); the scraper normalizes them to ``int``.
"""
__tablename__ = "voice_clip"
voice_clip_id: Mapped[int] = mapped_column(
BigInteger, primary_key=True, autoincrement=False
)
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
user_real_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
wx_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
talker: Mapped[str | None] = mapped_column(String(128), nullable=True)
status: Mapped[str | None] = mapped_column(String(32), nullable=True)
is_send: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
company_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
start_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
end_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
oss_file_name: Mapped[str | None] = mapped_column(String(1024), nullable=True)
mp3_duration: Mapped[int | None] = mapped_column(Integer, nullable=True)
duration_format: Mapped[str | None] = mapped_column(String(32), nullable=True)
channel: Mapped[str] = mapped_column(String(16), nullable=False)
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
fetched_at: Mapped[dt.datetime] = _fetched_at()
updated_at: Mapped[dt.datetime] = _updated_at()
__table_args__ = (Index("ix_voice_clip_user_time", "user_id", "start_time"),)
class Sms(Base): class Sms(Base):
"""短信记录 — source: ``POST /pc/customer/smsLogsList``.""" """短信记录 — source: ``POST /pc/customer/smsLogsList``."""
@@ -617,6 +663,7 @@ __all__ = [
"Staff", "Staff",
"SyncRun", "SyncRun",
"SyncState", "SyncState",
"VoiceClip",
"WechatAccount", "WechatAccount",
"WhatsappAccount", "WhatsappAccount",
"YkAccount", "YkAccount",
+183
View File
@@ -0,0 +1,183 @@
"""Voice-clip (长语音) scraper.
Wraps ``POST /wechatweb/call/list`` — the endpoint behind the 云客 微信长语音 /
WA wavoicecalllist modals. Unlike :func:`app.scraper.wechat.scrape_wechat_voice_calls`
(which is scoped by ``departmentId`` + ``callSource``), this endpoint is scoped
**per employee**: the request body takes a required ``userId`` (employee id) and
returns that employee's long audio segments (the modal opens from a CallList
row keyed on the employee).
Endpoint contract (Playwright-confirmed 2026-08-07, see spec §2):
* Request body ``{pageNum, pageSize, userId}`` (JSON). ``startTime`` / ``endTime``
ms-epoch string filters are OPTIONAL — omitted for full-history sync.
* Response envelope ``{code, message, success, data: {pageNum, pageSize, total,
data: [...]}}`` — the SAME shape as ``queryWeChatVoiceList`` (rows under
``data.data``, declared total on ``data.total``), so the pagination helpers
mirror the voice-call scraper.
* Row fields: ``id`` (bigint PK) / ``userId`` / ``userRealName`` / ``wxId`` /
``talker`` / ``status`` / ``isSend`` / ``companyCode`` / ``startTime`` /
``endTime`` (both ms-epoch STRINGS — normalized to int) / ``ossFileName``
(recording URL) / ``mp3Duration`` (seconds) / ``durationFormat`` (HH:MM:SS).
Channel handling: the endpoint is SHARED by WeChat and WhatsApp employees
(distinct ``userId`` ranges, same path). The response does NOT carry the
channel, so :func:`scrape_voice_clips` takes a ``channel`` argument and stamps
it locally on every yielded record — the caller (sync orchestrator) resolves
it from the owning staff-account table (WechatAccount -> ``wechat``,
WhatsappAccount -> ``whatsapp``).
Reuses :func:`app.scraper.base.paginate_by_page` and emits snake_case dicts
plus a verbatim ``raw_json`` copy, mirroring :mod:`app.scraper.wechat`.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from app.scraper.base import paginate_by_page
from app.scraper.parsers import coerce_int_or_none
from app.session.yunke_client import YunkeClient
__all__ = ["scrape_voice_clips"]
VOICE_CLIP_LIST_PATH = "/wechatweb/call/list"
def _parse_voice_clip_total(resp: Any) -> int | None:
"""Read the declared row count from the ``call/list`` envelope.
Shape: ``data.total`` (an int) — the same location as
``queryWeChatVoiceList``. Returns ``None`` if the path is absent so the
paginator falls back to the empty-page stop signal.
"""
try:
body = resp.json()
except (AttributeError, ValueError):
return None
data = body.get("data") if isinstance(body, dict) else None
if not isinstance(data, dict):
return None
total = data.get("total")
return int(total) if isinstance(total, (int, float)) and not isinstance(
total, bool
) else None
def _parse_voice_clip_rows(resp: Any) -> list[dict[str, Any]]:
"""Extract the voice-clip list from the ``call/list`` envelope.
Shape: ``data.data`` (a list of records). Missing / non-list values
collapse to an empty list so the paginator sees "empty page".
"""
try:
body = resp.json()
except (AttributeError, ValueError):
return []
data = body.get("data") if isinstance(body, dict) else None
if not isinstance(data, dict):
return []
rows = data.get("data")
return rows if isinstance(rows, list) else []
def _build_voice_clip_body(
page_num: int,
page_size: int,
*,
user_id: str,
) -> dict[str, Any]:
"""Build the ``call/list`` request body for a given page.
``userId`` is the REQUIRED employee filter (the endpoint returns nothing
without it). The optional ``startTime`` / ``endTime`` ms-epoch filters are
omitted so the scrape pulls the employee's full history in one pass (the
sync orchestrator wants all segments, not a time window).
"""
return {
"pageNum": page_num,
"pageSize": page_size,
"userId": user_id,
}
def _normalize_voice_clip(
record: dict[str, Any], *, channel: str
) -> dict[str, Any]:
"""Map one upstream ``call/list`` record to the ``VoiceClip`` column shape.
Stamps the locally-resolved ``channel`` (``wechat`` / ``whatsapp``) — the
API response does not carry it. ``startTime`` / ``endTime`` arrive as
ms-epoch STRINGS (e.g. ``"1786072733378"``); they are coerced to ``int``
via :func:`coerce_int_or_none` so the BigInteger columns land clean.
``mp3Duration`` (seconds, already int in the capture) is coerced to int
defensively. ``isSend`` is carried through verbatim — the repo coerces it
to bool. ``None``-safe on every field: a missing field becomes ``None``
rather than raising, so a partial payload does not break the scrape.
Preserves the original record under ``raw_json``. See spec §3 for the
column-by-column mapping.
"""
return {
"voice_clip_id": record.get("id"),
"user_id": record.get("userId"),
"user_real_name": record.get("userRealName"),
"wx_id": record.get("wxId"),
"talker": record.get("talker"),
"status": record.get("status"),
"is_send": record.get("isSend"),
"company_code": record.get("companyCode"),
# ms-epoch strings -> int (the BigInteger column type).
"start_time": coerce_int_or_none(record.get("startTime")),
"end_time": coerce_int_or_none(record.get("endTime")),
"oss_file_name": record.get("ossFileName"),
"mp3_duration": coerce_int_or_none(record.get("mp3Duration")),
"duration_format": record.get("durationFormat"),
# Local label — resolved by the orchestrator from the staff-account
# table that owns this user_id (NOT from the API response).
"channel": channel,
"raw_json": record,
}
async def scrape_voice_clips(
client: YunkeClient,
user_id: str,
*,
channel: str = "wechat",
page_size: int = 50,
) -> AsyncIterator[dict[str, Any]]:
"""Yield normalized voice-clip records for one employee from ``call/list``.
Walks ``POST /wechatweb/call/list`` page by page (body
``{pageNum, pageSize, userId}``) until the upstream reports no more rows
(empty ``data.data`` array — the primary stop signal), mapping each record
to the ``VoiceClip`` column shape plus a verbatim ``raw_json`` copy.
Parameters
----------
client:
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
handled by the client.
user_id:
Employee id (the REQUIRED ``userId`` request field). The sync
orchestrator resolves this from the staff-account tables.
channel:
Local messaging-channel label (``"wechat"`` / ``"whatsapp"``) stamped
on every yielded record. The endpoint is shared and does NOT carry the
channel; the caller resolves it from the owning account table.
page_size:
Page size to request. Defaults to 50 (a sane middle ground between
request count and per-response payload size for this endpoint).
"""
async for record in paginate_by_page(
client,
VOICE_CLIP_LIST_PATH,
page_size=page_size,
parse_total=_parse_voice_clip_total,
parse_rows=_parse_voice_clip_rows,
build_body=_build_voice_clip_body,
user_id=user_id,
):
yield _normalize_voice_clip(record, channel=channel)
+30
View File
@@ -63,6 +63,7 @@ from app.db.models import (
PhoneCall, PhoneCall,
Sms, Sms,
Staff, Staff,
VoiceClip,
WechatAccount, WechatAccount,
WhatsappAccount, WhatsappAccount,
) )
@@ -79,6 +80,7 @@ __all__ = [
"upsert_staff", "upsert_staff",
"upsert_staff_from_account", "upsert_staff_from_account",
"upsert_voice_call", "upsert_voice_call",
"upsert_voice_clip",
"upsert_wechat_account", "upsert_wechat_account",
"upsert_whatsapp_account", "upsert_whatsapp_account",
] ]
@@ -108,6 +110,12 @@ _INT_FIELDS: frozenset[str] = frozenset(
"start_call_time", "start_call_time",
"qc_status", "qc_status",
"create_time", "create_time",
# VoiceClip (voice_clip): ms-epoch + duration fields arrive as strings
# or ints depending on the capture; coerce to int defensively. The
# scraper already emits ints, but a hand-built record may not.
"start_time",
"end_time",
"mp3_duration",
} }
) )
@@ -565,6 +573,28 @@ async def upsert_sms(session: AsyncSession, rec: dict[str, Any]) -> None:
await _upsert_one(session, Sms, rec) await _upsert_one(session, Sms, rec)
# ---------------------------------------------------------------------------
# VoiceClip (长语音) upsert
# ---------------------------------------------------------------------------
async def upsert_voice_clip(session: AsyncSession, rec: dict[str, Any]) -> None:
"""Idempotently upsert one :class:`VoiceClip` (PK = ``voice_clip_id``).
Same ``voice_clip_id`` twice -> one row, with non-PK fields refreshed from
the later record. Applies the documented int coercions (``start_time`` /
``end_time`` / ``mp3_duration`` -> int when they arrive as numeric strings;
the voice-clip scraper already emits ints via :func:`coerce_int_or_none`,
but a hand-built record may not). ``is_send`` is a bool field and goes
through :func:`_coerce_bool` like the other bool columns.
``channel`` is stamped locally by the sync orchestrator (``wechat`` /
``whatsapp``) based on which staff-account table owns the scraped
``user_id``; the endpoint does not carry it.
"""
await _upsert_one(session, VoiceClip, rec)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Media upsert (M2, Task 2.6) # Media upsert (M2, Task 2.6)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+138 -1
View File
@@ -49,9 +49,10 @@ from typing import Any
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Department, SyncRun from app.db.models import Department, SyncRun, WechatAccount, WhatsappAccount
from app.scraper.call import scrape_phone_calls, scrape_sms from app.scraper.call import scrape_phone_calls, scrape_sms
from app.scraper.media import download_call_recordings, download_message_media from app.scraper.media import download_call_recordings, download_message_media
from app.scraper.voice_clip import scrape_voice_clips
from app.scraper.wechat import ( from app.scraper.wechat import (
scrape_departments, scrape_departments,
scrape_wechat_accounts, scrape_wechat_accounts,
@@ -76,6 +77,7 @@ from app.storage.repos import (
upsert_phone_call, upsert_phone_call,
upsert_sms, upsert_sms,
upsert_voice_call, upsert_voice_call,
upsert_voice_clip,
upsert_wechat_account, upsert_wechat_account,
upsert_whatsapp_account, upsert_whatsapp_account,
) )
@@ -91,6 +93,7 @@ __all__ = [
"sync_departments", "sync_departments",
"sync_phone_calls", "sync_phone_calls",
"sync_sms", "sync_sms",
"sync_voice_clips",
"sync_wechat_accounts", "sync_wechat_accounts",
"sync_wechat_friends", "sync_wechat_friends",
"sync_wechat_groups", "sync_wechat_groups",
@@ -131,6 +134,10 @@ ENTITY_SMS = "sms"
# object store (records a SyncRun with this entity_type; no SyncState row # object store (records a SyncRun with this entity_type; no SyncState row
# because the phone_call table itself is the cursor). # because the phone_call table itself is the cursor).
ENTITY_CALL_RECORDINGS = "call_recordings" ENTITY_CALL_RECORDINGS = "call_recordings"
# 长语音 (voice_clip) — per-employee scrape of ``/wechatweb/call/list``. The
# sync walks every distinct user_id across both staff-account tables and stamps
# the channel locally (WechatAccount -> wechat, WhatsappAccount -> whatsapp).
ENTITY_VOICE_CLIP = "voice_clip"
CHANNEL_WECHAT = "wechat" CHANNEL_WECHAT = "wechat"
CHANNEL_WHATSAPP = "whatsapp" CHANNEL_WHATSAPP = "whatsapp"
@@ -889,6 +896,136 @@ async def sync_sms(
) )
# ---------------------------------------------------------------------------
# VoiceClip (长语音) — per-employee scrape across both staff-account tables
# ---------------------------------------------------------------------------
async def _collect_voice_clip_staff(
session: AsyncSession,
) -> list[tuple[str, str]]:
"""Return ``(user_id, channel)`` pairs for voice-clip scraping.
One pair per DISTINCT ``user_id`` across both staff-account tables. The
``call/list`` endpoint is scoped per employee (``userId`` request field)
and is shared by WeChat + WhatsApp staff (distinct ``userId`` ranges);
each employee is scraped exactly once. WeChat accounts take precedence on
collision (deterministic — in practice the two ``userId`` ranges are
disjoint, so collisions are not expected).
The returned ``channel`` is the LOCAL messaging-channel label stamped on
every scraped record: ``wechat`` for ids found in :class:`WechatAccount`,
``whatsapp`` for ids found only in :class:`WhatsappAccount`. The endpoint
response does NOT carry the channel, so it must be resolved here from the
owning account table.
"""
pairs: list[tuple[str, str]] = []
seen: set[str] = set()
wechat_rows = (
await session.execute(
sa.select(WechatAccount.user_id)
.where(WechatAccount.user_id.is_not(None))
.where(WechatAccount.user_id != "")
.distinct()
)
).all()
for (uid,) in wechat_rows:
if uid and uid not in seen:
seen.add(uid)
pairs.append((uid, CHANNEL_WECHAT))
wa_rows = (
await session.execute(
sa.select(WhatsappAccount.user_id)
.where(WhatsappAccount.user_id.is_not(None))
.where(WhatsappAccount.user_id != "")
.distinct()
)
).all()
for (uid,) in wa_rows:
if uid and uid not in seen:
seen.add(uid)
pairs.append((uid, CHANNEL_WHATSAPP))
return pairs
async def sync_voice_clips(
session: AsyncSession, client: YunkeClient
) -> SyncResult:
"""Full re-scrape of all 长语音 (voice_clip) records across all staff.
Walks every distinct ``user_id`` in the local staff-account tables
(:class:`WechatAccount` + :class:`WhatsappAccount`) and, per employee,
scrapes ``POST /wechatweb/call/list`` (body ``{pageNum, pageSize,
userId}``), upserting each record into :class:`VoiceClip`. The channel is
stamped locally (``wechat`` / ``whatsapp``) based on which account table
owns the ``user_id`` — the shared endpoint does not carry it.
Mirrors :func:`sync_wechat_voice_calls`'s failure policy + SyncRun
bookkeeping, but iterates MULTIPLE per-employee scrape streams (rather than
one department-scoped stream). A scraper failure on one employee is caught,
recorded as ``status='failed'`` for the whole entity, and a failed
:class:`SyncResult` is returned (no re-raise) — the caller decides whether
to surface / retry.
When no staff ``user_id`` is present (e.g. accounts not yet synced), the
scrape is SKIPPED with a logged warning and a ``status='skipped'``
:class:`SyncResult` (no ``SyncRun`` recorded), so a fresh DB does not abort
a larger batch.
"""
pairs = await _collect_voice_clip_staff(session)
if not pairs:
logger.warning(
"sync_voice_clips: no staff user_ids found in wechat_account /"
" whatsapp_account (sync accounts first?); skipping call/list scrape"
)
return SyncResult(
entity=ENTITY_VOICE_CLIP,
status="skipped",
fetched=0,
upserted=0,
error="no staff user_ids available",
)
run = await _start_run(session, ENTITY_VOICE_CLIP)
fetched = 0
upserted = 0
try:
for uid, channel in pairs:
async for rec in scrape_voice_clips(client, uid, channel=channel):
await upsert_voice_clip(session, rec)
fetched += 1
upserted += 1
except Exception as exc:
logger.exception("scrape_voice_clips failed")
error = repr(exc)
await _finish_run(
session, run, status="failed",
fetched=fetched, upserted=upserted, error=error,
)
return SyncResult(
entity=ENTITY_VOICE_CLIP,
status="failed",
fetched=fetched,
upserted=upserted,
error=error,
)
await set_watermark(session, ENTITY_VOICE_CLIP)
await _finish_run(
session, run, status="success", fetched=fetched, upserted=upserted
)
return SyncResult(
entity=ENTITY_VOICE_CLIP,
status="success",
fetched=fetched,
upserted=upserted,
)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Coordinator # Coordinator
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+13
View File
@@ -59,6 +59,7 @@ from app.sync.orchestrator import (
sync_departments, sync_departments,
sync_phone_calls, sync_phone_calls,
sync_sms, sync_sms,
sync_voice_clips,
sync_wa_voice_calls, sync_wa_voice_calls,
sync_wechat_accounts, sync_wechat_accounts,
sync_wechat_friends, sync_wechat_friends,
@@ -188,6 +189,13 @@ async def _run_sms(
return [await sync_sms(session, client)] return [await sync_sms(session, client)]
async def _run_voice_clips(
session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> list[SyncResult]:
"""Per-employee scrape of 长语音 (voice_clip) across all staff."""
return [await sync_voice_clips(session, client)]
async def _run_call_recordings( async def _run_call_recordings(
session: AsyncSession, client: YunkeClient, store: ObjectStore session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> list[SyncResult]: ) -> list[SyncResult]:
@@ -264,6 +272,10 @@ async def _run_all(
results.extend(await _run_wa_messages(session, client, store)) results.extend(await _run_wa_messages(session, client, store))
results.append(await sync_phone_calls(session, client)) results.append(await sync_phone_calls(session, client))
results.append(await sync_sms(session, client)) results.append(await sync_sms(session, client))
# 长语音 (voice_clip): per-employee scrape across both staff-account
# tables. Must run AFTER wechat_accounts + wa_accounts (which seed the
# user_id -> channel mapping the voice-clip sync reads).
results.append(await sync_voice_clips(session, client))
return results return results
@@ -283,6 +295,7 @@ ENTITY_FUNCS: dict[
"wa_friends": _run_wa_friends, "wa_friends": _run_wa_friends,
"phone_calls": _run_phone_calls, "phone_calls": _run_phone_calls,
"sms": _run_sms, "sms": _run_sms,
"voice_clips": _run_voice_clips,
"call_recordings": _run_call_recordings, "call_recordings": _run_call_recordings,
"messages": _run_messages, "messages": _run_messages,
"wa_messages": _run_wa_messages, "wa_messages": _run_wa_messages,
@@ -0,0 +1,49 @@
"""voice_clip table
Revision ID: a1b2c3d4e5f6
Revises: 8c4f1d2a9e6b
Create Date: 2026-08-10 00:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'a1b2c3d4e5f6'
down_revision: str | Sequence[str] | None = '8c4f1d2a9e6b'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema."""
op.create_table('voice_clip',
sa.Column('voice_clip_id', sa.BigInteger(), autoincrement=False, nullable=False),
sa.Column('user_id', sa.String(length=64), nullable=False),
sa.Column('user_real_name', sa.String(length=255), nullable=True),
sa.Column('wx_id', sa.String(length=128), nullable=True),
sa.Column('talker', sa.String(length=128), nullable=True),
sa.Column('status', sa.String(length=32), nullable=True),
sa.Column('is_send', sa.Boolean(), nullable=True),
sa.Column('company_code', sa.String(length=64), nullable=True),
sa.Column('start_time', sa.BigInteger(), nullable=True),
sa.Column('end_time', sa.BigInteger(), nullable=True),
sa.Column('oss_file_name', sa.String(length=1024), nullable=True),
sa.Column('mp3_duration', sa.Integer(), nullable=True),
sa.Column('duration_format', sa.String(length=32), nullable=True),
sa.Column('channel', sa.String(length=16), nullable=False),
sa.Column('raw_json', sa.JSON(), nullable=True),
sa.Column('fetched_at', sa.DateTime(), server_default=sa.func.now(), nullable=False),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), nullable=False),
sa.PrimaryKeyConstraint('voice_clip_id')
)
op.create_index('ix_voice_clip_user_time', 'voice_clip', ['user_id', 'start_time'], unique=False)
def downgrade() -> None:
"""Downgrade schema."""
op.drop_index('ix_voice_clip_user_time', table_name='voice_clip')
op.drop_table('voice_clip')
@@ -0,0 +1,152 @@
"""Fixture payloads for ``POST /wechatweb/call/list`` (长语音 voice_clip).
These records mirror the real field names confirmed by the 2026-08-07
Playwright capture (see spec §2): ``id`` (bigint PK) / ``userId`` /
``userRealName`` / ``wxId`` (WA=null) / ``talker`` (WA=null) / ``status`` /
``isSend`` / ``companyCode`` (企业串码) / ``startTime`` / ``endTime`` (both
ms-epoch STRINGS) / ``ossFileName`` (recording URL) / ``mp3Duration`` (秒) /
``durationFormat`` (HH:MM:SS).
The envelope is the SAME shape as ``queryWeChatVoiceList`` (rows under
``data.data``, declared total on ``data.total``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "pageNum":.., "pageSize":..}}
The cassette ships four records across two pages so a test can drive the
page-number paginator across the empty-page stop signal:
* page 1 -> 2 of the 4 fixture records, ``total=4``
* page 2 -> the other 2 records, ``total=4``
* (page 3 -> empty ``data`` array, served inline by the test)
Records deliberately exercise the documented mapping edges:
* ``vc_66001`` — wechat clip with start/end as ms-epoch strings, non-null
``ossFileName``, ``isSend=null`` (the WA/wechat long-clip payload carries
``isSend`` as null in the capture).
* ``vc_66002`` — clip with ``ossFileName=null`` (no recording URL).
* ``vc_66003`` — clip with ``mp3Duration=0`` + ``durationFormat="00:00:00"``.
* ``vc_66004`` — clip with a very long ``ossFileName`` URL.
"""
# Module name mirrors the upstream feature (长语音 voice_clip list).
# ruff: noqa: N999
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped voice-clip records (4 total across 2 pages).
# ---------------------------------------------------------------------------
VOICE_CLIP_RECORDS: list[dict[str, Any]] = [
{
# Standard wechat clip: start/end as ms-epoch strings, recording URL.
"id": 66001,
"userId": "832EC001_wechat_staff",
"userRealName": "聂玉婷",
"wxId": "wxid_staff_alice01",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072733378", # ms-epoch string -> int
"endTime": "1786072926741",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/imei001/1786072733378_1786072926741.mp3"
),
"mp3Duration": 193,
"durationFormat": "00:03:13",
},
{
# Clip with no recording URL.
"id": 66002,
"userId": "832EC001_wechat_staff",
"userRealName": "聂玉婷",
"wxId": "wxid_staff_alice01",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072926742",
"endTime": "1786072926742",
"ossFileName": None,
"mp3Duration": 0,
"durationFormat": "00:00:00",
},
{
# Clip with zero duration.
"id": 66003,
"userId": "832EC002_wechat_staff",
"userRealName": "李四",
"wxId": "wxid_staff_bob02",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786000000000",
"endTime": "1786000000000",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/imei002/1786000000000_1786000000000.mp3"
),
"mp3Duration": 0,
"durationFormat": "00:00:00",
},
{
# WA employee (userId range F33DA..., wxId=null) with a long URL.
"id": 66004,
"userId": "F33DA003_wa_staff",
"userRealName": "王五",
"wxId": None,
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786100000000",
"endTime": "1786100123000",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/wa/F33DA003/1786100000000_1786100123000.mp3"
),
"mp3Duration": 123,
"durationFormat": "00:02:03",
},
]
# Default page size for the fixture (and the scraper). total=4 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(VOICE_CLIP_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_voice_clip_list_response(
records: list[dict[str, Any]],
*,
total: int,
page_num: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``call/list`` response envelope.
Shape (rows + pagination fields directly under ``data``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "pageNum":.., "pageSize":..}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-{page_num}",
"data": {
"data": list(records),
"total": total,
"pageNum": page_num,
"pageSize": page_size,
},
}
+239
View File
@@ -0,0 +1,239 @@
"""Tests for GET /api/voice-clips — 长语音 (voice_clip) list.
Seeds 3 clips (2 wechat for user u1 at end_time 1000/3000, 1 whatsapp for u2
at end_time 2000) and asserts filters + ordering (``end_time desc``) +
pagination + the ``user_id``-beats-``user_real_name`` precedence rule.
"""
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import VoiceClip
async def _seed(session: AsyncSession) -> None:
session.add_all(
[
VoiceClip(
voice_clip_id=1,
user_id="u1",
channel="wechat",
user_real_name="Alice",
company_code="8m0atd",
start_time=900,
end_time=1000,
oss_file_name="https://oss/clip1.mp3",
mp3_duration=100,
duration_format="00:01:40",
),
VoiceClip(
voice_clip_id=2,
user_id="u2",
channel="whatsapp",
user_real_name="Bob",
company_code="8m0atd",
start_time=1900,
end_time=2000,
oss_file_name="https://oss/clip2.mp3",
mp3_duration=60,
duration_format="00:01:00",
),
VoiceClip(
voice_clip_id=3,
user_id="u1",
channel="wechat",
user_real_name="Alice",
company_code="other",
start_time=2900,
end_time=3000,
oss_file_name=None,
mp3_duration=0,
duration_format="00:00:00",
),
]
)
await session.commit()
async def test_default_list_returns_all_ordered_desc(make_client):
"""No filters -> 3 rows, ordered by ``end_time desc``."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips")
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert [it["voice_clip_id"] for it in body["items"]] == [3, 2, 1]
for key in (
"voice_clip_id",
"user_id",
"channel",
"user_real_name",
"wx_id",
"talker",
"status",
"is_send",
"company_code",
"start_time",
"end_time",
"oss_file_name",
"mp3_duration",
"duration_format",
"raw_json",
):
assert key in body["items"][0]
async def test_filter_by_user_id(make_client):
"""``?user_id=u1`` -> 2 clips for Alice."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"user_id": "u1"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 3}
async def test_user_id_takes_precedence_over_user_real_name(make_client):
"""When both ``user_id`` and ``user_real_name`` are given, user_id wins.
The 云客 modal switches between the two based on whether the user edited
the staff-name box; the id is the more precise filter and wins on conflict.
Here ``user_id=u1`` (Alice) + ``user_real_name=Bob`` should still return
Alice's clips (user_id path), NOT Bob's.
"""
client = await make_client(_seed)
r = await client.get(
"/api/voice-clips", params={"user_id": "u1", "user_real_name": "Bob"}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 3}
async def test_filter_by_user_real_name_fuzzy(make_client):
"""``?user_real_name=ob`` -> Bob's clip only (ILIKE contains)."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"user_real_name": "ob"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filter_by_channel(make_client):
"""``?channel=whatsapp`` -> the single WA clip."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"channel": "whatsapp"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filter_by_company_code(make_client):
"""``?company_code=8m0atd`` -> 2 clips (clip 3 has 'other')."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"company_code": "8m0atd"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 2}
async def test_time_window_filter(make_client):
"""``?start=1500&end=2500`` -> only clip 2 (end_time 2000)."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"start": 1500, "end": 2500})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filters_compose(make_client):
"""``user_id=u1 & start=1500`` -> only clip 3 (u1's later clip)."""
client = await make_client(_seed)
r = await client.get(
"/api/voice-clips", params={"user_id": "u1", "start": 1500}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 3
async def test_pagination(make_client):
"""``page_size=1`` -> 1 item, total 3."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"page": 1, "page_size": 1})
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert len(body["items"]) == 1
assert body["items"][0]["voice_clip_id"] == 3
async def test_oss_file_name_serializes(make_client):
"""``oss_file_name`` (OSS recording URL) round-trips so the modal can
render an inline ``<audio>`` player — present when set, None when unset.
"""
client = await make_client(_seed)
r = await client.get("/api/voice-clips")
assert r.status_code == 200
by_id = {it["voice_clip_id"]: it for it in r.json()["items"]}
assert by_id[1]["oss_file_name"] == "https://oss/clip1.mp3"
assert by_id[3]["oss_file_name"] is None
async def test_raw_json_serializes_verbatim(make_client):
"""``raw_json`` round-trips the verbatim upstream record when set."""
async def seed(session: AsyncSession) -> None:
await _seed(session)
session.add(
VoiceClip(
voice_clip_id=99,
user_id="u1",
channel="wechat",
user_real_name="Alice",
start_time=5000,
end_time=5100,
raw_json={
"id": 99,
"userId": "u1",
"userRealName": "Alice",
"startTime": "5000",
"endTime": "5100",
"ossFileName": "https://oss/clip99.mp3",
"mp3Duration": 100,
"durationFormat": "00:01:40",
"companyCode": "8m0atd",
},
)
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/voice-clips", params={"user_id": "u1", "start": 5000})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 99
assert body["items"][0]["raw_json"] == {
"id": 99,
"userId": "u1",
"userRealName": "Alice",
"startTime": "5000",
"endTime": "5100",
"ossFileName": "https://oss/clip99.mp3",
"mp3Duration": 100,
"durationFormat": "00:01:40",
"companyCode": "8m0atd",
}
# Seeded without an explicit raw_json -> JSON null.
r_all = await client.get("/api/voice-clips")
by_id = {it["voice_clip_id"]: it for it in r_all.json()["items"]}
assert by_id[1]["raw_json"] is None
@@ -0,0 +1,297 @@
"""Tests for the voice-clip (长语音) scraper.
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the page-number paginator's primary stop signal):
* page 1 -> 2 records, ``total=4``
* page 2 -> 2 records, ``total=4``
* page 3 -> empty ``data`` array (no more rows)
The scraper must yield exactly the 4 records normalized to the ``VoiceClip``
column shape, make exactly 3 calls, and stop without a 4th. Field mapping
asserted end-to-end incl.:
* ``id`` -> ``voice_clip_id`` (PK, int)
* ``userId`` -> ``user_id``
* ``userRealName`` -> ``user_real_name``
* ``startTime``/``endTime`` -> ``start_time``/``end_time`` (ms-epoch STRING
coerced to int)
* ``ossFileName`` -> ``oss_file_name`` (may be null)
* ``mp3Duration`` -> ``mp3_duration`` (int seconds)
* ``channel`` -> stamped locally from the ``channel`` arg (``wechat`` /
``whatsapp``), NOT from the API response
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.voice_clip import scrape_voice_clips
from app.session.yunke_client import YunkeClient
from tests.cassettes.voiceClip_fixture import (
DEFAULT_PAGE_SIZE,
TOTAL,
VOICE_CLIP_RECORDS,
make_voice_clip_list_response,
)
BASE = "https://crm2.yunkecn.com"
VOICE_CLIP_LIST_URL = f"{BASE}/wechatweb/call/list"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + raw_json
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_voice_clip_list_response(
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client,
"832EC001_wechat_staff",
channel="wechat",
page_size=DEFAULT_PAGE_SIZE,
)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 4 fixture records (not duplicated, not dropped).
assert len(results) == 4
assert {r["voice_clip_id"] for r in results} == {
rec["id"] for rec in VOICE_CLIP_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, VOICE_CLIP_RECORDS, strict=True):
# Identity + locally-stamped channel.
assert result["voice_clip_id"] == record["id"]
assert result["channel"] == "wechat"
# Staff side.
assert result["user_id"] == record["userId"]
assert result["user_real_name"] == record["userRealName"]
assert result["company_code"] == record["companyCode"]
# ms-epoch strings coerced to int.
assert result["start_time"] == int(record["startTime"])
assert result["end_time"] == int(record["endTime"])
# Recording URL (may be None).
assert result["oss_file_name"] == record["ossFileName"]
# Duration fields.
assert result["mp3_duration"] == record["mp3Duration"]
assert result["duration_format"] == record["durationFormat"]
# raw_json preserves the original record verbatim.
assert result["raw_json"] == record
# ---------------------------------------------------------------------------
# Channel is stamped locally (NOT from the API response)
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_stamps_whatsapp_channel():
"""``channel="whatsapp"`` is stamped on every record regardless of payload."""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:1], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "F33DA003_wa_staff", channel="whatsapp"
)
]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["channel"] == "whatsapp"
# ---------------------------------------------------------------------------
# Null ossFileName -> oss_file_name None
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_null_oss_file_name_yields_none():
"""A record with ``ossFileName=null`` -> ``oss_file_name`` None."""
rec = {
"id": 66010,
"userId": "staff_x",
"userRealName": "X",
"wxId": None,
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072733000",
"endTime": "1786072733000",
"ossFileName": None,
"mp3Duration": 0,
"durationFormat": "00:00:00",
}
page1 = make_voice_clip_list_response(
[rec], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "staff_x", channel="wechat", page_size=DEFAULT_PAGE_SIZE
)
]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["oss_file_name"] is None
assert results[0]["voice_clip_id"] == 66010
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize/userId
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_sends_pagination_body():
"""Body carries ``userId`` + advancing ``pageNum``/``pageSize`` only.
The ``call/list`` endpoint takes ``{pageNum, pageSize, userId}``. The
optional ``startTime``/``endTime`` filters are omitted for full-history
sync (the scraper pulls the employee's whole voice-clip set in one pass).
"""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
)
page2 = make_voice_clip_list_response(
[], total=TOTAL, page_num=2, page_size=1
)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "EMP_USER_001", channel="wechat", page_size=1
)
]
finally:
await client.aclose()
# One record on page 1, empty page 2 stops the loop.
assert len(results) == 1
assert route.call_count == 2
# Page 1 body: userId + standard pagination pair, no time window.
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["userId"] == "EMP_USER_001"
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
# No startTime/endTime (full-history scrape).
assert "startTime" not in body1
assert "endTime" not in body1
# Page 2 advances pageNum and keeps userId verbatim.
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
assert body2["userId"] == "EMP_USER_001"
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_voice_clip_list_response([], total=0, page_num=1)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_voice_clips(client, "any_user")]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1