"""Voice-clip (长语音) scraper. Wraps ``POST /wechatweb/call/list`` — the endpoint behind the 云客 微信长语音 / WA wavoicecalllist modals. Unlike :func:`app.scraper.wechat.scrape_wechat_voice_calls` (which is scoped by ``departmentId`` + ``callSource``), this endpoint is scoped **per employee**: the request body takes a required ``userId`` (employee id) and returns that employee's long audio segments (the modal opens from a CallList row keyed on the employee). Endpoint contract (Playwright-confirmed 2026-08-07, see spec §2): * Request body ``{pageNum, pageSize, userId}`` (JSON). ``startTime`` / ``endTime`` ms-epoch string filters are OPTIONAL — omitted for full-history sync. * Response envelope ``{code, message, success, data: {pageNum, pageSize, total, data: [...]}}`` — the SAME shape as ``queryWeChatVoiceList`` (rows under ``data.data``, declared total on ``data.total``), so the pagination helpers mirror the voice-call scraper. * Row fields: ``id`` (bigint PK) / ``userId`` / ``userRealName`` / ``wxId`` / ``talker`` / ``status`` / ``isSend`` / ``companyCode`` / ``startTime`` / ``endTime`` (both ms-epoch STRINGS — normalized to int) / ``ossFileName`` (recording URL) / ``mp3Duration`` (seconds) / ``durationFormat`` (HH:MM:SS). Channel handling: the endpoint is SHARED by WeChat and WhatsApp employees (distinct ``userId`` ranges, same path). The response does NOT carry the channel, so :func:`scrape_voice_clips` takes a ``channel`` argument and stamps it locally on every yielded record — the caller (sync orchestrator) resolves it from the owning staff-account table (WechatAccount -> ``wechat``, WhatsappAccount -> ``whatsapp``). Reuses :func:`app.scraper.base.paginate_by_page` and emits snake_case dicts plus a verbatim ``raw_json`` copy, mirroring :mod:`app.scraper.wechat`. """ from __future__ import annotations from collections.abc import AsyncIterator from typing import Any from app.scraper.base import paginate_by_page from app.scraper.parsers import coerce_int_or_none from app.session.yunke_client import YunkeClient __all__ = ["scrape_voice_clips"] VOICE_CLIP_LIST_PATH = "/wechatweb/call/list" def _parse_voice_clip_total(resp: Any) -> int | None: """Read the declared row count from the ``call/list`` envelope. Shape: ``data.total`` (an int) — the same location as ``queryWeChatVoiceList``. Returns ``None`` if the path is absent so the paginator falls back to the empty-page stop signal. """ try: body = resp.json() except (AttributeError, ValueError): return None data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return None total = data.get("total") return int(total) if isinstance(total, (int, float)) and not isinstance( total, bool ) else None def _parse_voice_clip_rows(resp: Any) -> list[dict[str, Any]]: """Extract the voice-clip list from the ``call/list`` envelope. Shape: ``data.data`` (a list of records). Missing / non-list values collapse to an empty list so the paginator sees "empty page". """ try: body = resp.json() except (AttributeError, ValueError): return [] data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [] rows = data.get("data") return rows if isinstance(rows, list) else [] def _build_voice_clip_body( page_num: int, page_size: int, *, user_id: str, ) -> dict[str, Any]: """Build the ``call/list`` request body for a given page. ``userId`` is the REQUIRED employee filter (the endpoint returns nothing without it). The optional ``startTime`` / ``endTime`` ms-epoch filters are omitted so the scrape pulls the employee's full history in one pass (the sync orchestrator wants all segments, not a time window). """ return { "pageNum": page_num, "pageSize": page_size, "userId": user_id, } def _normalize_voice_clip( record: dict[str, Any], *, channel: str ) -> dict[str, Any]: """Map one upstream ``call/list`` record to the ``VoiceClip`` column shape. Stamps the locally-resolved ``channel`` (``wechat`` / ``whatsapp``) — the API response does not carry it. ``startTime`` / ``endTime`` arrive as ms-epoch STRINGS (e.g. ``"1786072733378"``); they are coerced to ``int`` via :func:`coerce_int_or_none` so the BigInteger columns land clean. ``mp3Duration`` (seconds, already int in the capture) is coerced to int defensively. ``isSend`` is carried through verbatim — the repo coerces it to bool. ``None``-safe on every field: a missing field becomes ``None`` rather than raising, so a partial payload does not break the scrape. Preserves the original record under ``raw_json``. See spec §3 for the column-by-column mapping. """ return { "voice_clip_id": record.get("id"), "user_id": record.get("userId"), "user_real_name": record.get("userRealName"), "wx_id": record.get("wxId"), "talker": record.get("talker"), "status": record.get("status"), "is_send": record.get("isSend"), "company_code": record.get("companyCode"), # ms-epoch strings -> int (the BigInteger column type). "start_time": coerce_int_or_none(record.get("startTime")), "end_time": coerce_int_or_none(record.get("endTime")), "oss_file_name": record.get("ossFileName"), "mp3_duration": coerce_int_or_none(record.get("mp3Duration")), "duration_format": record.get("durationFormat"), # Local label — resolved by the orchestrator from the staff-account # table that owns this user_id (NOT from the API response). "channel": channel, "raw_json": record, } async def scrape_voice_clips( client: YunkeClient, user_id: str, *, channel: str = "wechat", page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized voice-clip records for one employee from ``call/list``. Walks ``POST /wechatweb/call/list`` page by page (body ``{pageNum, pageSize, userId}``) until the upstream reports no more rows (empty ``data.data`` array — the primary stop signal), mapping each record to the ``VoiceClip`` column shape plus a verbatim ``raw_json`` copy. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. user_id: Employee id (the REQUIRED ``userId`` request field). The sync orchestrator resolves this from the staff-account tables. channel: Local messaging-channel label (``"wechat"`` / ``"whatsapp"``) stamped on every yielded record. The endpoint is shared and does NOT carry the channel; the caller resolves it from the owning account table. page_size: Page size to request. Defaults to 50 (a sane middle ground between request count and per-response payload size for this endpoint). """ async for record in paginate_by_page( client, VOICE_CLIP_LIST_PATH, page_size=page_size, parse_total=_parse_voice_clip_total, parse_rows=_parse_voice_clip_rows, build_body=_build_voice_clip_body, user_id=user_id, ): yield _normalize_voice_clip(record, channel=channel)