yunke/yunke-server/app/scraper/voice_clip.py

"""Voice-clip (长语音) scraper.

Wraps ``POST /wechatweb/call/list`` — the endpoint behind the 云客 微信长语音 /
WA wavoicecalllist modals. Unlike :func:`app.scraper.wechat.scrape_wechat_voice_calls`
(which is scoped by ``departmentId`` + ``callSource``), this endpoint is scoped
**per employee**: the request body takes a required ``userId`` (employee id) and
returns that employee's long audio segments (the modal opens from a CallList
row keyed on the employee).

Endpoint contract (Playwright-confirmed 2026-08-07, see spec §2):

* Request body ``{pageNum, pageSize, userId}`` (JSON). ``startTime`` / ``endTime``
  ms-epoch string filters are OPTIONAL — omitted for full-history sync.
* Response envelope ``{code, message, success, data: {pageNum, pageSize, total,
  data: [...]}}`` — the SAME shape as ``queryWeChatVoiceList`` (rows under
  ``data.data``, declared total on ``data.total``), so the pagination helpers
  mirror the voice-call scraper.
* Row fields: ``id`` (bigint PK) / ``userId`` / ``userRealName`` / ``wxId`` /
  ``talker`` / ``status`` / ``isSend`` / ``companyCode`` / ``startTime`` /
  ``endTime`` (both ms-epoch STRINGS — normalized to int) / ``ossFileName``
  (recording URL) / ``mp3Duration`` (seconds) / ``durationFormat`` (HH:MM:SS).

Channel handling: the endpoint is SHARED by WeChat and WhatsApp employees
(distinct ``userId`` ranges, same path). The response does NOT carry the
channel, so :func:`scrape_voice_clips` takes a ``channel`` argument and stamps
it locally on every yielded record — the caller (sync orchestrator) resolves
it from the owning staff-account table (WechatAccount -> ``wechat``,
WhatsappAccount -> ``whatsapp``).

Reuses :func:`app.scraper.base.paginate_by_page` and emits snake_case dicts
plus a verbatim ``raw_json`` copy, mirroring :mod:`app.scraper.wechat`.
"""

from __future__ import annotations

from collections.abc import AsyncIterator
from typing import Any

from app.scraper.base import paginate_by_page
from app.scraper.parsers import coerce_int_or_none
from app.session.yunke_client import YunkeClient

__all__ = ["scrape_voice_clips"]

VOICE_CLIP_LIST_PATH = "/wechatweb/call/list"


def _parse_voice_clip_total(resp: Any) -> int | None:
    """Read the declared row count from the ``call/list`` envelope.

    Shape: ``data.total`` (an int) — the same location as
    ``queryWeChatVoiceList``. Returns ``None`` if the path is absent so the
    paginator falls back to the empty-page stop signal.
    """
    try:
        body = resp.json()
    except (AttributeError, ValueError):
        return None
    data = body.get("data") if isinstance(body, dict) else None
    if not isinstance(data, dict):
        return None
    total = data.get("total")
    return int(total) if isinstance(total, (int, float)) and not isinstance(
        total, bool
    ) else None


def _parse_voice_clip_rows(resp: Any) -> list[dict[str, Any]]:
    """Extract the voice-clip list from the ``call/list`` envelope.

    Shape: ``data.data`` (a list of records). Missing / non-list values
    collapse to an empty list so the paginator sees "empty page".
    """
    try:
        body = resp.json()
    except (AttributeError, ValueError):
        return []
    data = body.get("data") if isinstance(body, dict) else None
    if not isinstance(data, dict):
        return []
    rows = data.get("data")
    return rows if isinstance(rows, list) else []


def _build_voice_clip_body(
    page_num: int,
    page_size: int,
    *,
    user_id: str,
) -> dict[str, Any]:
    """Build the ``call/list`` request body for a given page.

    ``userId`` is the REQUIRED employee filter (the endpoint returns nothing
    without it). The optional ``startTime`` / ``endTime`` ms-epoch filters are
    omitted so the scrape pulls the employee's full history in one pass (the
    sync orchestrator wants all segments, not a time window).
    """
    return {
        "pageNum": page_num,
        "pageSize": page_size,
        "userId": user_id,
    }


def _normalize_voice_clip(
    record: dict[str, Any], *, channel: str
) -> dict[str, Any]:
    """Map one upstream ``call/list`` record to the ``VoiceClip`` column shape.

    Stamps the locally-resolved ``channel`` (``wechat`` / ``whatsapp``) — the
    API response does not carry it. ``startTime`` / ``endTime`` arrive as
    ms-epoch STRINGS (e.g. ``"1786072733378"``); they are coerced to ``int``
    via :func:`coerce_int_or_none` so the BigInteger columns land clean.
    ``mp3Duration`` (seconds, already int in the capture) is coerced to int
    defensively. ``isSend`` is carried through verbatim — the repo coerces it
    to bool. ``None``-safe on every field: a missing field becomes ``None``
    rather than raising, so a partial payload does not break the scrape.

    Preserves the original record under ``raw_json``. See spec §3 for the
    column-by-column mapping.
    """
    return {
        "voice_clip_id": record.get("id"),
        "user_id": record.get("userId"),
        "user_real_name": record.get("userRealName"),
        "wx_id": record.get("wxId"),
        "talker": record.get("talker"),
        "status": record.get("status"),
        "is_send": record.get("isSend"),
        "company_code": record.get("companyCode"),
        # ms-epoch strings -> int (the BigInteger column type).
        "start_time": coerce_int_or_none(record.get("startTime")),
        "end_time": coerce_int_or_none(record.get("endTime")),
        "oss_file_name": record.get("ossFileName"),
        "mp3_duration": coerce_int_or_none(record.get("mp3Duration")),
        "duration_format": record.get("durationFormat"),
        # Local label — resolved by the orchestrator from the staff-account
        # table that owns this user_id (NOT from the API response).
        "channel": channel,
        "raw_json": record,
    }


async def scrape_voice_clips(
    client: YunkeClient,
    user_id: str,
    *,
    channel: str = "wechat",
    page_size: int = 50,
) -> AsyncIterator[dict[str, Any]]:
    """Yield normalized voice-clip records for one employee from ``call/list``.

    Walks ``POST /wechatweb/call/list`` page by page (body
    ``{pageNum, pageSize, userId}``) until the upstream reports no more rows
    (empty ``data.data`` array — the primary stop signal), mapping each record
    to the ``VoiceClip`` column shape plus a verbatim ``raw_json`` copy.

    Parameters
    ----------
    client:
        Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
        handled by the client.
    user_id:
        Employee id (the REQUIRED ``userId`` request field). The sync
        orchestrator resolves this from the staff-account tables.
    channel:
        Local messaging-channel label (``"wechat"`` / ``"whatsapp"``) stamped
        on every yielded record. The endpoint is shared and does NOT carry the
        channel; the caller resolves it from the owning account table.
    page_size:
        Page size to request. Defaults to 50 (a sane middle ground between
        request count and per-response payload size for this endpoint).
    """
    async for record in paginate_by_page(
        client,
        VOICE_CLIP_LIST_PATH,
        page_size=page_size,
        parse_total=_parse_voice_clip_total,
        parse_rows=_parse_voice_clip_rows,
        build_body=_build_voice_clip_body,
        user_id=user_id,
    ):
        yield _normalize_voice_clip(record, channel=channel)