"""WhatsApp scrapers (M3). Endpoints covered: * :func:`scrape_whatsapp_accounts` — wraps ``POST /qqweb/WhatsAppUser/businessSalesList`` and yields normalized :class:`app.db.models.WhatsappAccount` dicts (plus the embedded ``userId`` / ``userName`` staff pair). Page-number pagination. * :func:`scrape_whatsapp_friends` — wraps ``POST /qqweb/WhatsAppUser/businessFriendList`` and yields normalized dicts ready to be upserted into :class:`app.db.models.Customer` with ``channel="whatsapp"``. Page-number pagination. * :func:`scrape_whatsapp_groups` — wraps the SAME ``businessFriendList`` endpoint with ``type="1"`` to select WA chat-groups (``"0"`` selects friends) and yields normalized :class:`app.db.models.ChatGroup` dicts with ``channel="whatsapp"``. The ``type`` group switch is UNVERIFIED (see the section docstring) — the structure is correct, only the param name may need a post-risk-control adjustment. * :func:`scrape_whatsapp_messages` — wraps ``POST /qqweb/WhatsAppUser/chatRecords`` and yields normalized chat-message dicts with ``channel="whatsapp"``. Backward-cursor pagination by timestamp (the same strategy as WeChat ``msg/chats``), but with a **JSON** request body (NOT form-encoded). All three reuse :func:`app.scraper.base.paginate_by_page` / :func:`paginate_by_cursor` and emit snake_case dicts + a verbatim ``raw_json`` copy, mirroring :mod:`app.scraper.wechat`. Envelope difference from WeChat list endpoints: rows sit under ``data.overviews`` (NOT ``data.data``) and the declared row count is ``data.totalCount`` (NOT ``data.total``); the page count is ``data.pageCount``. The same envelope is shared by ``businessSalesList`` and ``businessFriendList``. Auth is the same crm2 session cookie the WeChat scrapers use (Phase-0 §1 confirmed cookie-only auth suffices for every data endpoint). """ from __future__ import annotations from collections.abc import AsyncIterator from typing import Any from app.scraper.base import paginate_by_cursor, paginate_by_page from app.scraper.parsers import coerce_int_or_none, normalize_whatsapp_type, parse_datetime_to_ms from app.scraper.wechat import _normalize_chat_group from app.session.yunke_client import YunkeClient __all__ = [ "scrape_wa_voice_calls", "scrape_whatsapp_accounts", "scrape_whatsapp_friends", "scrape_whatsapp_groups", "scrape_whatsapp_messages", ] WHATSAPP_ACCOUNTS_PATH = "/qqweb/WhatsAppUser/businessSalesList" WHATSAPP_FRIENDS_PATH = "/qqweb/WhatsAppUser/businessFriendList" # WhatsApp list endpoints return wall-clock strings like "2026-08-01 16:39" # (no seconds) for lastUpdate/createTime — parse to ms epoch (naive UTC). # The voice-call endpoint reports ``startTime`` as ``YYYY-MM-DD HH:MM:SS`` # (with seconds); the second format below matches it so this list is reused # by the WA voice-call scraper without a private copy. _WA_DATETIME_FORMATS = ["%Y-%m-%d %H:%M", "%Y-%m-%d %H:%M:%S"] WHATSAPP_MESSAGES_PATH = "/qqweb/WhatsAppUser/chatRecords" # The unified yunke voice-call endpoint is shared across the WeChat and # WhatsApp channels — same path, same envelope, distinguished by the # ``callSource`` flag in the request body (``"0"`` WeChat / ``"4"`` WA). # ``scrape_wa_voice_calls`` below wraps the WA variant; the WeChat variant # lives in :mod:`app.scraper.wechat` (``scrape_wechat_voice_calls``). WA_VOICE_LIST_PATH = "/wechatweb/callRecord/queryWeChatVoiceList" # =========================================================================== # Account scraper (M3) — POST /qqweb/WhatsAppUser/businessSalesList # =========================================================================== def _parse_wa_account_total(resp: Any) -> int | None: """Read the declared row count from the ``businessSalesList`` envelope. Shape: ``data.totalCount`` (an int). The page count is ``data.pageCount`` but the paginator only consumes the row-count ceiling; the empty-page signal remains the primary stop. Returns ``None`` if the path is absent so the paginator falls back to the empty-page stop signal. """ try: body = resp.json() except (AttributeError, ValueError): return None data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return None total = data.get("totalCount") if isinstance(total, bool): return None return int(total) if isinstance(total, (int, float)) else None def _parse_wa_account_rows(resp: Any) -> list[dict[str, Any]]: """Extract the account list from the ``businessSalesList`` envelope. Shape: ``data.overviews`` (a list of records). Missing / non-list values collapse to an empty list so the paginator sees "empty page". """ try: body = resp.json() except (AttributeError, ValueError): return [] data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [] rows = data.get("overviews") return rows if isinstance(rows, list) else [] def _build_wa_account_body(page_num: int, page_size: int) -> dict[str, Any]: """Build the ``businessSalesList`` request body for a given page. The live capture carries no filter parameters — just the page slice. """ return {"pageNum": page_num, "pageSize": page_size} def _normalize_whatsapp_account(record: dict[str, Any]) -> dict[str, Any]: """Map one ``businessSalesList`` record to the ``WhatsappAccount`` shape. Carries the embedded staff pair (``user_id`` <- ``userId``, ``user_name`` <- ``userName``) so the persistence layer can upsert the owning :class:`app.db.models.Staff` row alongside the account. Adds ``raw_json`` (the verbatim upstream record). ``None``-safe: missing fields become ``None`` rather than raising, so a partial payload does not break the scrape. """ return { "uid": record.get("uid"), "user_id": record.get("userId"), "user_name": record.get("userName"), "nick_name": record.get("nickName"), "phone": record.get("phone"), "sale_phone": record.get("salePhone"), "sales_bind_phone": record.get("salesBindPhone"), "contacts_num": record.get("contactsNum"), "group_num": record.get("groupNum"), "talker_count": record.get("talkerCount"), "friend_add_num": record.get("friendAddNum"), "is_on_job": record.get("isOnJob"), "last_update": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS), "version": record.get("version"), "raw_json": record, } async def scrape_whatsapp_accounts( client: YunkeClient, *, page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp staff-account records from ``businessSalesList``. Walks the endpoint page by page until the upstream reports no more rows (empty ``data.overviews`` array — the primary stop signal), mapping each record to the ``WhatsappAccount`` column shape plus a verbatim ``raw_json`` copy. The envelope differs from WeChat ``wxUser/list``: rows sit under ``data.overviews`` (not ``data.data``) and the declared total is ``data.totalCount`` (not ``data.total``). The page count is ``data.pageCount``. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. page_size: Page size to request. Defaults to 50 (matches the observed live page size for this endpoint). """ async for record in paginate_by_page( client, WHATSAPP_ACCOUNTS_PATH, page_size=page_size, parse_total=_parse_wa_account_total, parse_rows=_parse_wa_account_rows, build_body=_build_wa_account_body, ): yield _normalize_whatsapp_account(record) # =========================================================================== # Friend scraper (M3) — POST /qqweb/WhatsAppUser/businessFriendList # =========================================================================== def _parse_wa_friend_total(resp: Any) -> int | None: """Read the declared row count from the ``businessFriendList`` envelope. Shape: ``data.totalCount`` (live total is 11292 per phase0-findings.md §5). """ try: body = resp.json() except (AttributeError, ValueError): return None data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return None total = data.get("totalCount") if isinstance(total, bool): return None return int(total) if isinstance(total, (int, float)) else None def _parse_wa_friend_rows(resp: Any) -> list[dict[str, Any]]: """Extract the friend list from the ``businessFriendList`` envelope. Shape: ``data.overviews`` (same envelope as ``businessSalesList``). """ try: body = resp.json() except (AttributeError, ValueError): return [] data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [] rows = data.get("overviews") return rows if isinstance(rows, list) else [] def _build_wa_friend_body( page_num: int, page_size: int, *, department_id: str = "" ) -> dict[str, Any]: """Build the ``businessFriendList`` request body for a given page. Live capture: ``{"searchInfo":"","friendSearchInfo":"","page":1, "pageSize":10,"departmentId":"","type":"0"}``. Note this endpoint uses ``page`` (NOT ``pageNum`` like businessSalesList) and REQUIRES ``departmentId`` (the company-root dept id) + ``type`` or it returns empty. """ body: dict[str, Any] = { "searchInfo": "", "friendSearchInfo": "", "page": page_num, "pageSize": page_size, "type": "0", } if department_id: body["departmentId"] = department_id return body def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]: """Map one ``businessFriendList`` record to the ``Customer`` column shape. Adds ``channel="whatsapp"``. Field mapping per phase0-findings.md §5: * ``friend_id`` <- ``friendUid`` (PK, ``@s.whatsapp.net``) * ``owner_account_id`` <- ``uid`` (the owning staff WA account) * ``user_id`` <- ``userId`` (owning staff) * ``sale_name`` <- ``salesNickName`` (owning staff sale name) * ``friend_nickname`` <- ``nickName`` * ``phone`` <- ``phone`` * ``message_count`` <- ``messageCount`` * ``last_chat_time`` <- ``lastUpdate`` (ms epoch of last activity) Preserves the verbatim record under ``raw_json`` (so the owner-staff fields ``userName`` / ``salePhone`` / ``salesBindPhone`` / ``createTime`` and the per-contact counts ``groupNum`` / ``contactsNum`` / ``talkerCount`` / ``friendAddNum`` / ``sensitiveWordNum`` survive end-to-end even though they have no direct ``Customer`` column). ``None``-safe on every field. ``add_time`` is deliberately ``None``: the WhatsApp ``businessFriendList`` payload carries no customer-side add-time field (the ``createTime`` field on this record is the **owner-staff** account's createTime, not the contact's — confirmed via smoke.db raw_json dump). Mapping it to ``createTime`` would be semantically wrong; the column stays NULL for WA contacts (the wechat channel carries the truthful ``addTime``). """ return { "friend_id": record.get("friendUid"), "channel": "whatsapp", "owner_account_id": record.get("uid"), "friend_nickname": record.get("nickName"), "phone": record.get("phone"), "sale_name": record.get("salesNickName"), "user_id": record.get("userId"), "message_count": record.get("messageCount"), "last_chat_time": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS), # No customer-side add-time on this payload (see docstring); left NULL. "add_time": None, "version": record.get("version"), "raw_json": record, } async def scrape_whatsapp_friends( client: YunkeClient, *, department_id: str = "", page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp contact records from ``businessFriendList``. Walks the endpoint page by page until the upstream reports no more rows (empty ``data.overviews`` array — the primary stop signal), mapping each record to the ``Customer`` column shape with ``channel="whatsapp"`` plus a verbatim ``raw_json`` copy. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. page_size: Page size to request. Defaults to 50. """ async for record in paginate_by_page( client, WHATSAPP_FRIENDS_PATH, page_size=page_size, parse_total=_parse_wa_friend_total, parse_rows=_parse_wa_friend_rows, build_body=_build_wa_friend_body, department_id=department_id, ): yield _normalize_whatsapp_friend(record) # =========================================================================== # Chat-group scraper — POST /qqweb/WhatsAppUser/businessFriendList # with ``type="1"`` (groups). ``type="0"`` selects friends (see above). # # UNVERIFIED (云客 risk-control blocking live API calls as of 2026-08-11): # the group-type body parameter is inferred, not captured. The WA friends # body sends ``type="0"`` and the WA CRM SPA exposes two parallel routes — # ``friendlist/0`` (friends) and ``friendlist/1`` (groups) — so by symmetry # ``type="1"`` is the most likely group selector. If the post-risk-control # smoke returns 0 rows, try alternative param names BEFORE concluding the # endpoint is wrong: ``friendType: "1"``, ``friendTypeYunke: "2"`` (the # WeChat switch value), or a sibling path entirely. The field mapping and # pagination structure here are correct regardless; only the switch param # name may need adjustment. # =========================================================================== def _build_wa_group_body( page_num: int, page_size: int, *, department_id: str = "" ) -> dict[str, Any]: """Build the ``businessFriendList`` request body for chat-groups (type=1). Identical to :func:`_build_wa_friend_body` except for ``type``: ``"1"`` selects groups (``"0"`` selects friends). UNVERIFIED — see the section docstring above for the post-risk-control verification plan. ``departmentId`` is omitted when blank so a test/fixture call without one still forms a valid body (the live endpoint requires it for non-empty results, same as the friends variant). """ body: dict[str, Any] = { "searchInfo": "", "friendSearchInfo": "", "page": page_num, "pageSize": page_size, # UNVERIFIED group-type switch — see section docstring. "type": "1", } if department_id: body["departmentId"] = department_id return body def _normalize_whatsapp_group(record: dict[str, Any]) -> dict[str, Any]: """Map one WA ``businessFriendList`` group record to the ``ChatGroup`` shape. Reuses :func:`app.scraper.wechat._normalize_chat_group` for the full field mapping — the WA endpoint returns the SAME field names the WeChat ``friend_list/es`` endpoint does for the group-type switch (``friendWxId`` is the ``@chatroom`` id, ``friendWxNickname`` is the group name, ``friendNum`` is the member count, etc.) — then overrides ``channel`` to ``"whatsapp"`` so WA groups are distinguishable from WeChat groups in the shared :class:`app.db.models.ChatGroup` table. Member ids are NOT surfaced by this endpoint (only the ``friendNum`` count); ``member_ids`` / ``member_ids_list`` are emitted as ``None`` / ``[]`` by the shared WeChat normalizer and left untouched here. """ out = _normalize_chat_group(record) out["channel"] = "whatsapp" return out async def scrape_whatsapp_groups( client: YunkeClient, *, department_id: str = "", page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp chat-group records from ``businessFriendList``. Mirrors :func:`scrape_whatsapp_friends` but sends ``type="1"`` in the body to select groups (the WA CRM SPA's ``friendlist/1`` route). Shares the friends endpoint's envelope (rows under ``data.overviews``, declared total under ``data.totalCount``), so the same ``_parse_wa_friend_total`` / ``_parse_wa_friend_rows`` helpers are reused. Each record is mapped to the ``ChatGroup`` column shape with ``channel="whatsapp"`` (via :func:`_normalize_whatsapp_group`, which reuses the WeChat group normalizer for the field mapping) plus a verbatim ``raw_json`` copy. Member ids are NOT available — ``member_ids`` / ``member_ids_list`` are ``None`` / ``[]``. .. note:: UNVERIFIED (2026-08-11, risk-control): the ``type="1"`` group switch is inferred from the WA SPA's ``friendlist/0`` / ``friendlist/1`` routes, not a live capture. If the post-risk-control smoke returns 0 rows, try ``friendType`` / ``friendTypeYunke`` before concluding the endpoint is wrong. See the section docstring above. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. department_id: Company-root dept id (the ``departmentId`` body field). Required by the live endpoint for non-empty results (same as friends); resolved by the orchestrator when omitted. page_size: Page size to request. Defaults to 50. """ async for record in paginate_by_page( client, WHATSAPP_FRIENDS_PATH, page_size=page_size, parse_total=_parse_wa_friend_total, parse_rows=_parse_wa_friend_rows, build_body=_build_wa_group_body, department_id=department_id, ): yield _normalize_whatsapp_group(record) # =========================================================================== # Message scraper (M3) — POST /qqweb/WhatsAppUser/chatRecords # =========================================================================== async def _fetch_wa_msg_page( client: YunkeClient, *, uid: str, friend_uid: str, start: int, end_cursor: int, ) -> tuple[list[dict[str, Any]], int, bool]: """Fetch one backward page of ``chatRecords``. Returns the ``(rows, next_cursor, has_more)`` triple expected by :func:`paginate_by_cursor`. Unlike WeChat ``msg/chats`` this endpoint takes a **JSON** body (content-type ``application/json``), not form-encoded. ``next_cursor`` is ``data.end - 1`` so the next call's window is ``(end-1, ...]`` — i.e. strictly older than the oldest message in this batch; ``has_more`` is ``data.hasLast``. """ resp = await client.post( WHATSAPP_MESSAGES_PATH, json={ "uid": uid, "friendUid": friend_uid, "start": start, "end": end_cursor, }, ) try: body = resp.json() except (AttributeError, ValueError): # Malformed envelope — treat as empty + no-more so the paginator stops. return [], end_cursor, False data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [], end_cursor, False rows = data.get("messages") if not isinstance(rows, list): rows = [] end = data.get("end") next_cursor = ( int(end) - 1 if isinstance(end, (int, float)) and not isinstance(end, bool) else end_cursor ) has_more = bool(data.get("hasLast")) return rows, next_cursor, has_more def _normalize_whatsapp_message(record: dict[str, Any]) -> dict[str, Any]: """Map one ``chatRecords`` record to the ``Message`` column shape. Adds ``channel="whatsapp"`` and preserves the original record under ``raw_json``. Per design §4.3 the message type is carried as TWO fields: * ``type`` — the normalized enum label (``"text"`` / ``"image"`` / ``"unknown"`` / ...) via :func:`normalize_whatsapp_type`. * ``type_raw`` — the original upstream type code (``type`` falling back to ``msgType``), coerced to ``int | None`` so the raw code survives end-to-end without round-tripping through ``raw_json``. ``mediaName`` (a deferred-download filename, NOT a URL — see phase0 §6) is carried on ``file_url`` so the media layer can route on ``channel`` / ``type``; this mirrors the WeChat ``type=3`` image-legacy pattern where an opaque download key (not a URL) is stored on ``file_url``. ``raw_json`` preserves it verbatim either way. ``quotedId`` / ``quotedMessage`` collapse into ``refer_msg`` as a small dict when either is present. No ``room_id`` for WA 1:1 chats. ``None``-safe on every field. """ # ``type`` is the documented field; fall back to ``msgType`` (some captures # populate one, some the other — same disambiguation strategy as WeChat). raw_type = record.get("type") if raw_type is None: raw_type = record.get("msgType") # ``createTime`` is the canonical timestamp; fall back to ``dateTime``. timestamp = record.get("createTime") if timestamp is None: timestamp = record.get("dateTime") # Quoted-message reference: preserve both id + content when either present # (a bare None/None pair would just be noise on the JSON column). quoted_id = record.get("quotedId") quoted_message = record.get("quotedMessage") refer_msg = ( {"quotedId": quoted_id, "quotedMessage": quoted_message} if quoted_id is not None or quoted_message is not None else None ) return { "msg_id": record.get("keyId"), "channel": "whatsapp", "owner_account_id": record.get("uid"), "talker_id": record.get("friendUid"), "is_send": record.get("isSend"), "type": normalize_whatsapp_type(raw_type), "type_raw": coerce_int_or_none(raw_type), "text": record.get("content"), # file 常带直连 OSS URL(1:1 图片实测 93% 覆盖,2026-08-20),优先于 # mediaName 不透明名 "file_url": record.get("file") or record.get("mediaName"), "refer_msg": refer_msg, "sender_name": record.get("nickName"), "timestamp": timestamp, "raw_json": record, } def _is_wa_divider(record: dict[str, Any]) -> bool: """True iff this record is a date-divider (raw type code ``-2``). These are not real messages — the upstream inserts them as visual date bars in the chat UI — and must never be yielded or persisted. Per the M3 spec we drop a record when EITHER ``type`` or ``msgType`` is ``-2``; all other unknown codes pass through. The check is on the RAW code (not the normalized label) so the drop decision is robust against any future change to :func:`normalize_whatsapp_type`'s label vocabulary. """ return coerce_int_or_none(record.get("type")) == -2 or ( coerce_int_or_none(record.get("msgType")) == -2 ) async def scrape_whatsapp_messages( client: YunkeClient, uid: str, friend_uid: str, start: int, end: int, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp chat messages, paging backward by timestamp. Wraps ``POST /qqweb/WhatsAppUser/chatRecords`` with a **JSON** body (``{uid, friendUid, start, end}`` — NOT form-encoded like WeChat ``msg/chats``). The endpoint pages BACKWARD: each response carries ``data.end`` (oldest timestamp in this batch) and ``data.hasLast`` (older messages exist?). The scraper advances the cursor to ``end - 1`` and repeats until ``hasLast`` is false, then stops. Date dividers (``type == -2``) are dropped — they are UI chrome, not real messages. All other type codes pass through (mapped to ``"unknown"`` when not in our small known-table) so no real message is silently lost. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. uid: Owning staff WhatsApp account id (``uid`` JSON field, of the form ``@s.whatsapp.net``) — maps to ``owner_account_id`` on each yielded record. friend_uid: Customer WhatsApp id (``friendUid`` JSON field) — selects the 1:1 conversation to pull. start, end: Time window in ms epoch. ``end`` is also the initial backward cursor; the first request queries the ``(start, end]`` window and the cursor walks toward ``start`` from there. """ async for record in paginate_by_cursor( lambda end_cursor: _fetch_wa_msg_page( client, uid=uid, friend_uid=friend_uid, start=start, end_cursor=end_cursor, ), initial_cursor=end, ): if _is_wa_divider(record): continue yield _normalize_whatsapp_message(record) # =========================================================================== # Voice-call scraper (Plan 4a Task 1) — POST /wechatweb/callRecord/queryWeChatVoiceList # with ``callSource=4`` (WhatsApp). # # Reuses the unified yunke voice interface also used by the WeChat voice # scraper (``scrape_wechat_voice_calls``): same path, same flat envelope # (rows under ``data.data``, declared total under ``data.total``, no # ``basicResult`` wrapper). The only wire-level difference is the request # body flag ``callSource="4"`` (WA) vs ``"0"`` (WeChat). Output records are # tagged ``channel="wa_voice_call"`` so they share the ``PhoneCall`` table # with the WeChat channel without colliding. # =========================================================================== def _parse_wa_voice_total(resp: Any) -> int | None: """Read the declared row count from the ``queryWeChatVoiceList`` envelope. Shape: ``data.total`` (an int) — identical to the WeChat voice envelope (same endpoint). Returns ``None`` if the path is absent so the paginator falls back to the empty-page stop signal. (Live WA ``total`` is 143 per phase0-findings.md §4.) """ try: body = resp.json() except (AttributeError, ValueError): return None data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return None total = data.get("total") return int(total) if isinstance(total, (int, float)) and not isinstance( total, bool ) else None def _parse_wa_voice_rows(resp: Any) -> list[dict[str, Any]]: """Extract the WA voice-call list from the ``queryWeChatVoiceList`` envelope. Shape: ``data.data`` (a list of records) — identical to the WeChat voice envelope. Missing / non-list values collapse to an empty list so the paginator sees "empty page". """ try: body = resp.json() except (AttributeError, ValueError): return [] data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [] rows = data.get("data") return rows if isinstance(rows, list) else [] def _build_wa_voice_body( page_num: int, page_size: int, *, department_id: str = "", start: str = "2000-01-01 00:00:00", end: str = "2099-12-31 23:59:59", ) -> dict[str, Any]: """Build the WA ``queryWeChatVoiceList`` request body for a given page. Live diagnosis (2026-08-07 browser capture) confirmed the endpoint REQUIRES ``callSource`` + ``departmentId`` + ``isRoom`` + ``startTime`` + ``endTime`` — omit any one and the HTTP layer returns 200 with an EMPTY ``data.data`` array (0 rows). The WA variant uses ``callSource="4"``; a wide time window (2000-01-01 .. 2099-12-31) pulls the full history in one scrape (live total=608). ``userId`` / ``sort`` / ``ids`` may stay empty. ``departmentId`` is omitted when blank so a test/fixture call without one still forms a valid body. """ body: dict[str, Any] = { "pageNum": page_num, "pageSize": page_size, "callSource": "4", "isRoom": "0", "startTime": start, "endTime": end, "sort": "", "ids": [], "userId": "", } if department_id: body["departmentId"] = department_id return body def _normalize_wa_voice_call(record: dict[str, Any]) -> dict[str, Any]: """Map one WA ``queryWeChatVoiceList`` record to the ``PhoneCall`` shape. Adds ``channel="wa_voice_call"`` and preserves the original record under ``raw_json``. Field mapping is identical to the WeChat voice normalizer (``_normalize_voice_call`` in :mod:`app.scraper.wechat`) — the endpoint is shared, so the record schema is too. The two intentional differences: * ``channel`` is ``"wa_voice_call"`` (not ``"voice_call"``) so the WA rows are distinguishable from WeChat rows in the shared ``PhoneCall`` table. * WA-specific fields with no matching ``PhoneCall`` column survive only via ``raw_json``: ``weChatId`` (the owning WA account id, e.g. ``@s.whatsapp.net``), ``callFileId``, ``nextId``, ``isRoom`` and ``from`` (the WA source flag). Mapping (same as the WeChat voice normalizer, repeated here so this scraper is self-documenting): * ``id`` -> ``call_id`` (PK) * ``startTime`` -> ``start_call_time`` (``YYYY-MM-DD HH:MM:SS`` -> ms epoch via :func:`parse_datetime_to_ms` + ``_WA_DATETIME_FORMATS``) * ``duration`` -> ``call_duration`` (string seconds coerced to int) * ``callStatus`` -> ``call_status`` (1=connected w/ recording, 2=no answer) * ``isSend`` -> ``incoming_call`` with INVERTED polarity (``0=incoming -> 1`` / ``1=outgoing -> 0``; ``None`` preserved when ``isSend`` is missing/non-numeric). * ``userId`` -> ``user_id`` (owning staff) * ``userName`` -> ``user_id_name`` * ``talker`` -> ``call_number`` (the customer WA id) * ``talkerNickName`` -> ``plan_customer_name`` * ``ossFileName`` -> ``record_file`` (the recording URL; may be null) * ``version`` -> ``version`` (carried verbatim) Non-applicable ``PhoneCall`` columns (``ring_time`` / ``sim_phone`` / ``sim_number`` / ``imei`` / ``is_upload`` / ``download_count`` / ``ai_asr`` / ``qc_status`` / ``department_id``) are left ``None`` — the WA voice payload carries none of them. ``None``-safe on every mapped field. """ is_send = coerce_int_or_none(record.get("isSend")) # Flip isSend (0=incoming/1=outgoing) to incoming_call (1=incoming/0=outgoing). incoming_call = None if is_send is None else 1 - is_send return { "call_id": record.get("id"), "channel": "wa_voice_call", "start_call_time": parse_datetime_to_ms( record.get("startTime"), _WA_DATETIME_FORMATS ), "call_duration": coerce_int_or_none(record.get("duration")), "call_status": coerce_int_or_none(record.get("callStatus")), "incoming_call": incoming_call, "user_id": record.get("userId"), "user_id_name": record.get("userName"), "call_number": record.get("talker"), "plan_customer_name": record.get("talkerNickName"), "record_file": record.get("ossFileName"), "version": record.get("version"), "raw_json": record, } async def scrape_wa_voice_calls( client: YunkeClient, *, department_id: str = "", page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp voice-call records from ``queryWeChatVoiceList``. Wraps ``POST /wechatweb/callRecord/queryWeChatVoiceList`` with ``callSource=4`` — the unified yunke voice interface's WA variant (the WeChat variant, ``callSource=0``, lives in :mod:`app.scraper.wechat`). Walks page by page until the upstream reports no more rows (empty ``data.data`` array — the primary stop signal), mapping each record to the ``PhoneCall`` column shape with ``channel="wa_voice_call"`` plus a verbatim ``raw_json`` copy. The endpoint REQUIRES ``callSource`` + ``departmentId`` + ``isRoom`` + a ``startTime``/``endTime`` window — without them it returns 200 + an empty ``data.data`` (0 rows). The orchestrator resolves the company-root ``department_id`` from the mirrored ``Department`` table (see :func:`app.sync.orchestrator.sync_wa_voice_calls`); the default wide time window ``2000-01-01 .. 2099-12-31`` pulls the full history (live total=608) and is not exposed as a parameter. The envelope is identical to the WeChat voice endpoint: rows sit under ``data.data``, declared total under ``data.total``, pagination fields under ``data.pageNum`` / ``data.pageSize``. The WA-specific fields (``weChatId`` / ``callFileId`` / ``nextId`` / ``isRoom`` / ``from``) have no ``PhoneCall`` column and survive only via ``raw_json``. Persistence reuses :func:`app.storage.repos.upsert_voice_call` (Task 2 wires the orchestration); the ``channel`` value distinguishes WA rows from WeChat rows in the shared table. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. department_id: Company-root department id (the ``departmentId`` form field). Required by the live endpoint for non-empty results; resolved by the orchestrator when omitted (empty skips the field in the body). page_size: Page size to request. Defaults to 50 (matches the WeChat voice scraper; a sane middle ground between request count and per-response payload size for this endpoint). """ async for record in paginate_by_page( client, WA_VOICE_LIST_PATH, page_size=page_size, parse_total=_parse_wa_voice_total, parse_rows=_parse_wa_voice_rows, build_body=_build_wa_voice_body, department_id=department_id, ): yield _normalize_wa_voice_call(record)