"""WhatsApp scrapers (M3). Endpoints covered: * :func:`scrape_whatsapp_accounts` — wraps ``POST /qqweb/WhatsAppUser/businessSalesList`` and yields normalized :class:`app.db.models.WhatsappAccount` dicts (plus the embedded ``userId`` / ``userName`` staff pair). Page-number pagination. * :func:`scrape_whatsapp_friends` — wraps ``POST /qqweb/WhatsAppUser/businessFriendList`` and yields normalized dicts ready to be upserted into :class:`app.db.models.Customer` with ``channel="whatsapp"``. Page-number pagination. * :func:`scrape_whatsapp_messages` — wraps ``POST /qqweb/WhatsAppUser/chatRecords`` and yields normalized chat-message dicts with ``channel="whatsapp"``. Backward-cursor pagination by timestamp (the same strategy as WeChat ``msg/chats``), but with a **JSON** request body (NOT form-encoded). All three reuse :func:`app.scraper.base.paginate_by_page` / :func:`paginate_by_cursor` and emit snake_case dicts + a verbatim ``raw_json`` copy, mirroring :mod:`app.scraper.wechat`. Envelope difference from WeChat list endpoints: rows sit under ``data.overviews`` (NOT ``data.data``) and the declared row count is ``data.totalCount`` (NOT ``data.total``); the page count is ``data.pageCount``. The same envelope is shared by ``businessSalesList`` and ``businessFriendList``. Auth is the same crm2 session cookie the WeChat scrapers use (Phase-0 §1 confirmed cookie-only auth suffices for every data endpoint). """ from __future__ import annotations from collections.abc import AsyncIterator from typing import Any from app.scraper.base import paginate_by_cursor, paginate_by_page from app.scraper.parsers import coerce_int_or_none, normalize_whatsapp_type, parse_datetime_to_ms from app.session.yunke_client import YunkeClient __all__ = [ "scrape_whatsapp_accounts", "scrape_whatsapp_friends", "scrape_whatsapp_messages", ] WHATSAPP_ACCOUNTS_PATH = "/qqweb/WhatsAppUser/businessSalesList" WHATSAPP_FRIENDS_PATH = "/qqweb/WhatsAppUser/businessFriendList" # WhatsApp list endpoints return wall-clock strings like "2026-08-01 16:39" # (no seconds) for lastUpdate/createTime — parse to ms epoch (naive UTC). _WA_DATETIME_FORMATS = ["%Y-%m-%d %H:%M", "%Y-%m-%d %H:%M:%S"] WHATSAPP_MESSAGES_PATH = "/qqweb/WhatsAppUser/chatRecords" # =========================================================================== # Account scraper (M3) — POST /qqweb/WhatsAppUser/businessSalesList # =========================================================================== def _parse_wa_account_total(resp: Any) -> int | None: """Read the declared row count from the ``businessSalesList`` envelope. Shape: ``data.totalCount`` (an int). The page count is ``data.pageCount`` but the paginator only consumes the row-count ceiling; the empty-page signal remains the primary stop. Returns ``None`` if the path is absent so the paginator falls back to the empty-page stop signal. """ try: body = resp.json() except (AttributeError, ValueError): return None data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return None total = data.get("totalCount") if isinstance(total, bool): return None return int(total) if isinstance(total, (int, float)) else None def _parse_wa_account_rows(resp: Any) -> list[dict[str, Any]]: """Extract the account list from the ``businessSalesList`` envelope. Shape: ``data.overviews`` (a list of records). Missing / non-list values collapse to an empty list so the paginator sees "empty page". """ try: body = resp.json() except (AttributeError, ValueError): return [] data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [] rows = data.get("overviews") return rows if isinstance(rows, list) else [] def _build_wa_account_body(page_num: int, page_size: int) -> dict[str, Any]: """Build the ``businessSalesList`` request body for a given page. The live capture carries no filter parameters — just the page slice. """ return {"pageNum": page_num, "pageSize": page_size} def _normalize_whatsapp_account(record: dict[str, Any]) -> dict[str, Any]: """Map one ``businessSalesList`` record to the ``WhatsappAccount`` shape. Carries the embedded staff pair (``user_id`` <- ``userId``, ``user_name`` <- ``userName``) so the persistence layer can upsert the owning :class:`app.db.models.Staff` row alongside the account. Adds ``raw_json`` (the verbatim upstream record). ``None``-safe: missing fields become ``None`` rather than raising, so a partial payload does not break the scrape. """ return { "uid": record.get("uid"), "user_id": record.get("userId"), "user_name": record.get("userName"), "nick_name": record.get("nickName"), "phone": record.get("phone"), "sale_phone": record.get("salePhone"), "sales_bind_phone": record.get("salesBindPhone"), "contacts_num": record.get("contactsNum"), "group_num": record.get("groupNum"), "talker_count": record.get("talkerCount"), "friend_add_num": record.get("friendAddNum"), "is_on_job": record.get("isOnJob"), "last_update": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS), "version": record.get("version"), "raw_json": record, } async def scrape_whatsapp_accounts( client: YunkeClient, *, page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp staff-account records from ``businessSalesList``. Walks the endpoint page by page until the upstream reports no more rows (empty ``data.overviews`` array — the primary stop signal), mapping each record to the ``WhatsappAccount`` column shape plus a verbatim ``raw_json`` copy. The envelope differs from WeChat ``wxUser/list``: rows sit under ``data.overviews`` (not ``data.data``) and the declared total is ``data.totalCount`` (not ``data.total``). The page count is ``data.pageCount``. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. page_size: Page size to request. Defaults to 50 (matches the observed live page size for this endpoint). """ async for record in paginate_by_page( client, WHATSAPP_ACCOUNTS_PATH, page_size=page_size, parse_total=_parse_wa_account_total, parse_rows=_parse_wa_account_rows, build_body=_build_wa_account_body, ): yield _normalize_whatsapp_account(record) # =========================================================================== # Friend scraper (M3) — POST /qqweb/WhatsAppUser/businessFriendList # =========================================================================== def _parse_wa_friend_total(resp: Any) -> int | None: """Read the declared row count from the ``businessFriendList`` envelope. Shape: ``data.totalCount`` (live total is 11292 per phase0-findings.md §5). """ try: body = resp.json() except (AttributeError, ValueError): return None data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return None total = data.get("totalCount") if isinstance(total, bool): return None return int(total) if isinstance(total, (int, float)) else None def _parse_wa_friend_rows(resp: Any) -> list[dict[str, Any]]: """Extract the friend list from the ``businessFriendList`` envelope. Shape: ``data.overviews`` (same envelope as ``businessSalesList``). """ try: body = resp.json() except (AttributeError, ValueError): return [] data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [] rows = data.get("overviews") return rows if isinstance(rows, list) else [] def _build_wa_friend_body( page_num: int, page_size: int, *, department_id: str = "" ) -> dict[str, Any]: """Build the ``businessFriendList`` request body for a given page. Live capture: ``{"searchInfo":"","friendSearchInfo":"","page":1, "pageSize":10,"departmentId":"","type":"0"}``. Note this endpoint uses ``page`` (NOT ``pageNum`` like businessSalesList) and REQUIRES ``departmentId`` (the company-root dept id) + ``type`` or it returns empty. """ body: dict[str, Any] = { "searchInfo": "", "friendSearchInfo": "", "page": page_num, "pageSize": page_size, "type": "0", } if department_id: body["departmentId"] = department_id return body def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]: """Map one ``businessFriendList`` record to the ``Customer`` column shape. Adds ``channel="whatsapp"``. Field mapping per phase0-findings.md §5: * ``friend_id`` <- ``friendUid`` (PK, ``@s.whatsapp.net``) * ``owner_account_id`` <- ``uid`` (the owning staff WA account) * ``user_id`` <- ``userId`` (owning staff) * ``sale_name`` <- ``salesNickName`` (owning staff sale name) * ``friend_nickname`` <- ``nickName`` * ``phone`` <- ``phone`` * ``message_count`` <- ``messageCount`` * ``last_chat_time`` <- ``lastUpdate`` (ms epoch of last activity) Preserves the verbatim record under ``raw_json`` (so the owner-staff fields ``userName`` / ``salePhone`` / ``salesBindPhone`` / ``createTime`` and the per-contact counts ``groupNum`` / ``contactsNum`` / ``talkerCount`` / ``friendAddNum`` / ``sensitiveWordNum`` survive end-to-end even though they have no direct ``Customer`` column). ``None``-safe on every field. ``add_time`` is deliberately ``None``: the WhatsApp ``businessFriendList`` payload carries no customer-side add-time field (the ``createTime`` field on this record is the **owner-staff** account's createTime, not the contact's — confirmed via smoke.db raw_json dump). Mapping it to ``createTime`` would be semantically wrong; the column stays NULL for WA contacts (the wechat channel carries the truthful ``addTime``). """ return { "friend_id": record.get("friendUid"), "channel": "whatsapp", "owner_account_id": record.get("uid"), "friend_nickname": record.get("nickName"), "phone": record.get("phone"), "sale_name": record.get("salesNickName"), "user_id": record.get("userId"), "message_count": record.get("messageCount"), "last_chat_time": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS), # No customer-side add-time on this payload (see docstring); left NULL. "add_time": None, "version": record.get("version"), "raw_json": record, } async def scrape_whatsapp_friends( client: YunkeClient, *, department_id: str = "", page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp contact records from ``businessFriendList``. Walks the endpoint page by page until the upstream reports no more rows (empty ``data.overviews`` array — the primary stop signal), mapping each record to the ``Customer`` column shape with ``channel="whatsapp"`` plus a verbatim ``raw_json`` copy. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. page_size: Page size to request. Defaults to 50. """ async for record in paginate_by_page( client, WHATSAPP_FRIENDS_PATH, page_size=page_size, parse_total=_parse_wa_friend_total, parse_rows=_parse_wa_friend_rows, build_body=_build_wa_friend_body, department_id=department_id, ): yield _normalize_whatsapp_friend(record) # =========================================================================== # Message scraper (M3) — POST /qqweb/WhatsAppUser/chatRecords # =========================================================================== async def _fetch_wa_msg_page( client: YunkeClient, *, uid: str, friend_uid: str, start: int, end_cursor: int, ) -> tuple[list[dict[str, Any]], int, bool]: """Fetch one backward page of ``chatRecords``. Returns the ``(rows, next_cursor, has_more)`` triple expected by :func:`paginate_by_cursor`. Unlike WeChat ``msg/chats`` this endpoint takes a **JSON** body (content-type ``application/json``), not form-encoded. ``next_cursor`` is ``data.end - 1`` so the next call's window is ``(end-1, ...]`` — i.e. strictly older than the oldest message in this batch; ``has_more`` is ``data.hasLast``. """ resp = await client.post( WHATSAPP_MESSAGES_PATH, json={ "uid": uid, "friendUid": friend_uid, "start": start, "end": end_cursor, }, ) try: body = resp.json() except (AttributeError, ValueError): # Malformed envelope — treat as empty + no-more so the paginator stops. return [], end_cursor, False data = body.get("data") if isinstance(body, dict) else None if not isinstance(data, dict): return [], end_cursor, False rows = data.get("messages") if not isinstance(rows, list): rows = [] end = data.get("end") next_cursor = ( int(end) - 1 if isinstance(end, (int, float)) and not isinstance(end, bool) else end_cursor ) has_more = bool(data.get("hasLast")) return rows, next_cursor, has_more def _normalize_whatsapp_message(record: dict[str, Any]) -> dict[str, Any]: """Map one ``chatRecords`` record to the ``Message`` column shape. Adds ``channel="whatsapp"`` and preserves the original record under ``raw_json``. Per design §4.3 the message type is carried as TWO fields: * ``type`` — the normalized enum label (``"text"`` / ``"image"`` / ``"unknown"`` / ...) via :func:`normalize_whatsapp_type`. * ``type_raw`` — the original upstream type code (``type`` falling back to ``msgType``), coerced to ``int | None`` so the raw code survives end-to-end without round-tripping through ``raw_json``. ``mediaName`` (a deferred-download filename, NOT a URL — see phase0 §6) is carried on ``file_url`` so the media layer can route on ``channel`` / ``type``; this mirrors the WeChat ``type=3`` image-legacy pattern where an opaque download key (not a URL) is stored on ``file_url``. ``raw_json`` preserves it verbatim either way. ``quotedId`` / ``quotedMessage`` collapse into ``refer_msg`` as a small dict when either is present. No ``room_id`` for WA 1:1 chats. ``None``-safe on every field. """ # ``type`` is the documented field; fall back to ``msgType`` (some captures # populate one, some the other — same disambiguation strategy as WeChat). raw_type = record.get("type") if raw_type is None: raw_type = record.get("msgType") # ``createTime`` is the canonical timestamp; fall back to ``dateTime``. timestamp = record.get("createTime") if timestamp is None: timestamp = record.get("dateTime") # Quoted-message reference: preserve both id + content when either present # (a bare None/None pair would just be noise on the JSON column). quoted_id = record.get("quotedId") quoted_message = record.get("quotedMessage") refer_msg = ( {"quotedId": quoted_id, "quotedMessage": quoted_message} if quoted_id is not None or quoted_message is not None else None ) return { "msg_id": record.get("keyId"), "channel": "whatsapp", "owner_account_id": record.get("uid"), "talker_id": record.get("friendUid"), "is_send": record.get("isSend"), "type": normalize_whatsapp_type(raw_type), "type_raw": coerce_int_or_none(raw_type), "text": record.get("content"), "file_url": record.get("mediaName"), "refer_msg": refer_msg, "sender_name": record.get("nickName"), "timestamp": timestamp, "raw_json": record, } def _is_wa_divider(record: dict[str, Any]) -> bool: """True iff this record is a date-divider (raw type code ``-2``). These are not real messages — the upstream inserts them as visual date bars in the chat UI — and must never be yielded or persisted. Per the M3 spec we drop a record when EITHER ``type`` or ``msgType`` is ``-2``; all other unknown codes pass through. The check is on the RAW code (not the normalized label) so the drop decision is robust against any future change to :func:`normalize_whatsapp_type`'s label vocabulary. """ return coerce_int_or_none(record.get("type")) == -2 or ( coerce_int_or_none(record.get("msgType")) == -2 ) async def scrape_whatsapp_messages( client: YunkeClient, uid: str, friend_uid: str, start: int, end: int, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp chat messages, paging backward by timestamp. Wraps ``POST /qqweb/WhatsAppUser/chatRecords`` with a **JSON** body (``{uid, friendUid, start, end}`` — NOT form-encoded like WeChat ``msg/chats``). The endpoint pages BACKWARD: each response carries ``data.end`` (oldest timestamp in this batch) and ``data.hasLast`` (older messages exist?). The scraper advances the cursor to ``end - 1`` and repeats until ``hasLast`` is false, then stops. Date dividers (``type == -2``) are dropped — they are UI chrome, not real messages. All other type codes pass through (mapped to ``"unknown"`` when not in our small known-table) so no real message is silently lost. Parameters ---------- client: Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are handled by the client. uid: Owning staff WhatsApp account id (``uid`` JSON field, of the form ``@s.whatsapp.net``) — maps to ``owner_account_id`` on each yielded record. friend_uid: Customer WhatsApp id (``friendUid`` JSON field) — selects the 1:1 conversation to pull. start, end: Time window in ms epoch. ``end`` is also the initial backward cursor; the first request queries the ``(start, end]`` window and the cursor walks toward ``start`` from there. """ async for record in paginate_by_cursor( lambda end_cursor: _fetch_wa_msg_page( client, uid=uid, friend_uid=friend_uid, start=start, end_cursor=end_cursor, ), initial_cursor=end, ): if _is_wa_divider(record): continue yield _normalize_whatsapp_message(record)