浅念
839 lines
34 KiB
Python
839 lines
34 KiB
Python
"""WhatsApp scrapers (M3).
|
|
|
|
Endpoints covered:
|
|
|
|
* :func:`scrape_whatsapp_accounts` — wraps ``POST /qqweb/WhatsAppUser/businessSalesList``
|
|
and yields normalized :class:`app.db.models.WhatsappAccount` dicts (plus the
|
|
embedded ``userId`` / ``userName`` staff pair). Page-number pagination.
|
|
* :func:`scrape_whatsapp_friends` — wraps ``POST /qqweb/WhatsAppUser/businessFriendList``
|
|
and yields normalized dicts ready to be upserted into
|
|
:class:`app.db.models.Customer` with ``channel="whatsapp"``. Page-number
|
|
pagination.
|
|
* :func:`scrape_whatsapp_groups` — wraps the SAME ``businessFriendList``
|
|
endpoint with ``type="1"`` to select WA chat-groups (``"0"`` selects
|
|
friends) and yields normalized :class:`app.db.models.ChatGroup` dicts with
|
|
``channel="whatsapp"``. The ``type`` group switch is UNVERIFIED (see the
|
|
section docstring) — the structure is correct, only the param name may need
|
|
a post-risk-control adjustment.
|
|
* :func:`scrape_whatsapp_messages` — wraps ``POST /qqweb/WhatsAppUser/chatRecords``
|
|
and yields normalized chat-message dicts with ``channel="whatsapp"``.
|
|
Backward-cursor pagination by timestamp (the same strategy as WeChat
|
|
``msg/chats``), but with a **JSON** request body (NOT form-encoded).
|
|
|
|
All three reuse :func:`app.scraper.base.paginate_by_page` /
|
|
:func:`paginate_by_cursor` and emit snake_case dicts + a verbatim ``raw_json``
|
|
copy, mirroring :mod:`app.scraper.wechat`.
|
|
|
|
Envelope difference from WeChat list endpoints: rows sit under
|
|
``data.overviews`` (NOT ``data.data``) and the declared row count is
|
|
``data.totalCount`` (NOT ``data.total``); the page count is ``data.pageCount``.
|
|
The same envelope is shared by ``businessSalesList`` and ``businessFriendList``.
|
|
|
|
Auth is the same crm2 session cookie the WeChat scrapers use (Phase-0 §1
|
|
confirmed cookie-only auth suffices for every data endpoint).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
|
|
from app.scraper.base import paginate_by_cursor, paginate_by_page
|
|
from app.scraper.parsers import coerce_int_or_none, normalize_whatsapp_type, parse_datetime_to_ms
|
|
from app.scraper.wechat import _normalize_chat_group
|
|
from app.session.yunke_client import YunkeClient
|
|
|
|
__all__ = [
|
|
"scrape_wa_voice_calls",
|
|
"scrape_whatsapp_accounts",
|
|
"scrape_whatsapp_friends",
|
|
"scrape_whatsapp_groups",
|
|
"scrape_whatsapp_messages",
|
|
]
|
|
|
|
WHATSAPP_ACCOUNTS_PATH = "/qqweb/WhatsAppUser/businessSalesList"
|
|
WHATSAPP_FRIENDS_PATH = "/qqweb/WhatsAppUser/businessFriendList"
|
|
|
|
# WhatsApp list endpoints return wall-clock strings like "2026-08-01 16:39"
|
|
# (no seconds) for lastUpdate/createTime — parse to ms epoch (naive UTC).
|
|
# The voice-call endpoint reports ``startTime`` as ``YYYY-MM-DD HH:MM:SS``
|
|
# (with seconds); the second format below matches it so this list is reused
|
|
# by the WA voice-call scraper without a private copy.
|
|
_WA_DATETIME_FORMATS = ["%Y-%m-%d %H:%M", "%Y-%m-%d %H:%M:%S"]
|
|
WHATSAPP_MESSAGES_PATH = "/qqweb/WhatsAppUser/chatRecords"
|
|
|
|
# The unified yunke voice-call endpoint is shared across the WeChat and
|
|
# WhatsApp channels — same path, same envelope, distinguished by the
|
|
# ``callSource`` flag in the request body (``"0"`` WeChat / ``"4"`` WA).
|
|
# ``scrape_wa_voice_calls`` below wraps the WA variant; the WeChat variant
|
|
# lives in :mod:`app.scraper.wechat` (``scrape_wechat_voice_calls``).
|
|
WA_VOICE_LIST_PATH = "/wechatweb/callRecord/queryWeChatVoiceList"
|
|
|
|
|
|
# ===========================================================================
|
|
# Account scraper (M3) — POST /qqweb/WhatsAppUser/businessSalesList
|
|
# ===========================================================================
|
|
|
|
|
|
def _parse_wa_account_total(resp: Any) -> int | None:
|
|
"""Read the declared row count from the ``businessSalesList`` envelope.
|
|
|
|
Shape: ``data.totalCount`` (an int). The page count is ``data.pageCount``
|
|
but the paginator only consumes the row-count ceiling; the empty-page
|
|
signal remains the primary stop. Returns ``None`` if the path is absent so
|
|
the paginator falls back to the empty-page stop signal.
|
|
"""
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
return None
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
total = data.get("totalCount")
|
|
if isinstance(total, bool):
|
|
return None
|
|
return int(total) if isinstance(total, (int, float)) else None
|
|
|
|
|
|
def _parse_wa_account_rows(resp: Any) -> list[dict[str, Any]]:
|
|
"""Extract the account list from the ``businessSalesList`` envelope.
|
|
|
|
Shape: ``data.overviews`` (a list of records). Missing / non-list values
|
|
collapse to an empty list so the paginator sees "empty page".
|
|
"""
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
return []
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return []
|
|
rows = data.get("overviews")
|
|
return rows if isinstance(rows, list) else []
|
|
|
|
|
|
def _build_wa_account_body(page_num: int, page_size: int) -> dict[str, Any]:
|
|
"""Build the ``businessSalesList`` request body for a given page.
|
|
|
|
The live capture carries no filter parameters — just the page slice.
|
|
"""
|
|
return {"pageNum": page_num, "pageSize": page_size}
|
|
|
|
|
|
def _normalize_whatsapp_account(record: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map one ``businessSalesList`` record to the ``WhatsappAccount`` shape.
|
|
|
|
Carries the embedded staff pair (``user_id`` <- ``userId``,
|
|
``user_name`` <- ``userName``) so the persistence layer can upsert the
|
|
owning :class:`app.db.models.Staff` row alongside the account. Adds
|
|
``raw_json`` (the verbatim upstream record). ``None``-safe: missing fields
|
|
become ``None`` rather than raising, so a partial payload does not break
|
|
the scrape.
|
|
"""
|
|
return {
|
|
"uid": record.get("uid"),
|
|
"user_id": record.get("userId"),
|
|
"user_name": record.get("userName"),
|
|
"nick_name": record.get("nickName"),
|
|
"phone": record.get("phone"),
|
|
"sale_phone": record.get("salePhone"),
|
|
"sales_bind_phone": record.get("salesBindPhone"),
|
|
"contacts_num": record.get("contactsNum"),
|
|
"group_num": record.get("groupNum"),
|
|
"talker_count": record.get("talkerCount"),
|
|
"friend_add_num": record.get("friendAddNum"),
|
|
"is_on_job": record.get("isOnJob"),
|
|
"last_update": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS),
|
|
"version": record.get("version"),
|
|
"raw_json": record,
|
|
}
|
|
|
|
|
|
async def scrape_whatsapp_accounts(
|
|
client: YunkeClient,
|
|
*,
|
|
page_size: int = 50,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Yield normalized WhatsApp staff-account records from ``businessSalesList``.
|
|
|
|
Walks the endpoint page by page until the upstream reports no more rows
|
|
(empty ``data.overviews`` array — the primary stop signal), mapping each
|
|
record to the ``WhatsappAccount`` column shape plus a verbatim ``raw_json``
|
|
copy.
|
|
|
|
The envelope differs from WeChat ``wxUser/list``: rows sit under
|
|
``data.overviews`` (not ``data.data``) and the declared total is
|
|
``data.totalCount`` (not ``data.total``). The page count is
|
|
``data.pageCount``.
|
|
|
|
Parameters
|
|
----------
|
|
client:
|
|
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
|
handled by the client.
|
|
page_size:
|
|
Page size to request. Defaults to 50 (matches the observed live page
|
|
size for this endpoint).
|
|
"""
|
|
async for record in paginate_by_page(
|
|
client,
|
|
WHATSAPP_ACCOUNTS_PATH,
|
|
page_size=page_size,
|
|
parse_total=_parse_wa_account_total,
|
|
parse_rows=_parse_wa_account_rows,
|
|
build_body=_build_wa_account_body,
|
|
):
|
|
yield _normalize_whatsapp_account(record)
|
|
|
|
|
|
# ===========================================================================
|
|
# Friend scraper (M3) — POST /qqweb/WhatsAppUser/businessFriendList
|
|
# ===========================================================================
|
|
|
|
|
|
def _parse_wa_friend_total(resp: Any) -> int | None:
|
|
"""Read the declared row count from the ``businessFriendList`` envelope.
|
|
|
|
Shape: ``data.totalCount`` (live total is 11292 per phase0-findings.md §5).
|
|
"""
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
return None
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
total = data.get("totalCount")
|
|
if isinstance(total, bool):
|
|
return None
|
|
return int(total) if isinstance(total, (int, float)) else None
|
|
|
|
|
|
def _parse_wa_friend_rows(resp: Any) -> list[dict[str, Any]]:
|
|
"""Extract the friend list from the ``businessFriendList`` envelope.
|
|
|
|
Shape: ``data.overviews`` (same envelope as ``businessSalesList``).
|
|
"""
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
return []
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return []
|
|
rows = data.get("overviews")
|
|
return rows if isinstance(rows, list) else []
|
|
|
|
|
|
def _build_wa_friend_body(
|
|
page_num: int, page_size: int, *, department_id: str = ""
|
|
) -> dict[str, Any]:
|
|
"""Build the ``businessFriendList`` request body for a given page.
|
|
|
|
Live capture: ``{"searchInfo":"","friendSearchInfo":"","page":1,
|
|
"pageSize":10,"departmentId":"<root>","type":"0"}``. Note this endpoint
|
|
uses ``page`` (NOT ``pageNum`` like businessSalesList) and REQUIRES
|
|
``departmentId`` (the company-root dept id) + ``type`` or it returns empty.
|
|
"""
|
|
body: dict[str, Any] = {
|
|
"searchInfo": "",
|
|
"friendSearchInfo": "",
|
|
"page": page_num,
|
|
"pageSize": page_size,
|
|
"type": "0",
|
|
}
|
|
if department_id:
|
|
body["departmentId"] = department_id
|
|
return body
|
|
|
|
|
|
def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map one ``businessFriendList`` record to the ``Customer`` column shape.
|
|
|
|
Adds ``channel="whatsapp"``. Field mapping per phase0-findings.md §5:
|
|
|
|
* ``friend_id`` <- ``friendUid`` (PK, ``<phone>@s.whatsapp.net``)
|
|
* ``owner_account_id`` <- ``uid`` (the owning staff WA account)
|
|
* ``user_id`` <- ``userId`` (owning staff)
|
|
* ``sale_name`` <- ``salesNickName`` (owning staff sale name)
|
|
* ``friend_nickname`` <- ``nickName``
|
|
* ``phone`` <- ``phone``
|
|
* ``message_count`` <- ``messageCount``
|
|
* ``last_chat_time`` <- ``lastUpdate`` (ms epoch of last activity)
|
|
|
|
Preserves the verbatim record under ``raw_json`` (so the owner-staff
|
|
fields ``userName`` / ``salePhone`` / ``salesBindPhone`` / ``createTime``
|
|
and the per-contact counts ``groupNum`` / ``contactsNum`` / ``talkerCount``
|
|
/ ``friendAddNum`` / ``sensitiveWordNum`` survive end-to-end even though
|
|
they have no direct ``Customer`` column). ``None``-safe on every field.
|
|
|
|
``add_time`` is deliberately ``None``: the WhatsApp ``businessFriendList``
|
|
payload carries no customer-side add-time field (the ``createTime`` field
|
|
on this record is the **owner-staff** account's createTime, not the
|
|
contact's — confirmed via smoke.db raw_json dump). Mapping it to
|
|
``createTime`` would be semantically wrong; the column stays NULL for WA
|
|
contacts (the wechat channel carries the truthful ``addTime``).
|
|
"""
|
|
return {
|
|
"friend_id": record.get("friendUid"),
|
|
"channel": "whatsapp",
|
|
"owner_account_id": record.get("uid"),
|
|
"friend_nickname": record.get("nickName"),
|
|
"phone": record.get("phone"),
|
|
"sale_name": record.get("salesNickName"),
|
|
"user_id": record.get("userId"),
|
|
"message_count": record.get("messageCount"),
|
|
"last_chat_time": parse_datetime_to_ms(record.get("lastUpdate"), _WA_DATETIME_FORMATS),
|
|
# No customer-side add-time on this payload (see docstring); left NULL.
|
|
"add_time": None,
|
|
"version": record.get("version"),
|
|
"raw_json": record,
|
|
}
|
|
|
|
|
|
async def scrape_whatsapp_friends(
|
|
client: YunkeClient,
|
|
*,
|
|
department_id: str = "",
|
|
page_size: int = 50,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Yield normalized WhatsApp contact records from ``businessFriendList``.
|
|
|
|
Walks the endpoint page by page until the upstream reports no more rows
|
|
(empty ``data.overviews`` array — the primary stop signal), mapping each
|
|
record to the ``Customer`` column shape with ``channel="whatsapp"`` plus a
|
|
verbatim ``raw_json`` copy.
|
|
|
|
Parameters
|
|
----------
|
|
client:
|
|
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
|
handled by the client.
|
|
page_size:
|
|
Page size to request. Defaults to 50.
|
|
"""
|
|
async for record in paginate_by_page(
|
|
client,
|
|
WHATSAPP_FRIENDS_PATH,
|
|
page_size=page_size,
|
|
parse_total=_parse_wa_friend_total,
|
|
parse_rows=_parse_wa_friend_rows,
|
|
build_body=_build_wa_friend_body,
|
|
department_id=department_id,
|
|
):
|
|
yield _normalize_whatsapp_friend(record)
|
|
|
|
|
|
# ===========================================================================
|
|
# Chat-group scraper — POST /qqweb/WhatsAppUser/businessFriendList
|
|
# with ``type="1"`` (groups). ``type="0"`` selects friends (see above).
|
|
#
|
|
# UNVERIFIED (云客 risk-control blocking live API calls as of 2026-08-11):
|
|
# the group-type body parameter is inferred, not captured. The WA friends
|
|
# body sends ``type="0"`` and the WA CRM SPA exposes two parallel routes —
|
|
# ``friendlist/0`` (friends) and ``friendlist/1`` (groups) — so by symmetry
|
|
# ``type="1"`` is the most likely group selector. If the post-risk-control
|
|
# smoke returns 0 rows, try alternative param names BEFORE concluding the
|
|
# endpoint is wrong: ``friendType: "1"``, ``friendTypeYunke: "2"`` (the
|
|
# WeChat switch value), or a sibling path entirely. The field mapping and
|
|
# pagination structure here are correct regardless; only the switch param
|
|
# name may need adjustment.
|
|
# ===========================================================================
|
|
|
|
|
|
def _build_wa_group_body(
|
|
page_num: int, page_size: int, *, department_id: str = ""
|
|
) -> dict[str, Any]:
|
|
"""Build the ``businessFriendList`` request body for chat-groups (type=1).
|
|
|
|
Identical to :func:`_build_wa_friend_body` except for ``type``: ``"1"``
|
|
selects groups (``"0"`` selects friends). UNVERIFIED — see the section
|
|
docstring above for the post-risk-control verification plan. ``departmentId``
|
|
is omitted when blank so a test/fixture call without one still forms a
|
|
valid body (the live endpoint requires it for non-empty results, same as
|
|
the friends variant).
|
|
"""
|
|
body: dict[str, Any] = {
|
|
"searchInfo": "",
|
|
"friendSearchInfo": "",
|
|
"page": page_num,
|
|
"pageSize": page_size,
|
|
# UNVERIFIED group-type switch — see section docstring.
|
|
"type": "1",
|
|
}
|
|
if department_id:
|
|
body["departmentId"] = department_id
|
|
return body
|
|
|
|
|
|
def _normalize_whatsapp_group(record: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map one WA ``businessFriendList`` group record to the ``ChatGroup`` shape.
|
|
|
|
Reuses :func:`app.scraper.wechat._normalize_chat_group` for the full field
|
|
mapping — the WA endpoint returns the SAME field names the WeChat
|
|
``friend_list/es`` endpoint does for the group-type switch (``friendWxId``
|
|
is the ``@chatroom`` id, ``friendWxNickname`` is the group name,
|
|
``friendNum`` is the member count, etc.) — then overrides ``channel`` to
|
|
``"whatsapp"`` so WA groups are distinguishable from WeChat groups in the
|
|
shared :class:`app.db.models.ChatGroup` table.
|
|
|
|
Member ids are NOT surfaced by this endpoint (only the ``friendNum``
|
|
count); ``member_ids`` / ``member_ids_list`` are emitted as ``None`` /
|
|
``[]`` by the shared WeChat normalizer and left untouched here.
|
|
"""
|
|
out = _normalize_chat_group(record)
|
|
out["channel"] = "whatsapp"
|
|
return out
|
|
|
|
|
|
async def scrape_whatsapp_groups(
|
|
client: YunkeClient,
|
|
*,
|
|
department_id: str = "",
|
|
page_size: int = 50,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Yield normalized WhatsApp chat-group records from ``businessFriendList``.
|
|
|
|
Mirrors :func:`scrape_whatsapp_friends` but sends ``type="1"`` in the body
|
|
to select groups (the WA CRM SPA's ``friendlist/1`` route). Shares the
|
|
friends endpoint's envelope (rows under ``data.overviews``, declared total
|
|
under ``data.totalCount``), so the same ``_parse_wa_friend_total`` /
|
|
``_parse_wa_friend_rows`` helpers are reused.
|
|
|
|
Each record is mapped to the ``ChatGroup`` column shape with
|
|
``channel="whatsapp"`` (via :func:`_normalize_whatsapp_group`, which reuses
|
|
the WeChat group normalizer for the field mapping) plus a verbatim
|
|
``raw_json`` copy. Member ids are NOT available — ``member_ids`` /
|
|
``member_ids_list`` are ``None`` / ``[]``.
|
|
|
|
.. note::
|
|
|
|
UNVERIFIED (2026-08-11, risk-control): the ``type="1"`` group switch is
|
|
inferred from the WA SPA's ``friendlist/0`` / ``friendlist/1`` routes,
|
|
not a live capture. If the post-risk-control smoke returns 0 rows, try
|
|
``friendType`` / ``friendTypeYunke`` before concluding the endpoint is
|
|
wrong. See the section docstring above.
|
|
|
|
Parameters
|
|
----------
|
|
client:
|
|
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
|
handled by the client.
|
|
department_id:
|
|
Company-root dept id (the ``departmentId`` body field). Required by the
|
|
live endpoint for non-empty results (same as friends); resolved by the
|
|
orchestrator when omitted.
|
|
page_size:
|
|
Page size to request. Defaults to 50.
|
|
"""
|
|
async for record in paginate_by_page(
|
|
client,
|
|
WHATSAPP_FRIENDS_PATH,
|
|
page_size=page_size,
|
|
parse_total=_parse_wa_friend_total,
|
|
parse_rows=_parse_wa_friend_rows,
|
|
build_body=_build_wa_group_body,
|
|
department_id=department_id,
|
|
):
|
|
yield _normalize_whatsapp_group(record)
|
|
|
|
|
|
# ===========================================================================
|
|
# Message scraper (M3) — POST /qqweb/WhatsAppUser/chatRecords
|
|
# ===========================================================================
|
|
|
|
|
|
async def _fetch_wa_msg_page(
|
|
client: YunkeClient,
|
|
*,
|
|
uid: str,
|
|
friend_uid: str,
|
|
start: int,
|
|
end_cursor: int,
|
|
) -> tuple[list[dict[str, Any]], int, bool]:
|
|
"""Fetch one backward page of ``chatRecords``.
|
|
|
|
Returns the ``(rows, next_cursor, has_more)`` triple expected by
|
|
:func:`paginate_by_cursor`. Unlike WeChat ``msg/chats`` this endpoint takes
|
|
a **JSON** body (content-type ``application/json``), not form-encoded.
|
|
|
|
``next_cursor`` is ``data.end - 1`` so the next call's window is
|
|
``(end-1, ...]`` — i.e. strictly older than the oldest message in this
|
|
batch; ``has_more`` is ``data.hasLast``.
|
|
"""
|
|
resp = await client.post(
|
|
WHATSAPP_MESSAGES_PATH,
|
|
json={
|
|
"uid": uid,
|
|
"friendUid": friend_uid,
|
|
"start": start,
|
|
"end": end_cursor,
|
|
},
|
|
)
|
|
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
# Malformed envelope — treat as empty + no-more so the paginator stops.
|
|
return [], end_cursor, False
|
|
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return [], end_cursor, False
|
|
|
|
rows = data.get("messages")
|
|
if not isinstance(rows, list):
|
|
rows = []
|
|
|
|
end = data.get("end")
|
|
next_cursor = (
|
|
int(end) - 1
|
|
if isinstance(end, (int, float)) and not isinstance(end, bool)
|
|
else end_cursor
|
|
)
|
|
has_more = bool(data.get("hasLast"))
|
|
return rows, next_cursor, has_more
|
|
|
|
|
|
def _normalize_whatsapp_message(record: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map one ``chatRecords`` record to the ``Message`` column shape.
|
|
|
|
Adds ``channel="whatsapp"`` and preserves the original record under
|
|
``raw_json``. Per design §4.3 the message type is carried as TWO fields:
|
|
|
|
* ``type`` — the normalized enum label (``"text"`` / ``"image"`` /
|
|
``"unknown"`` / ...) via :func:`normalize_whatsapp_type`.
|
|
* ``type_raw`` — the original upstream type code (``type`` falling back to
|
|
``msgType``), coerced to ``int | None`` so the raw code survives
|
|
end-to-end without round-tripping through ``raw_json``.
|
|
|
|
``mediaName`` (a deferred-download filename, NOT a URL — see phase0 §6) is
|
|
carried on ``file_url`` so the media layer can route on ``channel`` /
|
|
``type``; this mirrors the WeChat ``type=3`` image-legacy pattern where an
|
|
opaque download key (not a URL) is stored on ``file_url``. ``raw_json``
|
|
preserves it verbatim either way. ``quotedId`` / ``quotedMessage`` collapse
|
|
into ``refer_msg`` as a small dict when either is present. No ``room_id``
|
|
for WA 1:1 chats.
|
|
|
|
``None``-safe on every field.
|
|
"""
|
|
# ``type`` is the documented field; fall back to ``msgType`` (some captures
|
|
# populate one, some the other — same disambiguation strategy as WeChat).
|
|
raw_type = record.get("type")
|
|
if raw_type is None:
|
|
raw_type = record.get("msgType")
|
|
|
|
# ``createTime`` is the canonical timestamp; fall back to ``dateTime``.
|
|
timestamp = record.get("createTime")
|
|
if timestamp is None:
|
|
timestamp = record.get("dateTime")
|
|
|
|
# Quoted-message reference: preserve both id + content when either present
|
|
# (a bare None/None pair would just be noise on the JSON column).
|
|
quoted_id = record.get("quotedId")
|
|
quoted_message = record.get("quotedMessage")
|
|
refer_msg = (
|
|
{"quotedId": quoted_id, "quotedMessage": quoted_message}
|
|
if quoted_id is not None or quoted_message is not None
|
|
else None
|
|
)
|
|
|
|
return {
|
|
"msg_id": record.get("keyId"),
|
|
"channel": "whatsapp",
|
|
"owner_account_id": record.get("uid"),
|
|
"talker_id": record.get("friendUid"),
|
|
"is_send": record.get("isSend"),
|
|
"type": normalize_whatsapp_type(raw_type),
|
|
"type_raw": coerce_int_or_none(raw_type),
|
|
"text": record.get("content"),
|
|
# file 常带直连 OSS URL(1:1 图片实测 93% 覆盖,2026-08-20),优先于
|
|
# mediaName 不透明名
|
|
"file_url": record.get("file") or record.get("mediaName"),
|
|
"refer_msg": refer_msg,
|
|
"sender_name": record.get("nickName"),
|
|
"timestamp": timestamp,
|
|
"raw_json": record,
|
|
}
|
|
|
|
|
|
def _is_wa_divider(record: dict[str, Any]) -> bool:
|
|
"""True iff this record is a date-divider (raw type code ``-2``).
|
|
|
|
These are not real messages — the upstream inserts them as visual date
|
|
bars in the chat UI — and must never be yielded or persisted. Per the M3
|
|
spec we drop a record when EITHER ``type`` or ``msgType`` is ``-2``; all
|
|
other unknown codes pass through.
|
|
|
|
The check is on the RAW code (not the normalized label) so the drop
|
|
decision is robust against any future change to
|
|
:func:`normalize_whatsapp_type`'s label vocabulary.
|
|
"""
|
|
return coerce_int_or_none(record.get("type")) == -2 or (
|
|
coerce_int_or_none(record.get("msgType")) == -2
|
|
)
|
|
|
|
|
|
async def scrape_whatsapp_messages(
|
|
client: YunkeClient,
|
|
uid: str,
|
|
friend_uid: str,
|
|
start: int,
|
|
end: int,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Yield normalized WhatsApp chat messages, paging backward by timestamp.
|
|
|
|
Wraps ``POST /qqweb/WhatsAppUser/chatRecords`` with a **JSON** body
|
|
(``{uid, friendUid, start, end}`` — NOT form-encoded like WeChat
|
|
``msg/chats``). The endpoint pages BACKWARD: each response carries
|
|
``data.end`` (oldest timestamp in this batch) and ``data.hasLast`` (older
|
|
messages exist?). The scraper advances the cursor to ``end - 1`` and
|
|
repeats until ``hasLast`` is false, then stops.
|
|
|
|
Date dividers (``type == -2``) are dropped — they are UI chrome, not real
|
|
messages. All other type codes pass through (mapped to ``"unknown"`` when
|
|
not in our small known-table) so no real message is silently lost.
|
|
|
|
Parameters
|
|
----------
|
|
client:
|
|
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
|
handled by the client.
|
|
uid:
|
|
Owning staff WhatsApp account id (``uid`` JSON field, of the form
|
|
``<phone>@s.whatsapp.net``) — maps to ``owner_account_id`` on each
|
|
yielded record.
|
|
friend_uid:
|
|
Customer WhatsApp id (``friendUid`` JSON field) — selects the 1:1
|
|
conversation to pull.
|
|
start, end:
|
|
Time window in ms epoch. ``end`` is also the initial backward cursor;
|
|
the first request queries the ``(start, end]`` window and the cursor
|
|
walks toward ``start`` from there.
|
|
"""
|
|
async for record in paginate_by_cursor(
|
|
lambda end_cursor: _fetch_wa_msg_page(
|
|
client,
|
|
uid=uid,
|
|
friend_uid=friend_uid,
|
|
start=start,
|
|
end_cursor=end_cursor,
|
|
),
|
|
initial_cursor=end,
|
|
):
|
|
if _is_wa_divider(record):
|
|
continue
|
|
yield _normalize_whatsapp_message(record)
|
|
|
|
|
|
# ===========================================================================
|
|
# Voice-call scraper (Plan 4a Task 1) — POST /wechatweb/callRecord/queryWeChatVoiceList
|
|
# with ``callSource=4`` (WhatsApp).
|
|
#
|
|
# Reuses the unified yunke voice interface also used by the WeChat voice
|
|
# scraper (``scrape_wechat_voice_calls``): same path, same flat envelope
|
|
# (rows under ``data.data``, declared total under ``data.total``, no
|
|
# ``basicResult`` wrapper). The only wire-level difference is the request
|
|
# body flag ``callSource="4"`` (WA) vs ``"0"`` (WeChat). Output records are
|
|
# tagged ``channel="wa_voice_call"`` so they share the ``PhoneCall`` table
|
|
# with the WeChat channel without colliding.
|
|
# ===========================================================================
|
|
|
|
|
|
def _parse_wa_voice_total(resp: Any) -> int | None:
|
|
"""Read the declared row count from the ``queryWeChatVoiceList`` envelope.
|
|
|
|
Shape: ``data.total`` (an int) — identical to the WeChat voice envelope
|
|
(same endpoint). Returns ``None`` if the path is absent so the paginator
|
|
falls back to the empty-page stop signal. (Live WA ``total`` is 143 per
|
|
phase0-findings.md §4.)
|
|
"""
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
return None
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
total = data.get("total")
|
|
return int(total) if isinstance(total, (int, float)) and not isinstance(
|
|
total, bool
|
|
) else None
|
|
|
|
|
|
def _parse_wa_voice_rows(resp: Any) -> list[dict[str, Any]]:
|
|
"""Extract the WA voice-call list from the ``queryWeChatVoiceList`` envelope.
|
|
|
|
Shape: ``data.data`` (a list of records) — identical to the WeChat voice
|
|
envelope. Missing / non-list values collapse to an empty list so the
|
|
paginator sees "empty page".
|
|
"""
|
|
try:
|
|
body = resp.json()
|
|
except (AttributeError, ValueError):
|
|
return []
|
|
data = body.get("data") if isinstance(body, dict) else None
|
|
if not isinstance(data, dict):
|
|
return []
|
|
rows = data.get("data")
|
|
return rows if isinstance(rows, list) else []
|
|
|
|
|
|
def _build_wa_voice_body(
|
|
page_num: int,
|
|
page_size: int,
|
|
*,
|
|
department_id: str = "",
|
|
start: str = "2000-01-01 00:00:00",
|
|
end: str = "2099-12-31 23:59:59",
|
|
) -> dict[str, Any]:
|
|
"""Build the WA ``queryWeChatVoiceList`` request body for a given page.
|
|
|
|
Live diagnosis (2026-08-07 browser capture) confirmed the endpoint
|
|
REQUIRES ``callSource`` + ``departmentId`` + ``isRoom`` + ``startTime`` +
|
|
``endTime`` — omit any one and the HTTP layer returns 200 with an EMPTY
|
|
``data.data`` array (0 rows). The WA variant uses ``callSource="4"``;
|
|
a wide time window (2000-01-01 .. 2099-12-31) pulls the full history in
|
|
one scrape (live total=608). ``userId`` / ``sort`` / ``ids`` may stay
|
|
empty. ``departmentId`` is omitted when blank so a test/fixture call
|
|
without one still forms a valid body.
|
|
"""
|
|
body: dict[str, Any] = {
|
|
"pageNum": page_num,
|
|
"pageSize": page_size,
|
|
"callSource": "4",
|
|
"isRoom": "0",
|
|
"startTime": start,
|
|
"endTime": end,
|
|
"sort": "",
|
|
"ids": [],
|
|
"userId": "",
|
|
}
|
|
if department_id:
|
|
body["departmentId"] = department_id
|
|
return body
|
|
|
|
|
|
def _normalize_wa_voice_call(record: dict[str, Any]) -> dict[str, Any]:
|
|
"""Map one WA ``queryWeChatVoiceList`` record to the ``PhoneCall`` shape.
|
|
|
|
Adds ``channel="wa_voice_call"`` and preserves the original record under
|
|
``raw_json``. Field mapping is identical to the WeChat voice normalizer
|
|
(``_normalize_voice_call`` in :mod:`app.scraper.wechat`) — the endpoint
|
|
is shared, so the record schema is too. The two intentional differences:
|
|
|
|
* ``channel`` is ``"wa_voice_call"`` (not ``"voice_call"``) so the WA
|
|
rows are distinguishable from WeChat rows in the shared ``PhoneCall``
|
|
table.
|
|
* WA-specific fields with no matching ``PhoneCall`` column survive only
|
|
via ``raw_json``: ``weChatId`` (the owning WA account id, e.g.
|
|
``<phone>@s.whatsapp.net``), ``callFileId``, ``nextId``, ``isRoom``
|
|
and ``from`` (the WA source flag).
|
|
|
|
Mapping (same as the WeChat voice normalizer, repeated here so this
|
|
scraper is self-documenting):
|
|
|
|
* ``id`` -> ``call_id`` (PK)
|
|
* ``startTime`` -> ``start_call_time`` (``YYYY-MM-DD HH:MM:SS``
|
|
-> ms epoch via :func:`parse_datetime_to_ms` + ``_WA_DATETIME_FORMATS``)
|
|
* ``duration`` -> ``call_duration`` (string seconds coerced to int)
|
|
* ``callStatus`` -> ``call_status`` (1=connected w/ recording,
|
|
2=no answer)
|
|
* ``isSend`` -> ``incoming_call`` with INVERTED polarity
|
|
(``0=incoming -> 1`` / ``1=outgoing -> 0``; ``None`` preserved when
|
|
``isSend`` is missing/non-numeric).
|
|
* ``userId`` -> ``user_id`` (owning staff)
|
|
* ``userName`` -> ``user_id_name``
|
|
* ``talker`` -> ``call_number`` (the customer WA id)
|
|
* ``talkerNickName`` -> ``plan_customer_name``
|
|
* ``ossFileName`` -> ``record_file`` (the recording URL; may be null)
|
|
* ``version`` -> ``version`` (carried verbatim)
|
|
|
|
Non-applicable ``PhoneCall`` columns (``ring_time`` / ``sim_phone`` /
|
|
``sim_number`` / ``imei`` / ``is_upload`` / ``download_count`` /
|
|
``ai_asr`` / ``qc_status`` / ``department_id``) are left ``None`` — the
|
|
WA voice payload carries none of them. ``None``-safe on every mapped
|
|
field.
|
|
"""
|
|
is_send = coerce_int_or_none(record.get("isSend"))
|
|
# Flip isSend (0=incoming/1=outgoing) to incoming_call (1=incoming/0=outgoing).
|
|
incoming_call = None if is_send is None else 1 - is_send
|
|
return {
|
|
"call_id": record.get("id"),
|
|
"channel": "wa_voice_call",
|
|
"start_call_time": parse_datetime_to_ms(
|
|
record.get("startTime"), _WA_DATETIME_FORMATS
|
|
),
|
|
"call_duration": coerce_int_or_none(record.get("duration")),
|
|
"call_status": coerce_int_or_none(record.get("callStatus")),
|
|
"incoming_call": incoming_call,
|
|
"user_id": record.get("userId"),
|
|
"user_id_name": record.get("userName"),
|
|
"call_number": record.get("talker"),
|
|
"plan_customer_name": record.get("talkerNickName"),
|
|
"record_file": record.get("ossFileName"),
|
|
"version": record.get("version"),
|
|
"raw_json": record,
|
|
}
|
|
|
|
|
|
async def scrape_wa_voice_calls(
|
|
client: YunkeClient,
|
|
*,
|
|
department_id: str = "",
|
|
page_size: int = 50,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Yield normalized WhatsApp voice-call records from ``queryWeChatVoiceList``.
|
|
|
|
Wraps ``POST /wechatweb/callRecord/queryWeChatVoiceList`` with
|
|
``callSource=4`` — the unified yunke voice interface's WA variant (the
|
|
WeChat variant, ``callSource=0``, lives in :mod:`app.scraper.wechat`).
|
|
Walks page by page until the upstream reports no more rows (empty
|
|
``data.data`` array — the primary stop signal), mapping each record to
|
|
the ``PhoneCall`` column shape with ``channel="wa_voice_call"`` plus a
|
|
verbatim ``raw_json`` copy.
|
|
|
|
The endpoint REQUIRES ``callSource`` + ``departmentId`` + ``isRoom`` +
|
|
a ``startTime``/``endTime`` window — without them it returns 200 + an
|
|
empty ``data.data`` (0 rows). The orchestrator resolves the
|
|
company-root ``department_id`` from the mirrored ``Department`` table
|
|
(see :func:`app.sync.orchestrator.sync_wa_voice_calls`); the default
|
|
wide time window ``2000-01-01 .. 2099-12-31`` pulls the full history
|
|
(live total=608) and is not exposed as a parameter.
|
|
|
|
The envelope is identical to the WeChat voice endpoint: rows sit under
|
|
``data.data``, declared total under ``data.total``, pagination fields
|
|
under ``data.pageNum`` / ``data.pageSize``. The WA-specific fields
|
|
(``weChatId`` / ``callFileId`` / ``nextId`` / ``isRoom`` / ``from``)
|
|
have no ``PhoneCall`` column and survive only via ``raw_json``.
|
|
|
|
Persistence reuses :func:`app.storage.repos.upsert_voice_call` (Task 2
|
|
wires the orchestration); the ``channel`` value distinguishes WA rows
|
|
from WeChat rows in the shared table.
|
|
|
|
Parameters
|
|
----------
|
|
client:
|
|
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
|
handled by the client.
|
|
department_id:
|
|
Company-root department id (the ``departmentId`` form field).
|
|
Required by the live endpoint for non-empty results; resolved by
|
|
the orchestrator when omitted (empty skips the field in the body).
|
|
page_size:
|
|
Page size to request. Defaults to 50 (matches the WeChat voice
|
|
scraper; a sane middle ground between request count and per-response
|
|
payload size for this endpoint).
|
|
"""
|
|
async for record in paginate_by_page(
|
|
client,
|
|
WA_VOICE_LIST_PATH,
|
|
page_size=page_size,
|
|
parse_total=_parse_wa_voice_total,
|
|
parse_rows=_parse_wa_voice_rows,
|
|
build_body=_build_wa_voice_body,
|
|
department_id=department_id,
|
|
):
|
|
yield _normalize_wa_voice_call(record)
|