feat(scraper): WA 语音通话抓取 scrape_wa_voice_calls（callSource=4）
复用云客统一语音接口 queryWeChatVoiceList + callSource=4；channel=wa_voice_call 区分；WA 特有字段落 raw_json；复用 PhoneCall 表 + upsert_voice_call。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -40,6 +40,7 @@ __all__ = [
|
||||
"scrape_whatsapp_accounts",
|
||||
"scrape_whatsapp_friends",
|
||||
"scrape_whatsapp_messages",
|
||||
"scrape_wa_voice_calls",
|
||||
]
|
||||
|
||||
WHATSAPP_ACCOUNTS_PATH = "/qqweb/WhatsAppUser/businessSalesList"
|
||||
@@ -47,9 +48,19 @@ WHATSAPP_FRIENDS_PATH = "/qqweb/WhatsAppUser/businessFriendList"
|
||||
|
||||
# WhatsApp list endpoints return wall-clock strings like "2026-08-01 16:39"
|
||||
# (no seconds) for lastUpdate/createTime — parse to ms epoch (naive UTC).
|
||||
# The voice-call endpoint reports ``startTime`` as ``YYYY-MM-DD HH:MM:SS``
|
||||
# (with seconds); the second format below matches it so this list is reused
|
||||
# by the WA voice-call scraper without a private copy.
|
||||
_WA_DATETIME_FORMATS = ["%Y-%m-%d %H:%M", "%Y-%m-%d %H:%M:%S"]
|
||||
WHATSAPP_MESSAGES_PATH = "/qqweb/WhatsAppUser/chatRecords"
|
||||
|
||||
# The unified yunke voice-call endpoint is shared across the WeChat and
|
||||
# WhatsApp channels — same path, same envelope, distinguished by the
|
||||
# ``callSource`` flag in the request body (``"0"`` WeChat / ``"4"`` WA).
|
||||
# ``scrape_wa_voice_calls`` below wraps the WA variant; the WeChat variant
|
||||
# lives in :mod:`app.scraper.wechat` (``scrape_wechat_voice_calls``).
|
||||
WA_VOICE_LIST_PATH = "/wechatweb/callRecord/queryWeChatVoiceList"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Account scraper (M3) — POST /qqweb/WhatsAppUser/businessSalesList
|
||||
@@ -490,3 +501,175 @@ async def scrape_whatsapp_messages(
|
||||
if _is_wa_divider(record):
|
||||
continue
|
||||
yield _normalize_whatsapp_message(record)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Voice-call scraper (Plan 4a Task 1) — POST /wechatweb/callRecord/queryWeChatVoiceList
|
||||
# with ``callSource=4`` (WhatsApp).
|
||||
#
|
||||
# Reuses the unified yunke voice interface also used by the WeChat voice
|
||||
# scraper (``scrape_wechat_voice_calls``): same path, same flat envelope
|
||||
# (rows under ``data.data``, declared total under ``data.total``, no
|
||||
# ``basicResult`` wrapper). The only wire-level difference is the request
|
||||
# body flag ``callSource="4"`` (WA) vs ``"0"`` (WeChat). Output records are
|
||||
# tagged ``channel="wa_voice_call"`` so they share the ``PhoneCall`` table
|
||||
# with the WeChat channel without colliding.
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _parse_wa_voice_total(resp: Any) -> int | None:
|
||||
"""Read the declared row count from the ``queryWeChatVoiceList`` envelope.
|
||||
|
||||
Shape: ``data.total`` (an int) — identical to the WeChat voice envelope
|
||||
(same endpoint). Returns ``None`` if the path is absent so the paginator
|
||||
falls back to the empty-page stop signal. (Live WA ``total`` is 143 per
|
||||
phase0-findings.md §4.)
|
||||
"""
|
||||
try:
|
||||
body = resp.json()
|
||||
except (AttributeError, ValueError):
|
||||
return None
|
||||
data = body.get("data") if isinstance(body, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
total = data.get("total")
|
||||
return int(total) if isinstance(total, (int, float)) and not isinstance(
|
||||
total, bool
|
||||
) else None
|
||||
|
||||
|
||||
def _parse_wa_voice_rows(resp: Any) -> list[dict[str, Any]]:
|
||||
"""Extract the WA voice-call list from the ``queryWeChatVoiceList`` envelope.
|
||||
|
||||
Shape: ``data.data`` (a list of records) — identical to the WeChat voice
|
||||
envelope. Missing / non-list values collapse to an empty list so the
|
||||
paginator sees "empty page".
|
||||
"""
|
||||
try:
|
||||
body = resp.json()
|
||||
except (AttributeError, ValueError):
|
||||
return []
|
||||
data = body.get("data") if isinstance(body, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
rows = data.get("data")
|
||||
return rows if isinstance(rows, list) else []
|
||||
|
||||
|
||||
def _build_wa_voice_body(page_num: int, page_size: int) -> dict[str, Any]:
|
||||
"""Build the WA ``queryWeChatVoiceList`` request body for a given page.
|
||||
|
||||
Adds ``callSource="4"`` (the WA flag; WeChat uses ``"0"``) to the
|
||||
standard ``pageNum`` / ``pageSize`` pagination pair. No filter
|
||||
parameters — the live capture carries none.
|
||||
"""
|
||||
return {"pageNum": page_num, "pageSize": page_size, "callSource": "4"}
|
||||
|
||||
|
||||
def _normalize_wa_voice_call(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Map one WA ``queryWeChatVoiceList`` record to the ``PhoneCall`` shape.
|
||||
|
||||
Adds ``channel="wa_voice_call"`` and preserves the original record under
|
||||
``raw_json``. Field mapping is identical to the WeChat voice normalizer
|
||||
(``_normalize_voice_call`` in :mod:`app.scraper.wechat`) — the endpoint
|
||||
is shared, so the record schema is too. The two intentional differences:
|
||||
|
||||
* ``channel`` is ``"wa_voice_call"`` (not ``"voice_call"``) so the WA
|
||||
rows are distinguishable from WeChat rows in the shared ``PhoneCall``
|
||||
table.
|
||||
* WA-specific fields with no matching ``PhoneCall`` column survive only
|
||||
via ``raw_json``: ``weChatId`` (the owning WA account id, e.g.
|
||||
``<phone>@s.whatsapp.net``), ``callFileId``, ``nextId``, ``isRoom``
|
||||
and ``from`` (the WA source flag).
|
||||
|
||||
Mapping (same as the WeChat voice normalizer, repeated here so this
|
||||
scraper is self-documenting):
|
||||
|
||||
* ``id`` -> ``call_id`` (PK)
|
||||
* ``startTime`` -> ``start_call_time`` (``YYYY-MM-DD HH:MM:SS``
|
||||
-> ms epoch via :func:`parse_datetime_to_ms` + ``_WA_DATETIME_FORMATS``)
|
||||
* ``duration`` -> ``call_duration`` (string seconds coerced to int)
|
||||
* ``callStatus`` -> ``call_status`` (1=connected w/ recording,
|
||||
2=no answer)
|
||||
* ``isSend`` -> ``incoming_call`` with INVERTED polarity
|
||||
(``0=incoming -> 1`` / ``1=outgoing -> 0``; ``None`` preserved when
|
||||
``isSend`` is missing/non-numeric).
|
||||
* ``userId`` -> ``user_id`` (owning staff)
|
||||
* ``userName`` -> ``user_id_name``
|
||||
* ``talker`` -> ``call_number`` (the customer WA id)
|
||||
* ``talkerNickName`` -> ``plan_customer_name``
|
||||
* ``ossFileName`` -> ``record_file`` (the recording URL; may be null)
|
||||
* ``version`` -> ``version`` (carried verbatim)
|
||||
|
||||
Non-applicable ``PhoneCall`` columns (``ring_time`` / ``sim_phone`` /
|
||||
``sim_number`` / ``imei`` / ``is_upload`` / ``download_count`` /
|
||||
``ai_asr`` / ``qc_status`` / ``department_id``) are left ``None`` — the
|
||||
WA voice payload carries none of them. ``None``-safe on every mapped
|
||||
field.
|
||||
"""
|
||||
is_send = coerce_int_or_none(record.get("isSend"))
|
||||
# Flip isSend (0=incoming/1=outgoing) to incoming_call (1=incoming/0=outgoing).
|
||||
incoming_call = None if is_send is None else 1 - is_send
|
||||
return {
|
||||
"call_id": record.get("id"),
|
||||
"channel": "wa_voice_call",
|
||||
"start_call_time": parse_datetime_to_ms(
|
||||
record.get("startTime"), _WA_DATETIME_FORMATS
|
||||
),
|
||||
"call_duration": coerce_int_or_none(record.get("duration")),
|
||||
"call_status": coerce_int_or_none(record.get("callStatus")),
|
||||
"incoming_call": incoming_call,
|
||||
"user_id": record.get("userId"),
|
||||
"user_id_name": record.get("userName"),
|
||||
"call_number": record.get("talker"),
|
||||
"plan_customer_name": record.get("talkerNickName"),
|
||||
"record_file": record.get("ossFileName"),
|
||||
"version": record.get("version"),
|
||||
"raw_json": record,
|
||||
}
|
||||
|
||||
|
||||
async def scrape_wa_voice_calls(
|
||||
client: YunkeClient,
|
||||
*,
|
||||
page_size: int = 50,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield normalized WhatsApp voice-call records from ``queryWeChatVoiceList``.
|
||||
|
||||
Wraps ``POST /wechatweb/callRecord/queryWeChatVoiceList`` with
|
||||
``callSource=4`` — the unified yunke voice interface's WA variant (the
|
||||
WeChat variant, ``callSource=0``, lives in :mod:`app.scraper.wechat`).
|
||||
Walks page by page until the upstream reports no more rows (empty
|
||||
``data.data`` array — the primary stop signal), mapping each record to
|
||||
the ``PhoneCall`` column shape with ``channel="wa_voice_call"`` plus a
|
||||
verbatim ``raw_json`` copy.
|
||||
|
||||
The envelope is identical to the WeChat voice endpoint: rows sit under
|
||||
``data.data``, declared total under ``data.total``, pagination fields
|
||||
under ``data.pageNum`` / ``data.pageSize``. The WA-specific fields
|
||||
(``weChatId`` / ``callFileId`` / ``nextId`` / ``isRoom`` / ``from``)
|
||||
have no ``PhoneCall`` column and survive only via ``raw_json``.
|
||||
|
||||
Persistence reuses :func:`app.storage.repos.upsert_voice_call` (Task 2
|
||||
wires the orchestration); the ``channel`` value distinguishes WA rows
|
||||
from WeChat rows in the shared table.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client:
|
||||
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
||||
handled by the client.
|
||||
page_size:
|
||||
Page size to request. Defaults to 50 (matches the WeChat voice
|
||||
scraper; a sane middle ground between request count and per-response
|
||||
payload size for this endpoint).
|
||||
"""
|
||||
async for record in paginate_by_page(
|
||||
client,
|
||||
WA_VOICE_LIST_PATH,
|
||||
page_size=page_size,
|
||||
parse_total=_parse_wa_voice_total,
|
||||
parse_rows=_parse_wa_voice_rows,
|
||||
build_body=_build_wa_voice_body,
|
||||
):
|
||||
yield _normalize_wa_voice_call(record)
|
||||
|
||||
Reference in New Issue
Block a user