浅念
8a22fd9ff8
Mirrors the WeChat scraper patterns (M3) using the shared paginator +
parser + repo building blocks. Three new async generators yield
snake_case dicts + verbatim raw_json, ready for the channel-generic
upsert_customer / upsert_messages repos.
- app/scraper/whatsapp.py (new): scrape_whatsapp_accounts /
scrape_whatsapp_friends (page-number pagination over the
data.overviews / data.totalCount envelope) and scrape_whatsapp_messages
(backward-cursor pagination, JSON body {uid,friendUid,start,end} —
NOT form-encoded like WeChat msg/chats). Drops type==-2 dividers;
maps mediaName -> file_url as a deferred-download key.
- app/scraper/parsers.py: add normalize_whatsapp_type (0->text,
1->image, -2->divider, else unknown; never raises) and a shared
coerce_int_or_none scraper helper (mirrors repos._coerce_int so the
scraper emits type_raw ints without a DB dependency).
- app/storage/repos.py: add upsert_whatsapp_account (PK uid) which
also upserts the embedded Staff(user_id, name) via the existing
upsert_staff_from_account helper; WhatsappAccount imported.
Tests (respx mocks, no network): 3 cassette fixtures + 3 scraper test
modules (accounts/friends/messages), parsers + repos extended with WA
cases. Divider drop, backward-cursor advance (end == E1-1), JSON body
asserts, embedded-staff upsert, and channel-generic Customer/Message
upserts all covered.
Suite: 182 -> 211 passing. ruff clean.
Co-Authored-By: Claude <noreply@anthropic.com>
143 lines
4.9 KiB
Python
143 lines
4.9 KiB
Python
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/chatRecords`` (M3).
|
|
|
|
These records mirror the real field names captured from the live CRM (see the
|
|
data dictionary §3 + phase0-findings.md §6):
|
|
|
|
``keyId`` (PK), ``type`` / ``msgType`` (``0``=text, ``1``=image/media,
|
|
``-2``=date divider), ``content`` (text), ``mediaName`` (media FILENAME — NOT a
|
|
URL; deferred download), ``nickName`` (sender), ``uid`` (owning staff account),
|
|
``friendUid`` (the other party), ``isSend``, ``createTime`` / ``dateTime``,
|
|
``timezone``, ``companyCode``, ``quotedId`` / ``quotedMessage`` (quoted reply).
|
|
|
|
The request is a **JSON body** ``{uid, friendUid, start, end}`` (ms epochs) —
|
|
NOT form-encoded like WeChat ``msg/chats``. The response pages BACKWARD: each
|
|
page carries ``data.end`` (oldest timestamp in this batch) and ``data.hasLast``
|
|
(older messages exist?); the scraper advances the cursor to ``end - 1`` and
|
|
repeats until ``hasLast`` is false::
|
|
|
|
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
|
"data":{"messages":[...], "start":.., "end":.., "hasNext":.., "hasLast":..}}
|
|
|
|
The cassette is shaped to drive the backward-cursor paginator across two pages:
|
|
|
|
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
|
|
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
|
|
|
|
The ``-2`` record is a date divider that the scraper must DROP. ``E1`` is the
|
|
oldest timestamp in page 1's batch, so the backward-cursor advance must set the
|
|
next request's ``end`` to ``E1 - 1``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
# Oldest-timestamp-in-batch returned by each page; the scraper uses these to
|
|
# advance the backward cursor (next_request_end = page_end - 1).
|
|
E1 = 1784517338999
|
|
E2 = 1784517000000
|
|
|
|
# Page 1 records, EXACT live-capture field names. m1 is text/incoming,
|
|
# m_datebar is a -2 divider (must be filtered), m2 is image/outgoing.
|
|
WA_MESSAGE_RECORDS_PAGE1: list[dict[str, Any]] = [
|
|
{ # m1 — text message, incoming.
|
|
"keyId": "wa_1000001",
|
|
"type": 0,
|
|
"msgType": 0,
|
|
"content": "hello from whatsapp",
|
|
"nickName": "Customer One",
|
|
"uid": "8613800000001@s.whatsapp.net",
|
|
"friendUid": "8613800000111@s.whatsapp.net",
|
|
"isSend": 0,
|
|
"createTime": E1 + 600000,
|
|
"dateTime": E1 + 600000,
|
|
"timezone": "Asia/Shanghai",
|
|
"companyCode": "8m0atd",
|
|
"version": "v1.0",
|
|
},
|
|
{ # date divider — type == -2; scraper MUST drop this, never yield it.
|
|
"keyId": "wa_1000000",
|
|
"type": -2,
|
|
"msgType": -2,
|
|
"content": "2026-01-01",
|
|
"nickName": "",
|
|
"uid": "8613800000001@s.whatsapp.net",
|
|
"friendUid": "8613800000111@s.whatsapp.net",
|
|
"isSend": 0,
|
|
"createTime": E1 + 300000,
|
|
"dateTime": E1 + 300000,
|
|
"timezone": "Asia/Shanghai",
|
|
"companyCode": "8m0atd",
|
|
"version": "v1.0",
|
|
},
|
|
{ # m2 — image message, outgoing, with a media filename (deferred download).
|
|
"keyId": "wa_1000002",
|
|
"type": 1,
|
|
"msgType": 1,
|
|
"content": "[image]",
|
|
"mediaName": "8613800000001@s.whatsapp.net_wa_1000002.jpg",
|
|
"nickName": "Alice Wang",
|
|
"uid": "8613800000001@s.whatsapp.net",
|
|
"friendUid": "8613800000111@s.whatsapp.net",
|
|
"isSend": 1,
|
|
"createTime": E1 + 100,
|
|
"dateTime": E1 + 100,
|
|
"timezone": "Asia/Shanghai",
|
|
"companyCode": "8m0atd",
|
|
"version": "v1.0",
|
|
},
|
|
]
|
|
|
|
# Page 2 record — text message with a quoted reply; oldest in the window.
|
|
WA_MESSAGE_RECORDS_PAGE2: list[dict[str, Any]] = [
|
|
{
|
|
"keyId": "wa_999999",
|
|
"type": 0,
|
|
"msgType": 0,
|
|
"content": "replying to an earlier message",
|
|
"nickName": "Customer One",
|
|
"uid": "8613800000001@s.whatsapp.net",
|
|
"friendUid": "8613800000111@s.whatsapp.net",
|
|
"isSend": 0,
|
|
"createTime": E2,
|
|
"dateTime": E2,
|
|
"timezone": "Asia/Shanghai",
|
|
"companyCode": "8m0atd",
|
|
"quotedId": "wa_888888",
|
|
"quotedMessage": "the earlier message text",
|
|
"version": "v1.0",
|
|
},
|
|
]
|
|
|
|
|
|
def make_wa_messages_response(
|
|
messages: list[dict[str, Any]],
|
|
*,
|
|
end: int,
|
|
has_last: bool,
|
|
start: int = 1,
|
|
has_next: bool = False,
|
|
trace_id: str = "trace-wa-msg-1",
|
|
) -> dict[str, Any]:
|
|
"""Build a capture-shaped ``chatRecords`` response envelope.
|
|
|
|
Shape (EXACT, per M3 spec)::
|
|
|
|
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
|
"data":{"messages":[...], "start":.., "end":<oldest ts in batch>,
|
|
"hasNext":<bool>, "hasLast":<bool — are there older messages?>}}
|
|
"""
|
|
return {
|
|
"message": "调用成功",
|
|
"code": "10000",
|
|
"success": True,
|
|
"traceId": trace_id,
|
|
"data": {
|
|
"messages": list(messages),
|
|
"start": start,
|
|
"end": end,
|
|
"hasNext": has_next,
|
|
"hasLast": has_last,
|
|
},
|
|
}
|