xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: whatsapp account/friend/message scrapers

Browse Source 
Mirrors the WeChat scraper patterns (M3) using the shared paginator +
parser + repo building blocks. Three new async generators yield
snake_case dicts + verbatim raw_json, ready for the channel-generic
upsert_customer / upsert_messages repos.

- app/scraper/whatsapp.py (new): scrape_whatsapp_accounts /
  scrape_whatsapp_friends (page-number pagination over the
  data.overviews / data.totalCount envelope) and scrape_whatsapp_messages
  (backward-cursor pagination, JSON body {uid,friendUid,start,end} —
  NOT form-encoded like WeChat msg/chats). Drops type==-2 dividers;
  maps mediaName -> file_url as a deferred-download key.
- app/scraper/parsers.py: add normalize_whatsapp_type (0->text,
  1->image, -2->divider, else unknown; never raises) and a shared
  coerce_int_or_none scraper helper (mirrors repos._coerce_int so the
  scraper emits type_raw ints without a DB dependency).
- app/storage/repos.py: add upsert_whatsapp_account (PK uid) which
  also upserts the embedded Staff(user_id, name) via the existing
  upsert_staff_from_account helper; WhatsappAccount imported.

Tests (respx mocks, no network): 3 cassette fixtures + 3 scraper test
modules (accounts/friends/messages), parsers + repos extended with WA
cases. Divider drop, backward-cursor advance (end == E1-1), JSON body
asserts, embedded-staff upsert, and channel-generic Customer/Message
upserts all covered.

Suite: 182 -> 211 passing. ruff clean.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 15:48:03 +08:00
parent 2508b5aec1
commit 8a22fd9ff8
11 changed files with 1854 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/tests/cassettes/wa_accounts_fixture.py
+148
View File
@@ -0,0 +1,148 @@
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/businessSalesList`` (M3).
These records mirror the real field names captured from the live CRM (see the
data dictionary §3 and phase0-findings.md §5):
``uid`` (PK, ``<phone>@s.whatsapp.net``), ``userId`` / ``userName`` (owning
staff real name), ``nickName``, ``phone`` / ``salePhone`` / ``salesBindPhone``,
``contactsNum`` / ``groupNum`` / ``talkerCount`` / ``friendAddNum``,
``isOnJob``, ``lastUpdate``.
The envelope differs from the WeChat list endpoints: rows sit under
``data.overviews`` (NOT ``data.data``) and the declared row count is
``data.totalCount`` (NOT ``data.total``); the page count is ``data.pageCount``::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
The cassette ships two pages so a test can drive the page-number paginator
across the empty-page stop signal:
* page 1 -> 2 of the 4 fixture records, ``totalCount=4``
* page 2 -> the other 2 records, ``totalCount=4``
* (page 3 -> empty ``overviews`` array, served inline by the test)
Records carry every field with a target ``WhatsappAccount`` column so the test
can assert a full mapping end-to-end, plus the embedded ``userId`` + ``userName``
staff pair (the owning employee) so the embedded-staff repo upsert can be
exercised against capture-shaped input (including a no-``userId`` orphan to
exercise the best-effort skip path).
"""
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped WhatsApp staff-account records (4 total across 2 pages).
# ---------------------------------------------------------------------------
WA_ACCOUNT_RECORDS: list[dict[str, Any]] = [
{
"uid": "8613800000001@s.whatsapp.net",
"userId": "wa_staff_001",
"userName": "Alice Wang",
"nickName": "Alice WA",
"phone": "8613800000001",
"salePhone": "8613800000001",
"salesBindPhone": "8613900000001",
"contactsNum": 421,
"groupNum": 12,
"talkerCount": 38,
"friendAddNum": 7,
"isOnJob": 1,
"lastUpdate": 1700000000123,
"version": "v1.20260731.alice.wa",
},
{
"uid": "8613800000002@s.whatsapp.net",
"userId": "wa_staff_002",
"userName": "Bob Chen",
"nickName": "Bob WA",
"phone": "8613800000002",
"salePhone": "8613800000002",
"salesBindPhone": "8613900000002",
"contactsNum": 58,
"groupNum": 2,
"talkerCount": 9,
"friendAddNum": 1,
"isOnJob": 1,
"lastUpdate": 1700000000456,
"version": "v1.20260731.bob.wa",
},
{
"uid": "8613800000003@s.whatsapp.net",
"userId": "wa_staff_003",
"userName": "Carol Liu",
"nickName": "Carol WA",
"phone": "8613800000003",
"salePhone": "8613800000003",
"salesBindPhone": "8613900000003",
"contactsNum": 1899,
"groupNum": 64,
"talkerCount": 210,
"friendAddNum": 33,
"isOnJob": 0, # departed staff — exercises the falsy isOnJob path
"lastUpdate": 1700000000789,
"version": "v1.20260731.carol.wa",
},
{
# Account with NO embedded staff (userId missing) — the scraper must
# still yield it; the embedded-staff repo upsert must skip it cleanly.
"uid": "8613800000004@s.whatsapp.net",
"userId": None,
"userName": None,
"nickName": "Orphan WA",
"phone": "8613800000004",
"salePhone": "8613800000004",
"salesBindPhone": "8613900000004",
"contactsNum": 3,
"groupNum": 0,
"talkerCount": 1,
"friendAddNum": 0,
"isOnJob": 0,
"lastUpdate": 1700000001000,
"version": "v1.20260731.orphan.wa",
},
]
# Default page size for the fixture (and the scraper). totalCount=4 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(WA_ACCOUNT_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_wa_accounts_response(
records: list[dict[str, Any]],
*,
total_count: int,
page: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
page_count: int | None = None,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``businessSalesList`` response envelope.
Shape::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
``page_count`` defaults to ``ceil(total_count / page_size)`` when omitted.
"""
if page_count is None:
import math
page_count = math.ceil(total_count / page_size) if page_size else 1
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-wa-acct-{page}",
"data": {
"page": page,
"pageCount": page_count,
"totalCount": total_count,
"overviews": list(records),
},
}
yunke-server/tests/cassettes/wa_friends_fixture.py
+129
View File
@@ -0,0 +1,129 @@
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/businessFriendList`` (M3).
These records mirror the real field names captured from the live CRM (see
phase0-findings.md §5; live total = **11292**):
``uid`` (owning staff ``<phone>@s.whatsapp.net``), ``userId`` / ``userName`` /
``salePhone`` / ``salesNickName`` / ``salesBindPhone`` (owning staff),
``friendUid`` (customer -> ``friend_id`` PK), ``nickName`` (customer nickname),
``phone``, ``createTime`` / ``lastUpdate``, ``groupNum`` / ``contactsNum`` /
``talkerCount`` / ``friendAddNum`` / ``messageCount`` / ``sensitiveWordNum``.
The envelope is the SAME shape as ``businessSalesList``: rows under
``data.overviews``, declared count under ``data.totalCount``, page count under
``data.pageCount``::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
The fixture's ``TOTAL`` is pinned to the record count (not the live 11292) so
the paginator's over-fetch ceiling does not fire early in the multi-page test.
"""
from __future__ import annotations
import math
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped WhatsApp contact records (3 total across 2 pages).
# ---------------------------------------------------------------------------
WA_FRIEND_RECORDS: list[dict[str, Any]] = [
{
"uid": "8613800000001@s.whatsapp.net", # owning staff account
"userId": "wa_staff_001",
"userName": "Alice Wang",
"salePhone": "8613800000001",
"salesNickName": "Alice Sale",
"salesBindPhone": "8613900000001",
"friendUid": "8613800000111@s.whatsapp.net", # customer -> friend_id
"nickName": "Customer One",
"phone": "8613800000111",
"createTime": 1699000000000,
"lastUpdate": 1700000000123,
"groupNum": 2,
"contactsNum": 421,
"talkerCount": 38,
"friendAddNum": 1,
"messageCount": 57,
"sensitiveWordNum": 0,
"version": "v1.20260731.cust1",
},
{
"uid": "8613800000001@s.whatsapp.net",
"userId": "wa_staff_001",
"userName": "Alice Wang",
"salePhone": "8613800000001",
"salesNickName": "Alice Sale",
"salesBindPhone": "8613900000001",
"friendUid": "8613800000222@s.whatsapp.net",
"nickName": "Customer Two",
"phone": "8613800000222",
"createTime": 1699100000000,
"lastUpdate": 1700000000456,
"groupNum": 1,
"contactsNum": 421,
"talkerCount": 38,
"friendAddNum": 1,
"messageCount": 9,
"sensitiveWordNum": 0,
"version": "v1.20260731.cust2",
},
{
# Contact owned by a different staff account.
"uid": "8613800000002@s.whatsapp.net",
"userId": "wa_staff_002",
"userName": "Bob Chen",
"salePhone": "8613800000002",
"salesNickName": "Bob Sale",
"salesBindPhone": "8613900000002",
"friendUid": "8613800000333@s.whatsapp.net",
"nickName": "Customer Three",
"phone": "8613800000333",
"createTime": 1699200000000,
"lastUpdate": 1700000000789,
"groupNum": 0,
"contactsNum": 58,
"talkerCount": 9,
"friendAddNum": 0,
"messageCount": 4,
"sensitiveWordNum": 1,
"version": "v1.20260731.cust3",
},
]
TOTAL = len(WA_FRIEND_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_wa_friends_response(
records: list[dict[str, Any]],
*,
total_count: int,
page: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
page_count: int | None = None,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``businessFriendList`` response envelope.
Shape (same as ``businessSalesList``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
"""
if page_count is None:
page_count = math.ceil(total_count / page_size) if page_size else 1
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-wa-friend-{page}",
"data": {
"page": page,
"pageCount": page_count,
"totalCount": total_count,
"overviews": list(records),
},
}
yunke-server/tests/cassettes/wa_messages_fixture.py
+142
View File
@@ -0,0 +1,142 @@
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/chatRecords`` (M3).
These records mirror the real field names captured from the live CRM (see the
data dictionary §3 + phase0-findings.md §6):
``keyId`` (PK), ``type`` / ``msgType`` (``0``=text, ``1``=image/media,
``-2``=date divider), ``content`` (text), ``mediaName`` (media FILENAME — NOT a
URL; deferred download), ``nickName`` (sender), ``uid`` (owning staff account),
``friendUid`` (the other party), ``isSend``, ``createTime`` / ``dateTime``,
``timezone``, ``companyCode``, ``quotedId`` / ``quotedMessage`` (quoted reply).
The request is a **JSON body** ``{uid, friendUid, start, end}`` (ms epochs) —
NOT form-encoded like WeChat ``msg/chats``. The response pages BACKWARD: each
page carries ``data.end`` (oldest timestamp in this batch) and ``data.hasLast``
(older messages exist?); the scraper advances the cursor to ``end - 1`` and
repeats until ``hasLast`` is false::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"messages":[...], "start":.., "end":.., "hasNext":.., "hasLast":..}}
The cassette is shaped to drive the backward-cursor paginator across two pages:
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
The ``-2`` record is a date divider that the scraper must DROP. ``E1`` is the
oldest timestamp in page 1's batch, so the backward-cursor advance must set the
next request's ``end`` to ``E1 - 1``.
"""
from __future__ import annotations
from typing import Any
# Oldest-timestamp-in-batch returned by each page; the scraper uses these to
# advance the backward cursor (next_request_end = page_end - 1).
E1 = 1784517338999
E2 = 1784517000000
# Page 1 records, EXACT live-capture field names. m1 is text/incoming,
# m_datebar is a -2 divider (must be filtered), m2 is image/outgoing.
WA_MESSAGE_RECORDS_PAGE1: list[dict[str, Any]] = [
{ # m1 — text message, incoming.
"keyId": "wa_1000001",
"type": 0,
"msgType": 0,
"content": "hello from whatsapp",
"nickName": "Customer One",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 0,
"createTime": E1 + 600000,
"dateTime": E1 + 600000,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"version": "v1.0",
},
{ # date divider — type == -2; scraper MUST drop this, never yield it.
"keyId": "wa_1000000",
"type": -2,
"msgType": -2,
"content": "2026-01-01",
"nickName": "",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 0,
"createTime": E1 + 300000,
"dateTime": E1 + 300000,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"version": "v1.0",
},
{ # m2 — image message, outgoing, with a media filename (deferred download).
"keyId": "wa_1000002",
"type": 1,
"msgType": 1,
"content": "[image]",
"mediaName": "8613800000001@s.whatsapp.net_wa_1000002.jpg",
"nickName": "Alice Wang",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 1,
"createTime": E1 + 100,
"dateTime": E1 + 100,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"version": "v1.0",
},
]
# Page 2 record — text message with a quoted reply; oldest in the window.
WA_MESSAGE_RECORDS_PAGE2: list[dict[str, Any]] = [
{
"keyId": "wa_999999",
"type": 0,
"msgType": 0,
"content": "replying to an earlier message",
"nickName": "Customer One",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 0,
"createTime": E2,
"dateTime": E2,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"quotedId": "wa_888888",
"quotedMessage": "the earlier message text",
"version": "v1.0",
},
]
def make_wa_messages_response(
messages: list[dict[str, Any]],
*,
end: int,
has_last: bool,
start: int = 1,
has_next: bool = False,
trace_id: str = "trace-wa-msg-1",
) -> dict[str, Any]:
"""Build a capture-shaped ``chatRecords`` response envelope.
Shape (EXACT, per M3 spec)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"messages":[...], "start":.., "end":<oldest ts in batch>,
"hasNext":<bool>, "hasLast":<bool — are there older messages?>}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id,
"data": {
"messages": list(messages),
"start": start,
"end": end,
"hasNext": has_next,
"hasLast": has_last,
},
}