xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: whatsapp account/friend/message scrapers

Browse Source 
Mirrors the WeChat scraper patterns (M3) using the shared paginator +
parser + repo building blocks. Three new async generators yield
snake_case dicts + verbatim raw_json, ready for the channel-generic
upsert_customer / upsert_messages repos.

- app/scraper/whatsapp.py (new): scrape_whatsapp_accounts /
  scrape_whatsapp_friends (page-number pagination over the
  data.overviews / data.totalCount envelope) and scrape_whatsapp_messages
  (backward-cursor pagination, JSON body {uid,friendUid,start,end} —
  NOT form-encoded like WeChat msg/chats). Drops type==-2 dividers;
  maps mediaName -> file_url as a deferred-download key.
- app/scraper/parsers.py: add normalize_whatsapp_type (0->text,
  1->image, -2->divider, else unknown; never raises) and a shared
  coerce_int_or_none scraper helper (mirrors repos._coerce_int so the
  scraper emits type_raw ints without a DB dependency).
- app/storage/repos.py: add upsert_whatsapp_account (PK uid) which
  also upserts the embedded Staff(user_id, name) via the existing
  upsert_staff_from_account helper; WhatsappAccount imported.

Tests (respx mocks, no network): 3 cassette fixtures + 3 scraper test
modules (accounts/friends/messages), parsers + repos extended with WA
cases. Divider drop, backward-cursor advance (end == E1-1), JSON body
asserts, embedded-staff upsert, and channel-generic Customer/Message
upserts all covered.

Suite: 182 -> 211 passing. ruff clean.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 15:48:03 +08:00
parent 2508b5aec1
commit 8a22fd9ff8
11 changed files with 1854 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/tests/cassettes/wa_accounts_fixture.py
+148
View File
@@ -0,0 +1,148 @@
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/businessSalesList`` (M3).
These records mirror the real field names captured from the live CRM (see the
data dictionary §3 and phase0-findings.md §5):
``uid`` (PK, ``<phone>@s.whatsapp.net``), ``userId`` / ``userName`` (owning
staff real name), ``nickName``, ``phone`` / ``salePhone`` / ``salesBindPhone``,
``contactsNum`` / ``groupNum`` / ``talkerCount`` / ``friendAddNum``,
``isOnJob``, ``lastUpdate``.
The envelope differs from the WeChat list endpoints: rows sit under
``data.overviews`` (NOT ``data.data``) and the declared row count is
``data.totalCount`` (NOT ``data.total``); the page count is ``data.pageCount``::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
The cassette ships two pages so a test can drive the page-number paginator
across the empty-page stop signal:
* page 1 -> 2 of the 4 fixture records, ``totalCount=4``
* page 2 -> the other 2 records, ``totalCount=4``
* (page 3 -> empty ``overviews`` array, served inline by the test)
Records carry every field with a target ``WhatsappAccount`` column so the test
can assert a full mapping end-to-end, plus the embedded ``userId`` + ``userName``
staff pair (the owning employee) so the embedded-staff repo upsert can be
exercised against capture-shaped input (including a no-``userId`` orphan to
exercise the best-effort skip path).
"""
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped WhatsApp staff-account records (4 total across 2 pages).
# ---------------------------------------------------------------------------
WA_ACCOUNT_RECORDS: list[dict[str, Any]] = [
{
"uid": "8613800000001@s.whatsapp.net",
"userId": "wa_staff_001",
"userName": "Alice Wang",
"nickName": "Alice WA",
"phone": "8613800000001",
"salePhone": "8613800000001",
"salesBindPhone": "8613900000001",
"contactsNum": 421,
"groupNum": 12,
"talkerCount": 38,
"friendAddNum": 7,
"isOnJob": 1,
"lastUpdate": 1700000000123,
"version": "v1.20260731.alice.wa",
},
{
"uid": "8613800000002@s.whatsapp.net",
"userId": "wa_staff_002",
"userName": "Bob Chen",
"nickName": "Bob WA",
"phone": "8613800000002",
"salePhone": "8613800000002",
"salesBindPhone": "8613900000002",
"contactsNum": 58,
"groupNum": 2,
"talkerCount": 9,
"friendAddNum": 1,
"isOnJob": 1,
"lastUpdate": 1700000000456,
"version": "v1.20260731.bob.wa",
},
{
"uid": "8613800000003@s.whatsapp.net",
"userId": "wa_staff_003",
"userName": "Carol Liu",
"nickName": "Carol WA",
"phone": "8613800000003",
"salePhone": "8613800000003",
"salesBindPhone": "8613900000003",
"contactsNum": 1899,
"groupNum": 64,
"talkerCount": 210,
"friendAddNum": 33,
"isOnJob": 0, # departed staff — exercises the falsy isOnJob path
"lastUpdate": 1700000000789,
"version": "v1.20260731.carol.wa",
},
{
# Account with NO embedded staff (userId missing) — the scraper must
# still yield it; the embedded-staff repo upsert must skip it cleanly.
"uid": "8613800000004@s.whatsapp.net",
"userId": None,
"userName": None,
"nickName": "Orphan WA",
"phone": "8613800000004",
"salePhone": "8613800000004",
"salesBindPhone": "8613900000004",
"contactsNum": 3,
"groupNum": 0,
"talkerCount": 1,
"friendAddNum": 0,
"isOnJob": 0,
"lastUpdate": 1700000001000,
"version": "v1.20260731.orphan.wa",
},
]
# Default page size for the fixture (and the scraper). totalCount=4 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(WA_ACCOUNT_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_wa_accounts_response(
records: list[dict[str, Any]],
*,
total_count: int,
page: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
page_count: int | None = None,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``businessSalesList`` response envelope.
Shape::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
``page_count`` defaults to ``ceil(total_count / page_size)`` when omitted.
"""
if page_count is None:
import math
page_count = math.ceil(total_count / page_size) if page_size else 1
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-wa-acct-{page}",
"data": {
"page": page,
"pageCount": page_count,
"totalCount": total_count,
"overviews": list(records),
},
}
yunke-server/tests/cassettes/wa_friends_fixture.py
+129
View File
@@ -0,0 +1,129 @@
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/businessFriendList`` (M3).
These records mirror the real field names captured from the live CRM (see
phase0-findings.md §5; live total = **11292**):
``uid`` (owning staff ``<phone>@s.whatsapp.net``), ``userId`` / ``userName`` /
``salePhone`` / ``salesNickName`` / ``salesBindPhone`` (owning staff),
``friendUid`` (customer -> ``friend_id`` PK), ``nickName`` (customer nickname),
``phone``, ``createTime`` / ``lastUpdate``, ``groupNum`` / ``contactsNum`` /
``talkerCount`` / ``friendAddNum`` / ``messageCount`` / ``sensitiveWordNum``.
The envelope is the SAME shape as ``businessSalesList``: rows under
``data.overviews``, declared count under ``data.totalCount``, page count under
``data.pageCount``::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
The fixture's ``TOTAL`` is pinned to the record count (not the live 11292) so
the paginator's over-fetch ceiling does not fire early in the multi-page test.
"""
from __future__ import annotations
import math
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped WhatsApp contact records (3 total across 2 pages).
# ---------------------------------------------------------------------------
WA_FRIEND_RECORDS: list[dict[str, Any]] = [
{
"uid": "8613800000001@s.whatsapp.net", # owning staff account
"userId": "wa_staff_001",
"userName": "Alice Wang",
"salePhone": "8613800000001",
"salesNickName": "Alice Sale",
"salesBindPhone": "8613900000001",
"friendUid": "8613800000111@s.whatsapp.net", # customer -> friend_id
"nickName": "Customer One",
"phone": "8613800000111",
"createTime": 1699000000000,
"lastUpdate": 1700000000123,
"groupNum": 2,
"contactsNum": 421,
"talkerCount": 38,
"friendAddNum": 1,
"messageCount": 57,
"sensitiveWordNum": 0,
"version": "v1.20260731.cust1",
},
{
"uid": "8613800000001@s.whatsapp.net",
"userId": "wa_staff_001",
"userName": "Alice Wang",
"salePhone": "8613800000001",
"salesNickName": "Alice Sale",
"salesBindPhone": "8613900000001",
"friendUid": "8613800000222@s.whatsapp.net",
"nickName": "Customer Two",
"phone": "8613800000222",
"createTime": 1699100000000,
"lastUpdate": 1700000000456,
"groupNum": 1,
"contactsNum": 421,
"talkerCount": 38,
"friendAddNum": 1,
"messageCount": 9,
"sensitiveWordNum": 0,
"version": "v1.20260731.cust2",
},
{
# Contact owned by a different staff account.
"uid": "8613800000002@s.whatsapp.net",
"userId": "wa_staff_002",
"userName": "Bob Chen",
"salePhone": "8613800000002",
"salesNickName": "Bob Sale",
"salesBindPhone": "8613900000002",
"friendUid": "8613800000333@s.whatsapp.net",
"nickName": "Customer Three",
"phone": "8613800000333",
"createTime": 1699200000000,
"lastUpdate": 1700000000789,
"groupNum": 0,
"contactsNum": 58,
"talkerCount": 9,
"friendAddNum": 0,
"messageCount": 4,
"sensitiveWordNum": 1,
"version": "v1.20260731.cust3",
},
]
TOTAL = len(WA_FRIEND_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_wa_friends_response(
records: list[dict[str, Any]],
*,
total_count: int,
page: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
page_count: int | None = None,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``businessFriendList`` response envelope.
Shape (same as ``businessSalesList``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":.., "pageCount":.., "totalCount":.., "overviews":[...]}}
"""
if page_count is None:
page_count = math.ceil(total_count / page_size) if page_size else 1
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-wa-friend-{page}",
"data": {
"page": page,
"pageCount": page_count,
"totalCount": total_count,
"overviews": list(records),
},
}
yunke-server/tests/cassettes/wa_messages_fixture.py
+142
View File
@@ -0,0 +1,142 @@
"""Fixture payloads for ``POST /qqweb/WhatsAppUser/chatRecords`` (M3).
These records mirror the real field names captured from the live CRM (see the
data dictionary §3 + phase0-findings.md §6):
``keyId`` (PK), ``type`` / ``msgType`` (``0``=text, ``1``=image/media,
``-2``=date divider), ``content`` (text), ``mediaName`` (media FILENAME — NOT a
URL; deferred download), ``nickName`` (sender), ``uid`` (owning staff account),
``friendUid`` (the other party), ``isSend``, ``createTime`` / ``dateTime``,
``timezone``, ``companyCode``, ``quotedId`` / ``quotedMessage`` (quoted reply).
The request is a **JSON body** ``{uid, friendUid, start, end}`` (ms epochs) —
NOT form-encoded like WeChat ``msg/chats``. The response pages BACKWARD: each
page carries ``data.end`` (oldest timestamp in this batch) and ``data.hasLast``
(older messages exist?); the scraper advances the cursor to ``end - 1`` and
repeats until ``hasLast`` is false::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"messages":[...], "start":.., "end":.., "hasNext":.., "hasLast":..}}
The cassette is shaped to drive the backward-cursor paginator across two pages:
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
The ``-2`` record is a date divider that the scraper must DROP. ``E1`` is the
oldest timestamp in page 1's batch, so the backward-cursor advance must set the
next request's ``end`` to ``E1 - 1``.
"""
from __future__ import annotations
from typing import Any
# Oldest-timestamp-in-batch returned by each page; the scraper uses these to
# advance the backward cursor (next_request_end = page_end - 1).
E1 = 1784517338999
E2 = 1784517000000
# Page 1 records, EXACT live-capture field names. m1 is text/incoming,
# m_datebar is a -2 divider (must be filtered), m2 is image/outgoing.
WA_MESSAGE_RECORDS_PAGE1: list[dict[str, Any]] = [
{ # m1 — text message, incoming.
"keyId": "wa_1000001",
"type": 0,
"msgType": 0,
"content": "hello from whatsapp",
"nickName": "Customer One",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 0,
"createTime": E1 + 600000,
"dateTime": E1 + 600000,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"version": "v1.0",
},
{ # date divider — type == -2; scraper MUST drop this, never yield it.
"keyId": "wa_1000000",
"type": -2,
"msgType": -2,
"content": "2026-01-01",
"nickName": "",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 0,
"createTime": E1 + 300000,
"dateTime": E1 + 300000,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"version": "v1.0",
},
{ # m2 — image message, outgoing, with a media filename (deferred download).
"keyId": "wa_1000002",
"type": 1,
"msgType": 1,
"content": "[image]",
"mediaName": "8613800000001@s.whatsapp.net_wa_1000002.jpg",
"nickName": "Alice Wang",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 1,
"createTime": E1 + 100,
"dateTime": E1 + 100,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"version": "v1.0",
},
]
# Page 2 record — text message with a quoted reply; oldest in the window.
WA_MESSAGE_RECORDS_PAGE2: list[dict[str, Any]] = [
{
"keyId": "wa_999999",
"type": 0,
"msgType": 0,
"content": "replying to an earlier message",
"nickName": "Customer One",
"uid": "8613800000001@s.whatsapp.net",
"friendUid": "8613800000111@s.whatsapp.net",
"isSend": 0,
"createTime": E2,
"dateTime": E2,
"timezone": "Asia/Shanghai",
"companyCode": "8m0atd",
"quotedId": "wa_888888",
"quotedMessage": "the earlier message text",
"version": "v1.0",
},
]
def make_wa_messages_response(
messages: list[dict[str, Any]],
*,
end: int,
has_last: bool,
start: int = 1,
has_next: bool = False,
trace_id: str = "trace-wa-msg-1",
) -> dict[str, Any]:
"""Build a capture-shaped ``chatRecords`` response envelope.
Shape (EXACT, per M3 spec)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"messages":[...], "start":.., "end":<oldest ts in batch>,
"hasNext":<bool>, "hasLast":<bool — are there older messages?>}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id,
"data": {
"messages": list(messages),
"start": start,
"end": end,
"hasNext": has_next,
"hasLast": has_last,
},
}
yunke-server/tests/test_parsers.py
+64 -1
View File
@@ -12,7 +12,11 @@ from __future__ import annotations
import pytest
from app.scraper.parsers import normalize_wechat_type
from app.scraper.parsers import (
coerce_int_or_none,
normalize_wechat_type,
normalize_whatsapp_type,
)
@pytest.mark.parametrize(
@@ -52,3 +56,62 @@ def test_normalize_wechat_type_never_raises():
out = normalize_wechat_type(raw)
assert isinstance(out, str)
assert out == "unknown"
# ===========================================================================
# WhatsApp message-type normalization (M3)
# ===========================================================================
@pytest.mark.parametrize(
"raw,expected",
[
# Phase-0 confirmed WhatsApp codes (see data dictionary §3):
(0, "text"),
(1, "image"), # image / media
(-2, "divider"), # date separator — NOT a real message (scraper drops it)
# Unconfirmed / other media codes — pass through as "unknown", never dropped.
(2, "unknown"),
(3, "unknown"),
(13, "unknown"),
(100, "unknown"),
# Stringified ints (the upstream API may send either form).
("0", "text"),
("1", "image"),
("-2", "divider"),
("2", "unknown"),
],
)
def test_normalize_whatsapp_type_maps_known_and_unknown(raw, expected):
assert normalize_whatsapp_type(raw) == expected
def test_normalize_whatsapp_type_never_raises():
"""Whatever we throw at it, it returns a string — never raises."""
for raw in (None, "", "junk", "1.5", [], {}, object(), float("nan")):
out = normalize_whatsapp_type(raw)
assert isinstance(out, str)
assert out == "unknown"
# ===========================================================================
# coerce_int_or_none — shared scraper helper (M3)
# ===========================================================================
def test_coerce_int_or_none_handles_documented_cases():
"""``type_raw``-style fields: str -> int; None preserved; 0 preserved."""
assert coerce_int_or_none(None) is None
assert coerce_int_or_none(0) == 0
assert coerce_int_or_none(2) == 2
assert coerce_int_or_none(-2) == -2
assert coerce_int_or_none("0") == 0
assert coerce_int_or_none("2") == 2
assert coerce_int_or_none("-2") == -2
def test_coerce_int_or_none_returns_none_for_non_numeric_string():
"""A non-numeric value (e.g. a label) -> None."""
assert coerce_int_or_none("text") is None
assert coerce_int_or_none("") is None
assert coerce_int_or_none("unknown") is None
yunke-server/tests/test_repos.py
+215
View File
@@ -31,6 +31,7 @@ from app.db.models import (
PhoneCall,
Staff,
WechatAccount,
WhatsappAccount,
)
from app.storage.repos import (
_coerce_bool,
@@ -44,6 +45,7 @@ from app.storage.repos import (
upsert_staff_from_account,
upsert_voice_call,
upsert_wechat_account,
upsert_whatsapp_account,
)
# ---------------------------------------------------------------------------
@@ -1004,3 +1006,216 @@ async def test_upsert_voice_call_null_record_file_and_coerces_ints():
assert fetched.incoming_call == 1
finally:
await engine.dispose()
# ---------------------------------------------------------------------------
# WhatsappAccount + embedded Staff upsert (M3)
# ---------------------------------------------------------------------------
async def test_upsert_whatsapp_account_writes_account_and_embedded_staff():
"""WhatsappAccount upsert creates the account row AND the embedded Staff row."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"uid": "8613800000001@s.whatsapp.net",
"user_id": "wa_u1",
"user_name": "Alice Wang",
"nick_name": "Alice WA",
"phone": "8613800000001",
"sale_phone": "8613800000001",
"sales_bind_phone": "8613900000001",
"contacts_num": 421,
"group_num": 12,
"talker_count": 38,
"friend_add_num": 7,
"is_on_job": 1,
"last_update": 1700000000123,
}
await upsert_whatsapp_account(session, rec)
await session.commit()
acct = (
await session.execute(
sa.select(WhatsappAccount).where(
WhatsappAccount.uid == "8613800000001@s.whatsapp.net"
)
)
).scalar_one()
assert acct.user_id == "wa_u1"
assert acct.user_name == "Alice Wang"
assert acct.sale_phone == "8613800000001"
assert acct.sales_bind_phone == "8613900000001"
assert acct.contacts_num == 421
assert acct.is_on_job is True
assert acct.last_update == 1700000000123
# Embedded staff row created (PK = user_id), name forwarded.
staff = (
await session.execute(
sa.select(Staff).where(Staff.user_id == "wa_u1")
)
).scalar_one()
assert staff.name == "Alice Wang"
assert staff.is_on_job is True
assert staff.sale_phone == "8613800000001"
finally:
await engine.dispose()
async def test_upsert_whatsapp_account_is_idempotent_and_refreshes_staff():
"""Re-upserting the account refreshes BOTH account + embedded staff rows."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec1 = {
"uid": "8613800000002@s.whatsapp.net",
"user_id": "wa_u2",
"user_name": "Old Name",
"nick_name": "Old Nick",
"is_on_job": 1,
}
rec2 = {
"uid": "8613800000002@s.whatsapp.net",
"user_id": "wa_u2",
"user_name": "New Name",
"nick_name": "New Nick",
"is_on_job": 0, # staff departed since last scrape
}
await upsert_whatsapp_account(session, rec1)
await session.commit()
await upsert_whatsapp_account(session, rec2)
await session.commit()
# One account row, refreshed.
acct_rows = (
await session.execute(
sa.select(WhatsappAccount)
.where(WhatsappAccount.uid == "8613800000002@s.whatsapp.net")
.execution_options(populate_existing=True)
)
).scalars().all()
assert len(acct_rows) == 1
assert acct_rows[0].nick_name == "New Nick"
# One staff row, refreshed (not duplicated).
staff_rows = (
await session.execute(
sa.select(Staff)
.where(Staff.user_id == "wa_u2")
.execution_options(populate_existing=True)
)
).scalars().all()
assert len(staff_rows) == 1
assert staff_rows[0].name == "New Name"
assert staff_rows[0].is_on_job is False
finally:
await engine.dispose()
async def test_upsert_whatsapp_account_without_user_id_skips_staff():
"""Account with no user_id is written; embedded-staff upsert is skipped."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"uid": "8613800000009@s.whatsapp.net",
"nick_name": "Orphan WA",
# no user_id / user_name
}
await upsert_whatsapp_account(session, rec)
await session.commit()
# Account row written.
acct = (
await session.execute(
sa.select(WhatsappAccount).where(
WhatsappAccount.uid == "8613800000009@s.whatsapp.net"
)
)
).scalar_one()
assert acct.nick_name == "Orphan WA"
# No staff rows at all.
staff_rows = (
await session.execute(sa.select(Staff))
).scalars().all()
assert staff_rows == []
finally:
await engine.dispose()
async def test_upsert_customer_with_whatsapp_channel():
"""WA contacts flow through the channel-generic upsert_customer.
The ``Customer`` table is shared across WeChat / WhatsApp / SMS peers; the
per-row ``channel`` value disambiguates them. This pins that a WA-shaped
record (``channel="whatsapp"``) upserts cleanly against the same PK contract.
"""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"friend_id": "8613800000111@s.whatsapp.net",
"channel": "whatsapp",
"owner_account_id": "8613800000001@s.whatsapp.net",
"friend_nickname": "Customer One",
"phone": "8613800000111",
"sale_name": "Alice Sale",
"user_id": "wa_staff_001",
"message_count": 57,
}
await upsert_customer(session, rec)
await session.commit()
fetched = (
await session.execute(
sa.select(Customer).where(
Customer.friend_id == "8613800000111@s.whatsapp.net"
)
)
).scalar_one()
assert fetched.channel == "whatsapp"
assert fetched.owner_account_id == "8613800000001@s.whatsapp.net"
assert fetched.friend_nickname == "Customer One"
assert fetched.sale_name == "Alice Sale"
assert fetched.message_count == 57
finally:
await engine.dispose()
async def test_upsert_messages_with_whatsapp_channel():
"""WA messages flow through the channel-generic upsert_messages."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"msg_id": "wa_msg_1",
"channel": "whatsapp",
"owner_account_id": "8613800000001@s.whatsapp.net",
"talker_id": "8613800000111@s.whatsapp.net",
"is_send": 0,
"type": "text",
"type_raw": 0,
"text": "hi",
}
await upsert_messages(session, [rec])
await session.commit()
fetched = (
await session.execute(
sa.select(Message).where(Message.msg_id == "wa_msg_1")
)
).scalar_one()
assert fetched.channel == "whatsapp"
assert fetched.type == "text"
assert fetched.type_raw == 0
assert fetched.is_send is False
finally:
await engine.dispose()
yunke-server/tests/test_scrape_whatsapp_accounts.py
+217
View File
@@ -0,0 +1,217 @@
"""Tests for the WhatsApp staff-account scraper (M3).
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal for page-number endpoints):
* page 1 -> 2 records, ``totalCount=4`` (rows under ``data.overviews``)
* page 2 -> 2 records, ``totalCount=4``
* page 3 -> empty ``overviews`` array (no more rows)
The scraper must yield exactly the 4 records (normalized to ``WhatsappAccount``
snake_case columns + ``raw_json`` + the embedded staff pair), make exactly 3
calls, and stop without a 4th. The empty-page stop is the primary signal;
``totalCount=4`` is only a defensive over-fetch ceiling that must NOT trigger
early (we yield exactly 4 == total, so the strict-``>`` ceiling never fires).
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.whatsapp import scrape_whatsapp_accounts
from app.session.yunke_client import YunkeClient
from tests.cassettes.wa_accounts_fixture import (
DEFAULT_PAGE_SIZE,
TOTAL,
WA_ACCOUNT_RECORDS,
make_wa_accounts_response,
)
BASE = "https://crm2.yunkecn.com"
WA_ACCOUNTS_URL = f"{BASE}/qqweb/WhatsAppUser/businessSalesList"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + embedded staff + raw_json
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_accounts_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
page1 = make_wa_accounts_response(
WA_ACCOUNT_RECORDS[:2], total_count=TOTAL, page=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_wa_accounts_response(
WA_ACCOUNT_RECORDS[2:], total_count=TOTAL, page=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_wa_accounts_response(
[], total_count=TOTAL, page=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(WA_ACCOUNTS_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_whatsapp_accounts(client, page_size=DEFAULT_PAGE_SIZE)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 4 fixture records (not duplicated, not dropped).
assert len(results) == 4
assert {r["uid"] for r in results} == {rec["uid"] for rec in WA_ACCOUNT_RECORDS}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, WA_ACCOUNT_RECORDS, strict=True):
# Identity PK.
assert result["uid"] == record["uid"]
# Embedded staff pair (the owning employee).
assert result["user_id"] == record["userId"]
assert result["user_name"] == record["userName"]
# Naming + phones.
assert result["nick_name"] == record["nickName"]
assert result["phone"] == record["phone"]
assert result["sale_phone"] == record["salePhone"]
assert result["sales_bind_phone"] == record["salesBindPhone"]
# Counts.
assert result["contacts_num"] == record["contactsNum"]
assert result["group_num"] == record["groupNum"]
assert result["talker_count"] == record["talkerCount"]
assert result["friend_add_num"] == record["friendAddNum"]
# Status / timestamps.
assert result["is_on_job"] == record["isOnJob"]
assert result["last_update"] == record["lastUpdate"]
# Version carried verbatim; raw_json preserves the original record.
assert result["version"] == record["version"]
assert result["raw_json"] == record
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_accounts_sends_pagination_body():
"""Outgoing request body carries pageNum/pageSize; advances per page."""
page1 = make_wa_accounts_response(
WA_ACCOUNT_RECORDS[:1], total_count=TOTAL, page=1, page_size=1
)
page2 = make_wa_accounts_response([], total_count=TOTAL, page=2, page_size=1)
route = respx.post(WA_ACCOUNTS_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [r async for r in scrape_whatsapp_accounts(client, page_size=1)]
finally:
await client.aclose()
# One record on page 1, empty page 2 stops the loop.
assert len(results) == 1
assert route.call_count == 2
# First request: pageNum=1.
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
# Second request: pageNum advanced to 2.
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_accounts_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_wa_accounts_response([], total_count=0, page=1)
route = respx.post(WA_ACCOUNTS_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_whatsapp_accounts(client)]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1
# ---------------------------------------------------------------------------
# Envelope: rows are read from data.overviews (NOT data.data)
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_accounts_reads_overviews_envelope():
"""Rows under ``data.overviews`` are picked up (not the WeChat ``data.data``)."""
# A response that ALSO carries an empty ``data.data`` list must not confuse
# the scraper — it must read ``overviews`` and yield the record.
page = make_wa_accounts_response(
WA_ACCOUNT_RECORDS[:1], total_count=1, page=1, page_size=1
)
page["data"]["data"] = [] # WeChat-style key present but empty
respx.post(WA_ACCOUNTS_URL).mock(return_value=httpx.Response(200, json=page))
# Serve an empty page next so the paginator stops.
respx.post(WA_ACCOUNTS_URL).mock(
side_effect=[
httpx.Response(200, json=page),
httpx.Response(
200,
json=make_wa_accounts_response([], total_count=1, page=2, page_size=1),
),
]
)
client = _primed_client()
try:
results = [r async for r in scrape_whatsapp_accounts(client, page_size=1)]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["uid"] == WA_ACCOUNT_RECORDS[0]["uid"]
yunke-server/tests/test_scrape_whatsapp_friends.py
+169
View File
@@ -0,0 +1,169 @@
"""Tests for the WhatsApp contact-list scraper (M3).
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 3 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal):
* page 1 -> 2 records, ``totalCount=3`` (rows under ``data.overviews``)
* page 2 -> 1 record, ``totalCount=3``
* page 3 -> empty ``overviews`` array (no more rows)
The scraper must yield exactly the 3 records normalized to the ``Customer``
column shape with ``channel="whatsapp"``, ``friend_id`` <- ``friendUid`` and
``owner_account_id`` <- ``uid``.
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.whatsapp import scrape_whatsapp_friends
from app.session.yunke_client import YunkeClient
from tests.cassettes.wa_friends_fixture import (
DEFAULT_PAGE_SIZE,
TOTAL,
WA_FRIEND_RECORDS,
make_wa_friends_response,
)
BASE = "https://crm2.yunkecn.com"
WA_FRIENDS_URL = f"{BASE}/qqweb/WhatsAppUser/businessFriendList"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + channel/ownership + raw_json
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_friends_yields_normalized_records_and_paginates():
"""Two full pages + a terminating empty page -> 3 records, 3 calls."""
page1 = make_wa_friends_response(
WA_FRIEND_RECORDS[:2], total_count=TOTAL, page=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_wa_friends_response(
WA_FRIEND_RECORDS[2:], total_count=TOTAL, page=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_wa_friends_response(
[], total_count=TOTAL, page=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(WA_FRIENDS_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r async for r in scrape_whatsapp_friends(client, page_size=DEFAULT_PAGE_SIZE)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 3 fixture records (not duplicated, not dropped).
assert len(results) == 3
assert {r["friend_id"] for r in results} == {
rec["friendUid"] for rec in WA_FRIEND_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, WA_FRIEND_RECORDS, strict=True):
# Identity / ownership / channel.
assert result["friend_id"] == record["friendUid"]
assert result["channel"] == "whatsapp"
assert result["owner_account_id"] == record["uid"]
assert result["user_id"] == record["userId"]
# Naming / phone.
assert result["friend_nickname"] == record["nickName"]
assert result["phone"] == record["phone"]
assert result["sale_name"] == record["salesNickName"]
# Counts / timestamps.
assert result["message_count"] == record["messageCount"]
assert result["last_chat_time"] == record["lastUpdate"]
# Version carried verbatim; raw_json preserves the original record.
assert result["version"] == record["version"]
assert result["raw_json"] == record
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_friends_sends_pagination_body():
"""Outgoing request body carries pageNum/pageSize; advances per page."""
page1 = make_wa_friends_response(
WA_FRIEND_RECORDS[:1], total_count=TOTAL, page=1, page_size=1
)
page2 = make_wa_friends_response([], total_count=TOTAL, page=2, page_size=1)
route = respx.post(WA_FRIENDS_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [r async for r in scrape_whatsapp_friends(client, page_size=1)]
finally:
await client.aclose()
assert len(results) == 1
assert route.call_count == 2
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_friends_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_wa_friends_response([], total_count=0, page=1)
route = respx.post(WA_FRIENDS_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_whatsapp_friends(client)]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1
yunke-server/tests/test_scrape_whatsapp_messages.py
+208
View File
@@ -0,0 +1,208 @@
"""Tests for the WhatsApp message scraper (M3).
Drives the real backward-cursor pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport. Two pages are served:
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
The scraper must yield exactly ``[m1, m2, m3]`` (the ``-2`` divider dropped),
map fields per the live-capture names, set ``channel="whatsapp"``, send a
**JSON** body (``uid``/``friendUid``/``start``/``end``), AND the 2nd request
body's ``end`` must equal ``E1 - 1`` (proving the backward cursor advances by
one ms).
"""
from __future__ import annotations
import json as _json
from typing import Any
import httpx
import respx
from app.scraper.whatsapp import scrape_whatsapp_messages
from app.session.yunke_client import YunkeClient
from tests.cassettes.wa_messages_fixture import (
E1,
E2,
WA_MESSAGE_RECORDS_PAGE1,
WA_MESSAGE_RECORDS_PAGE2,
make_wa_messages_response,
)
BASE = "https://crm2.yunkecn.com"
WA_MESSAGES_URL = f"{BASE}/qqweb/WhatsAppUser/chatRecords"
USERID = "a" * 32
COMPANY = "8m0atd"
UID = "8613800000001@s.whatsapp.net"
FRIEND_UID = "8613800000111@s.whatsapp.net"
START = 1
END = 1784518000000 # the initial upper-bound cursor
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
def _json_body(sent: httpx.Request) -> dict[str, Any]:
"""Parse the JSON body of an outgoing request into a dict."""
return _json.loads(sent.content.decode())
# ---------------------------------------------------------------------------
# Test A: end-to-end backward-cursor pagination + divider drop + field mapping
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_messages_drops_divider_and_paginates_backward():
page1 = make_wa_messages_response(
WA_MESSAGE_RECORDS_PAGE1, end=E1, has_last=True, trace_id="trace-wa-1"
)
page2 = make_wa_messages_response(
WA_MESSAGE_RECORDS_PAGE2, end=E2, has_last=False, trace_id="trace-wa-2"
)
route = respx.post(WA_MESSAGES_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_whatsapp_messages(
client, UID, FRIEND_UID, START, END
)
]
finally:
await client.aclose()
# --- Exactly 2 calls: hasLast=false on page 2 means no 3rd fetch. ---
assert route.call_count == 2
# --- Exactly [m1, m2, m3] yielded (the -2 divider dropped). ---
assert [r["msg_id"] for r in results] == ["wa_1000001", "wa_1000002", "wa_999999"]
assert len(results) == 3
r1, r2, r3 = results
# m1 — text / incoming.
assert r1["msg_id"] == "wa_1000001"
assert r1["channel"] == "whatsapp"
assert r1["owner_account_id"] == UID
assert r1["talker_id"] == FRIEND_UID
assert r1["is_send"] == 0
assert r1["type"] == "text" # code 0 -> text (phase0 §3)
assert r1["type_raw"] == 0 # original raw code preserved
assert r1["text"] == "hello from whatsapp"
assert r1["timestamp"] == E1 + 600000
assert r1["sender_name"] == "Customer One"
assert r1["raw_json"] == WA_MESSAGE_RECORDS_PAGE1[0]
# m2 — image / outgoing / media filename (deferred download, not a URL).
assert r2["msg_id"] == "wa_1000002"
assert r2["is_send"] == 1
assert r2["type"] == "image" # code 1 -> image/media (phase0 §3)
assert r2["type_raw"] == 1
assert r2["text"] == "[image]"
assert r2["file_url"] == "8613800000001@s.whatsapp.net_wa_1000002.jpg"
assert r2["raw_json"] == WA_MESSAGE_RECORDS_PAGE1[2]
# m3 — text / incoming / quoted reply.
assert r3["msg_id"] == "wa_999999"
assert r3["type"] == "text"
assert r3["type_raw"] == 0
assert r3["text"] == "replying to an earlier message"
assert r3["refer_msg"] == {
"quotedId": "wa_888888",
"quotedMessage": "the earlier message text",
}
assert r3["raw_json"] == WA_MESSAGE_RECORDS_PAGE2[0]
# --- Backward cursor advance: 2nd request body's end == E1 - 1. ---
first_req = route.calls[0].request
second_req = route.calls[1].request
body1 = _json_body(first_req)
body2 = _json_body(second_req)
assert body1["end"] == END
assert body2["end"] == E1 - 1
# --- JSON body is actually sent (NOT form-encoded). ---
assert first_req.headers["content-type"] == "application/json"
assert body1["uid"] == UID
assert body1["friendUid"] == FRIEND_UID
assert body1["start"] == START
assert body1["end"] == END
# ``start`` is constant across pages (only ``end`` walks backward).
assert body2["start"] == START
assert body2["uid"] == UID
assert body2["friendUid"] == FRIEND_UID
# ---------------------------------------------------------------------------
# Test B: hasLast=false on first page -> stop after one call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_messages_stops_when_has_last_false_on_first_page():
"""A single page with hasLast=false: scraper yields the records and stops."""
page = make_wa_messages_response(
[WA_MESSAGE_RECORDS_PAGE2[0]], end=E2, has_last=False
)
route = respx.post(WA_MESSAGES_URL).mock(
return_value=httpx.Response(200, json=page)
)
client = _primed_client()
try:
results = [
r
async for r in scrape_whatsapp_messages(
client, UID, FRIEND_UID, START, END
)
]
finally:
await client.aclose()
assert [r["msg_id"] for r in results] == ["wa_999999"]
assert route.call_count == 1
# ---------------------------------------------------------------------------
# Test C: empty messages array on the first page -> stop without a 3rd fetch
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_whatsapp_messages_empty_first_page_yields_nothing():
"""An empty messages array is a defensive stop signal — no rows, one call."""
page = make_wa_messages_response([], end=E1, has_last=True)
route = respx.post(WA_MESSAGES_URL).mock(
return_value=httpx.Response(200, json=page)
)
client = _primed_client()
try:
results = [
r
async for r in scrape_whatsapp_messages(
client, UID, FRIEND_UID, START, END
)
]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1