xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: wechat group+member and talker scrapers

Browse Source 
Add scrape_wechat_groups (POST /wechatweb/chatroom/getChatroomList) and
scrape_wechat_talkers (POST /wechatweb/wxFriends/talkerList), both reusing
the page-number paginator. Groups parse the semicolon-joined memberIds into
a member_ids_list so upsert_chat_group can mirror GroupMember rows
(composite PK channel/room_id/member_id) idempotently alongside the
ChatGroup row. Talkers are a sync-planning input (member_count>0 marks a
group conversation); no dedicated table.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 13:54:16 +08:00
parent 2b79e7a232
commit 2a74d3888d
7 changed files with 1163 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/tests/cassettes/chatroom_fixture.py
+137
View File
@@ -0,0 +1,137 @@
"""Fixture payloads for ``POST /wechatweb/chatroom/getChatroomList`` (M2).
These records mirror the real field names captured from the live CRM (see the
data dictionary §2: ``chatRoomId(@chatroom), chatRoomName, memberCount,
memberIds(分号分隔全部成员wxid), headImageUrl, wechatId/wechatNickName(归属),
userName/userPhone(群主员工), messageCount/ownMessageCount, lossMembers/
addMembers, lastChatTime/updateTime``; total=1515).
The envelope is the same flat shape as ``wxUser/list`` (no ``basicResult``
wrapper): pagination fields sit directly under ``data``::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "page":.., "pageCount":..}}
The cassette ships three records across two pages so a test can drive the
page-number paginator across the empty-page stop signal:
* page 1 -> 2 of the 3 fixture records, ``total=3``
* page 2 -> the remaining 1 record, ``total=3``
* (page 3 -> empty ``data`` array, served inline by the test)
Records carry every field with a target ``ChatGroup`` column so the test can
assert a full mapping end-to-end. The ``memberIds`` field is the
semicolon-joined member wxid list; record 2 deliberately includes a trailing
empty segment (``"...;"``) so the parser's drop-empties path is exercised.
"""
# The module name mirrors the upstream endpoint name (``chatroom``); the M2
# spec mandates this exact filename.
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped chatroom records (3 total across 2 pages).
# ---------------------------------------------------------------------------
CHATROOM_RECORDS: list[dict[str, Any]] = [
{
"chatRoomId": "12345678901@chatroom",
"chatRoomName": "VIP客户交流群",
"memberCount": 3,
# 3 members, no empties -> member_ids_list length 3.
"memberIds": "wxid_member_a1;wxid_member_a2;csf7169708",
"headImageUrl": "https://wx.qlogo.cn/room_12345678901.png",
"wechatId": "wxid_staff_alice01",
"wechatNickName": "Alice WeChat",
"userId": "staff_user_001",
"userName": "Alice Wang",
"userPhone": "13800000001",
"messageCount": 4521,
"ownMessageCount": 1204,
"lossMembers": 2,
"addMembers": 5,
"lastChatTime": 1700000000123,
"updateTime": 1700000001000,
"version": "v1.20260731.room1",
},
{
"chatRoomId": "98765432109@chatroom",
"chatRoomName": "华东销售大群",
"memberCount": 2,
# Trailing empty segment -> member_ids_list length 2 (empties dropped).
"memberIds": "wxid_member_b1;wxid_member_b2;",
"headImageUrl": "https://wx.qlogo.cn/room_98765432109.png",
"wechatId": "wxid_staff_bob02",
"wechatNickName": "Bob WeChat",
"userId": "staff_user_002",
"userName": "Bob Chen",
"userPhone": "13800000002",
"messageCount": 9821,
"ownMessageCount": 211,
"lossMembers": 0,
"addMembers": 1,
"lastChatTime": 1700000000456,
"updateTime": 1700000000800,
"version": "v1.20260731.room2",
},
{
# A group whose owner has no resolved staff id (userId None) — the
# scraper must still yield it; the repo must still upsert its members.
"chatRoomId": "55555555555@chatroom",
"chatRoomName": "External Coordination",
"memberCount": 1,
"memberIds": "wxid_member_c1",
"headImageUrl": "https://wx.qlogo.cn/room_55555555555.png",
"wechatId": "wxid_staff_orphan04",
"wechatNickName": "Orphan WeChat",
"userId": None,
"userName": None,
"userPhone": None,
"messageCount": 12,
"ownMessageCount": 0,
"lossMembers": 0,
"addMembers": 0,
"lastChatTime": 1700000000789,
"updateTime": 1700000000900,
"version": "v1.20260731.room3",
},
]
# Default page size for the fixture (and the scraper). total=3 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(CHATROOM_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_chatroom_list_response(
records: list[dict[str, Any]],
*,
total: int,
page_num: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``getChatroomList`` response envelope.
Shape (no ``basicResult`` wrapper; pagination fields directly under
``data``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "page":.., "pageCount":..}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-{page_num}",
"data": {
"data": list(records),
"total": total,
"page": page_num,
"pageCount": (
(total + page_size - 1) // page_size if page_size else 1
),
},
}
yunke-server/tests/cassettes/talkerList_fixture.py
+112
View File
@@ -0,0 +1,112 @@
"""Fixture payloads for ``POST /wechatweb/wxFriends/talkerList`` (M2).
These records mirror the real field names captured from the live CRM (see the
data dictionary §2 会话列表 talker: ``friendWechatId, nickName, remarkName,
messageCount, lastChatTime, headUrl, memberCount, analyseResult``). The
endpoint is **per-staff-wechat**: the request body carries the ``wechatId`` of
the owning staff account, and the response lists that account's recent
conversations (direct chats AND group chats — ``memberCount > 0`` ⇒ group).
The envelope carries the row list under ``data.page`` with a boolean
``data.hasMore`` flag (no ``total`` field is documented for this endpoint)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":[ {<talker record>}, ... ], "hasMore":..}}
Because the page-number paginator reuses the empty-page stop signal, the
cassette ships the records across two pages plus a terminating empty page:
* page 1 -> 2 of the 3 fixture talks, ``hasMore=true``
* page 2 -> the remaining 1 talk, ``hasMore=false``
* (page 3 -> empty ``page`` array, served inline by the test)
Records exercise both the direct-message path (``memberCount == 0``) and the
group-chat path (``memberCount > 0``) so the orchestrator can distinguish them.
"""
# The module name uses camelCase deliberately to mirror the upstream endpoint
# name (``talkerList``); the M2 spec mandates this exact filename.
# ruff: noqa: N999
from __future__ import annotations
from typing import Any
# The owning staff wechat for this talker stream (the ``wechatId`` request
# field). All records below belong to this account's conversation list.
OWNER_WECHAT_ID = "wxid_staff_alice01"
# ---------------------------------------------------------------------------
# Capture-shaped talker (conversation) records (3 total across 2 pages).
# ---------------------------------------------------------------------------
TALKER_RECORDS: list[dict[str, Any]] = [
{
# Direct message (1:1 friend). memberCount == 0.
"friendWechatId": "wxid_friend_0001",
"nickName": "Alice Wang",
"remarkName": "Alice-VIP",
"messageCount": 46,
"lastChatTime": 1700000000123,
"headUrl": "https://wx.qlogo.cn/avatar_alice.png",
"memberCount": 0,
"displayName": "",
"analyseResult": "",
},
{
# Group chat. memberCount > 0 ⇒ the friendWechatId is a @chatroom id.
"friendWechatId": "12345678901@chatroom",
"nickName": "VIP客户交流群",
"remarkName": "",
"messageCount": 4521,
"lastChatTime": 1700000000456,
"headUrl": "https://wx.qlogo.cn/room_12345678901.png",
"memberCount": 3,
"displayName": "Alice (group)",
"analyseResult": "",
},
{
# Another direct message — a less active contact.
"friendWechatId": "wxid_friend_0009",
"nickName": "Quiet Lead",
"remarkName": "",
"messageCount": 2,
"lastChatTime": 1700000000789,
"headUrl": "https://wx.qlogo.cn/avatar_quiet.png",
"memberCount": 0,
"displayName": "",
"analyseResult": "",
},
]
# Default page size for the fixture (and the scraper). 3 records fill exactly
# 2 pages of size 2 plus a terminating empty page.
TOTAL = len(TALKER_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_talker_list_response(
records: list[dict[str, Any]],
*,
page_num: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
has_more: bool = False,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``talkerList`` response envelope.
Shape (row list under ``data.page``, boolean ``data.hasMore``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"page":[...], "hasMore":..}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-{page_num}",
"data": {
"page": list(records),
"hasMore": has_more,
},
}
yunke-server/tests/test_repos.py
+185 -1
View File
@@ -22,10 +22,19 @@ import sqlalchemy as sa
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.db.base import Base
from app.db.models import Customer, Department, Message, Staff, WechatAccount
from app.db.models import (
ChatGroup,
Customer,
Department,
GroupMember,
Message,
Staff,
WechatAccount,
)
from app.storage.repos import (
_coerce_bool,
_coerce_int,
upsert_chat_group,
upsert_customer,
upsert_customers,
upsert_department,
@@ -701,3 +710,178 @@ async def test_upsert_wechat_account_without_user_id_skips_staff():
assert staff_rows == []
finally:
await engine.dispose()
# ---------------------------------------------------------------------------
# ChatGroup + GroupMember upsert (M2)
# ---------------------------------------------------------------------------
async def test_upsert_chat_group_inserts_group_and_members():
"""Upsert writes the ChatGroup row AND one GroupMember per member_ids_list."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"room_id": "12345678901@chatroom",
"channel": "wechat",
"room_name": "VIP客户交流群",
"owner_account_id": "wxid_staff_alice01",
"member_count": 3,
"member_ids": "wxid_m1;wxid_m2;wxid_m3",
"member_ids_list": ["wxid_m1", "wxid_m2", "wxid_m3"],
"message_count": 4521,
}
await upsert_chat_group(session, rec)
await session.commit()
group = (
await session.execute(
sa.select(ChatGroup).where(
ChatGroup.room_id == "12345678901@chatroom"
)
)
).scalar_one()
assert group.room_name == "VIP客户交流群"
assert group.owner_account_id == "wxid_staff_alice01"
assert group.member_count == 3
assert group.message_count == 4521
# The raw semicolon-joined string is stored on the group row.
assert group.member_ids == "wxid_m1;wxid_m2;wxid_m3"
# Members: one row per member_ids_list entry, composite PK.
members = (
await session.execute(
sa.select(GroupMember)
.where(GroupMember.room_id == "12345678901@chatroom")
.order_by(GroupMember.member_id)
)
).scalars().all()
assert len(members) == 3
assert {m.member_id for m in members} == {"wxid_m1", "wxid_m2", "wxid_m3"}
assert {m.channel for m in members} == {"wechat"}
# Phase-0: per-member nickname/avatar are not yet populated.
assert all(m.nickname is None for m in members)
assert all(m.avatar is None for m in members)
finally:
await engine.dispose()
async def test_upsert_chat_group_reupsert_updates_group_keeps_members_unique():
"""Re-upserting a group refreshes its row; members stay unique (no dups)."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec1 = {
"room_id": "98765432109@chatroom",
"channel": "wechat",
"room_name": "Old Name",
"member_count": 2,
"member_ids": "wxid_m1;wxid_m2",
"member_ids_list": ["wxid_m1", "wxid_m2"],
"message_count": 10,
}
await upsert_chat_group(session, rec1)
await session.commit()
rec2 = dict(
rec1,
room_name="New Name",
message_count=99,
)
await upsert_chat_group(session, rec2)
await session.commit()
# One group row, refreshed (name + count updated, not duplicated).
groups = (
await session.execute(
sa.select(ChatGroup)
.where(ChatGroup.room_id == "98765432109@chatroom")
.execution_options(populate_existing=True)
)
).scalars().all()
assert len(groups) == 1
assert groups[0].room_name == "New Name"
assert groups[0].message_count == 99
# Members still exactly 2 (idempotent re-upsert, no duplication).
members = (
await session.execute(
sa.select(GroupMember).where(
GroupMember.room_id == "98765432109@chatroom"
)
)
).scalars().all()
assert len(members) == 2
assert {m.member_id for m in members} == {"wxid_m1", "wxid_m2"}
finally:
await engine.dispose()
async def test_upsert_chat_group_drops_empty_member_id():
"""An empty member id in member_ids_list is dropped (no empty-PK row).
The scraper already drops empties when building ``member_ids_list``; this
test exercises the repo's own defensive guard so a hand-built record that
slips an empty id through cannot create a degenerate empty-PK member row.
"""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"room_id": "55555555555@chatroom",
"channel": "wechat",
# Empty string sneaks through; repo must skip it.
"member_ids_list": ["", "wxid_only", ""],
}
await upsert_chat_group(session, rec)
await session.commit()
members = (
await session.execute(
sa.select(GroupMember).where(
GroupMember.room_id == "55555555555@chatroom"
)
)
).scalars().all()
assert len(members) == 1
assert members[0].member_id == "wxid_only"
finally:
await engine.dispose()
async def test_upsert_chat_group_with_no_members_writes_group_only():
"""A group with an empty member list still writes the group row."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"room_id": "empty@chatroom",
"channel": "wechat",
"room_name": "No Members Yet",
"member_ids_list": [],
}
await upsert_chat_group(session, rec)
await session.commit()
# Group row written.
group = (
await session.execute(
sa.select(ChatGroup).where(ChatGroup.room_id == "empty@chatroom")
)
).scalar_one()
assert group.room_name == "No Members Yet"
# No member rows.
members = (
await session.execute(
sa.select(GroupMember).where(GroupMember.room_id == "empty@chatroom")
)
).scalars().all()
assert members == []
finally:
await engine.dispose()
yunke-server/tests/test_scrape_wechat_groups.py
+231
View File
@@ -0,0 +1,231 @@
"""Tests for the WeChat chat-group scraper (M2).
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 3 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal for page-number endpoints):
* page 1 -> 2 records, ``total=3``
* page 2 -> 1 record, ``total=3``
* page 3 -> empty ``data`` array (no more rows)
The scraper must yield exactly the 3 records (normalized to ``ChatGroup``
snake_case columns + ``raw_json``), parse ``memberIds`` into
``member_ids_list`` (dropping empty segments), make exactly 3 calls, and stop
without a 4th.
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.wechat import scrape_wechat_groups
from app.session.yunke_client import YunkeClient
from tests.cassettes.chatroom_fixture import (
CHATROOM_RECORDS,
DEFAULT_PAGE_SIZE,
TOTAL,
make_chatroom_list_response,
)
BASE = "https://crm2.yunkecn.com"
CHATROOM_LIST_URL = f"{BASE}/wechatweb/chatroom/getChatroomList"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + member_ids_list parsing
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 3 records, 3 calls."""
page1 = make_chatroom_list_response(
CHATROOM_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_chatroom_list_response(
CHATROOM_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_chatroom_list_response(
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(CHATROOM_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 3 fixture records (not duplicated, not dropped).
assert len(results) == 3
assert {r["room_id"] for r in results} == {
rec["chatRoomId"] for rec in CHATROOM_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, CHATROOM_RECORDS, strict=True):
# Identity + channel.
assert result["room_id"] == record["chatRoomId"]
assert result["channel"] == "wechat"
# Naming + ownership.
assert result["room_name"] == record["chatRoomName"]
assert result["owner_account_id"] == record["wechatId"]
assert result["owner_account_nickname"] == record["wechatNickName"]
assert result["owner_user_id"] == record["userId"]
assert result["owner_user_phone"] == record["userPhone"]
# Counts.
assert result["member_count"] == record["memberCount"]
assert result["message_count"] == record["messageCount"]
assert result["own_message_count"] == record["ownMessageCount"]
assert result["loss_members"] == record["lossMembers"]
assert result["add_members"] == record["addMembers"]
# Avatar + timestamps.
assert result["head_image_url"] == record["headImageUrl"]
assert result["last_chat_time"] == record["lastChatTime"]
assert result["update_time"] == record["updateTime"]
# member_ids carried raw; member_ids_list parsed (empties dropped).
assert result["member_ids"] == record["memberIds"]
assert result["member_ids_list"] == [
seg for seg in record["memberIds"].split(";") if seg
]
# Version carried verbatim; raw_json preserves the original record.
assert result["version"] == record["version"]
assert result["raw_json"] == record
# ---------------------------------------------------------------------------
# member_ids_list parsing edge cases
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_drops_empty_member_id_segments():
"""Trailing/empty ``memberIds`` segments are dropped from member_ids_list."""
# Record 2 has a trailing ";" -> one empty segment that must be dropped.
page1 = make_chatroom_list_response(
[CHATROOM_RECORDS[1]], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_chatroom_list_response([], total=1, page_num=2)
respx.post(CHATROOM_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)]
finally:
await client.aclose()
assert len(results) == 1
# "wxid_member_b1;wxid_member_b2;" -> 2 ids (trailing empty dropped).
assert results[0]["member_ids_list"] == ["wxid_member_b1", "wxid_member_b2"]
@respx.mock
async def test_scrape_wechat_groups_handles_missing_member_ids():
"""A group with no ``memberIds`` field yields an empty member_ids_list."""
record = {"chatRoomId": "x@chatroom", "chatRoomName": "Empty"}
page1 = make_chatroom_list_response([record], total=1, page_num=1)
page2 = make_chatroom_list_response([], total=1, page_num=2)
respx.post(CHATROOM_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["member_ids"] is None
assert results[0]["member_ids_list"] == []
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_sends_pagination_body():
"""Outgoing request body carries pageNum/pageSize; advances per page."""
page1 = make_chatroom_list_response(
[CHATROOM_RECORDS[0]], total=TOTAL, page_num=1, page_size=1
)
page2 = make_chatroom_list_response([], total=TOTAL, page_num=2, page_size=1)
route = respx.post(CHATROOM_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_groups(client, page_size=1)]
finally:
await client.aclose()
# One record on page 1, empty page 2 stops the loop.
assert len(results) == 1
assert route.call_count == 2
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_chatroom_list_response([], total=0, page_num=1)
route = respx.post(CHATROOM_LIST_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_groups(client)]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1
yunke-server/tests/test_scrape_wechat_talkers.py
+164
View File
@@ -0,0 +1,164 @@
"""Tests for the WeChat talker (conversation) list scraper (M2).
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 3 conversations across 2 pages of size 2, plus a
terminating empty page (the page-number paginator's primary stop signal):
* page 1 -> 2 records, ``hasMore=true``
* page 2 -> 1 record, ``hasMore=false``
* page 3 -> empty ``page`` array (no more rows)
The scraper must yield exactly the 3 talkers (normalized), send the owning
``wechatId`` in every request body, advance ``pageNum``, and stop after the
empty page. Both the direct-message (``memberCount == 0``) and group-chat
(``memberCount > 0``) paths are exercised by the fixture.
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.wechat import scrape_wechat_talkers
from app.session.yunke_client import YunkeClient
from tests.cassettes.talkerList_fixture import (
DEFAULT_PAGE_SIZE,
OWNER_WECHAT_ID,
TALKER_RECORDS,
make_talker_list_response,
)
BASE = "https://crm2.yunkecn.com"
TALKER_LIST_URL = f"{BASE}/wechatweb/wxFriends/talkerList"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + wechatId in request body
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_talkers_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 3 talkers, 3 calls."""
page1 = make_talker_list_response(
TALKER_RECORDS[:2], page_num=1, page_size=DEFAULT_PAGE_SIZE, has_more=True
)
page2 = make_talker_list_response(
TALKER_RECORDS[2:], page_num=2, page_size=DEFAULT_PAGE_SIZE, has_more=False
)
page3 = make_talker_list_response([], page_num=3, page_size=DEFAULT_PAGE_SIZE)
route = respx.post(TALKER_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_wechat_talkers(
client, OWNER_WECHAT_ID, page_size=DEFAULT_PAGE_SIZE
)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 3 fixture talkers (not duplicated, not dropped).
assert len(results) == 3
assert {r["friend_wechat_id"] for r in results} == {
rec["friendWechatId"] for rec in TALKER_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, TALKER_RECORDS, strict=True):
assert result["friend_wechat_id"] == record["friendWechatId"]
assert result["nick_name"] == record["nickName"]
assert result["remark_name"] == record["remarkName"]
assert result["message_count"] == record["messageCount"]
assert result["last_chat_time"] == record["lastChatTime"]
assert result["member_count"] == record["memberCount"]
assert result["head_url"] == record["headUrl"]
# The owning staff account is carried on every record.
assert result["owner_account_id"] == OWNER_WECHAT_ID
# raw_json preserves the original upstream record.
assert result["raw_json"] == record
@respx.mock
async def test_scrape_wechat_talkers_sends_wechat_id_and_pagination_body():
"""Every request carries the staff wechatId + advancing pageNum/pageSize."""
page1 = make_talker_list_response(
TALKER_RECORDS[:1], page_num=1, page_size=1, has_more=True
)
page2 = make_talker_list_response(
[], page_num=2, page_size=1, has_more=False
)
route = respx.post(TALKER_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
client = _primed_client()
try:
results = [
r async for r in scrape_wechat_talkers(client, OWNER_WECHAT_ID, page_size=1)
]
finally:
await client.aclose()
# One record on page 1, empty page 2 stops the loop.
assert len(results) == 1
assert route.call_count == 2
# Both requests must carry the owning wechatId + page slice.
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["wechatId"] == OWNER_WECHAT_ID
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["wechatId"] == OWNER_WECHAT_ID
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_talkers_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_talker_list_response([], page_num=1)
route = respx.post(TALKER_LIST_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_talkers(client, OWNER_WECHAT_ID)]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1