xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scraper): 群抓取改用 friend_list/es+friendTypeYunke=2(替换 getChatroomList)

Browse Source 
getChatroomList 在云客真机返回 0 行(已 Playwright 2026-08-10 验证)。
群与好友共用 /wechatweb/wxFriends/friend_list/es,仅以 friendTypeYunke 区分
(1=好友 / 2=群),响应信封一致(basicResult),复用 _parse_total/_parse_rows。

- 删除 CHATROOM_LIST_PATH / _parse_chatroom_total / _parse_chatroom_rows
  / _build_chatroom_body / _parse_member_ids(改写后已无引用)
- 新增 _build_group_body(pageNum/pageSize/friendTypeYunke="2")
- 重写 _normalize_chat_group 映射 friend_list/es 字段
  (friendWxId->room_id, friendWxNickname->room_name, friendNum->member_count,
   wxId/ownerName/userId, phone||wechatPhone, messageCount/sendNum/lastChatTime)
- friend_list/es 不返回成员明细,member_ids/member_ids_list 固定 None/[]
  (群列表可用;成员镜像延后)
- 删除 tests/cassettes/chatroom_fixture.py,改复用 wxFriends_fixture 信封
- 重写 test_scrape_wechat_groups:断言新端点 URL + friendTypeYunke="2" body
  + 新字段映射;member_ids 边界用例改为"始终 None/[]"

scrape_wechat_groups 签名不变(client + page_size),orchestrator/scheduler 未改。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-10 15:33:59 +08:00
parent 239868531f
commit d220fa1a42
3 changed files with 191 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/app/scraper/wechat.py
+69 -95
View File
@@ -19,11 +19,14 @@ Endpoints covered today:
``enddate`` or it answers ``code -90000`` "缺少..."; a recent ~7-day
window works (the date affects per-account stats, not which accounts are
returned). Page-number pagination (Task 2.1).
* :func:`scrape_wechat_groups` — wraps ``POST /wechatweb/chatroom/getChatroomList``
and yields normalized :class:`app.db.models.ChatGroup` dicts, parsing the
semicolon-joined ``memberIds`` into a ``member_ids_list`` so the
persistence layer can mirror :class:`app.db.models.GroupMember` rows
(M2). Page-number pagination.
* :func:`scrape_wechat_groups` — wraps ``POST /wechatweb/wxFriends/friend_list/es``
with ``friendTypeYunke="2"`` in the body (the SAME endpoint friends use;
``"1"`` selects friends, ``"2"`` selects groups) and yields normalized
:class:`app.db.models.ChatGroup` dicts. The response envelope is identical
to the friend scraper's, so the shared ``_parse_total`` / ``_parse_rows``
helpers are reused. Member details are NOT available from this endpoint —
``member_ids`` / ``member_ids_list`` are emitted as ``None`` / ``[]``
(group list works; member mirroring is deferred). Page-number pagination.
* :func:`scrape_wechat_talkers` — wraps ``POST /wechatweb/wxFriends/talkerList``
and yields the per-staff-wechat conversation list (M2). Used by the sync
orchestrator to decide which conversations have recent traffic to pull.
@@ -84,7 +87,6 @@ FRIEND_LIST_PATH = "/wechatweb/wxFriends/friend_list/es"
MSG_CHATS_PATH = "/wechatweb/msg/chats"
DEPT_TREE_PATH = "/usercenter/dept/tree"
WXUSER_LIST_PATH = "/wechatweb/wxUser/list"
CHATROOM_LIST_PATH = "/wechatweb/chatroom/getChatroomList"
TALKER_LIST_PATH = "/wechatweb/wxFriends/talkerList"
VOICE_LIST_PATH = "/wechatweb/callRecord/queryWeChatVoiceList"
@@ -843,98 +845,69 @@ async def scrape_wechat_accounts(
# ===========================================================================
# Chat-group scraper (M2) — POST /wechatweb/chatroom/getChatroomList
# Chat-group scraper (M2) — POST /wechatweb/wxFriends/friend_list/es
# (friendTypeYunke="2" selects groups; "1" selects friends)
# ===========================================================================
def _parse_chatroom_total(resp: Any) -> int | None:
"""Read the declared row count from the ``getChatroomList`` envelope.
def _build_group_body(page_num: int, page_size: int) -> dict[str, Any]:
"""Build the ``friend_list/es`` body for chat-groups (friendTypeYunke=2).
Shape: ``data.total`` (an int). Returns ``None`` if the path is absent so
the paginator falls back to the empty-page stop signal.
Groups share the friend-list endpoint; the ``friendTypeYunke`` body
parameter is the switch (``"1"`` = friends, ``"2"`` = groups). Verified
against the live CRM (2026-08-10 Playwright capture) — the previous
``POST /wechatweb/chatroom/getChatroomList`` endpoint returned 0 rows.
"""
try:
body = resp.json()
except (AttributeError, ValueError):
return None
data = body.get("data") if isinstance(body, dict) else None
if not isinstance(data, dict):
return None
total = data.get("total")
return int(total) if isinstance(total, (int, float)) and not isinstance(
total, bool
) else None
def _parse_chatroom_rows(resp: Any) -> list[dict[str, Any]]:
"""Extract the chatroom list from the ``getChatroomList`` envelope.
Shape: ``data.data`` (a list of records). Missing / non-list values
collapse to an empty list so the paginator sees "empty page".
"""
try:
body = resp.json()
except (AttributeError, ValueError):
return []
data = body.get("data") if isinstance(body, dict) else None
if not isinstance(data, dict):
return []
rows = data.get("data")
return rows if isinstance(rows, list) else []
def _build_chatroom_body(page_num: int, page_size: int) -> dict[str, Any]:
"""Build the ``getChatroomList`` request body for a given page.
The live capture carries no filter parameters — just the page slice.
"""
return {"pageNum": page_num, "pageSize": page_size}
def _parse_member_ids(member_ids: Any) -> list[str]:
"""Split a semicolon-joined member id list, dropping empty segments.
Upstream ``memberIds`` looks like ``"wxid_a;wxid_b;csf7169708"`` and may
carry a trailing ``";"`` (yielding an empty segment). Empties are dropped
so each entry is a real wxid; a missing/None/non-string value yields ``[]``.
"""
if not isinstance(member_ids, str):
return []
return [seg for seg in member_ids.split(";") if seg]
return {"pageNum": page_num, "pageSize": page_size, "friendTypeYunke": "2"}
def _normalize_chat_group(record: dict[str, Any]) -> dict[str, Any]:
"""Map one upstream ``getChatroomList`` record to the ``ChatGroup`` shape.
"""Map one ``friend_list/es`` group record (friendTypeYunke=2) to ChatGroup.
Field mapping (verified 2026-08-10 Playwright capture):
* ``friendWxId`` -> ``room_id`` (the ``@chatroom`` id)
* ``friendWxNickname`` -> ``room_name`` (the group name)
* ``friendNum`` -> ``member_count``
* ``friendHeadImageUrl`` -> ``head_image_url`` (group avatar)
* ``wxId`` -> ``owner_account_id`` (staff wechat account)
* ``ownerName`` -> ``owner_account_nickname`` (group owner nickname)
* ``userId`` -> ``owner_user_id`` (owning staff id)
* ``phone`` / ``wechatPhone`` -> ``owner_user_phone`` (phone preferred)
* ``messageCount`` -> ``message_count``
* ``sendNum`` -> ``own_message_count`` (this account's sends)
* ``lastChatTime`` -> ``last_chat_time`` (ms-epoch)
* ``version`` -> ``version``
Member ids are NOT surfaced by ``friend_list/es`` (only the ``friendNum``
count is); ``member_ids`` / ``member_ids_list`` are emitted as ``None`` /
``[]`` so the column shape stays stable. Group-member mirroring is
deferred to a future endpoint. Likewise ``loss_members`` / ``add_members``
/ ``update_time`` are not carried by this endpoint and are emitted as
``None``.
Adds ``channel="wechat"`` and preserves the original record under
``raw_json``. The raw ``memberIds`` string is carried verbatim as
``member_ids`` (matching the ``ChatGroup.member_ids`` Text column) AND
parsed into ``member_ids_list`` (a plain list of wxids with empties
dropped) so :func:`app.storage.repos.upsert_chat_group` can mirror one
:class:`app.db.models.GroupMember` row per entry without re-parsing.
``None``-safe: missing fields become ``None`` rather than raising, so a
partial payload does not break the whole scrape.
``raw_json``. ``None``-safe on every field: missing fields become ``None``
rather than raising, so a partial payload does not break the scrape.
"""
member_ids = record.get("memberIds")
return {
"room_id": record.get("chatRoomId"),
"room_id": record.get("friendWxId"),
"channel": "wechat",
"room_name": record.get("chatRoomName"),
"owner_account_id": record.get("wechatId"),
"owner_account_nickname": record.get("wechatNickName"),
"member_count": record.get("memberCount"),
"member_ids": member_ids,
"member_ids_list": _parse_member_ids(member_ids),
"head_image_url": record.get("headImageUrl"),
"room_name": record.get("friendWxNickname"),
"owner_account_id": record.get("wxId"),
"owner_account_nickname": record.get("ownerName"),
"member_count": record.get("friendNum"),
"member_ids": None,
"member_ids_list": [],
"head_image_url": record.get("friendHeadImageUrl"),
"owner_user_id": record.get("userId"),
"owner_user_phone": record.get("userPhone"),
"owner_user_phone": record.get("phone") or record.get("wechatPhone"),
"message_count": record.get("messageCount"),
"own_message_count": record.get("ownMessageCount"),
"loss_members": record.get("lossMembers"),
"add_members": record.get("addMembers"),
"own_message_count": record.get("sendNum"),
"loss_members": None,
"add_members": None,
"last_chat_time": record.get("lastChatTime"),
"update_time": record.get("updateTime"),
"update_time": None,
"version": record.get("version"),
"raw_json": record,
}
@@ -945,17 +918,18 @@ async def scrape_wechat_groups(
*,
page_size: int = 50,
) -> AsyncIterator[dict[str, Any]]:
"""Yield normalized WeChat chat-group records from ``getChatroomList``.
"""Yield normalized WeChat chat-group records from ``friend_list/es``.
Walks ``POST /wechatweb/chatroom/getChatroomList`` page by page until the
upstream reports no more rows (empty ``data.data`` array — the primary
stop signal), mapping each record to the ``ChatGroup`` column shape plus
a verbatim ``raw_json`` copy.
Walks ``POST /wechatweb/wxFriends/friend_list/es`` page by page (the SAME
endpoint :func:`scrape_wechat_friends` uses) with ``friendTypeYunke="2"``
in the body to select groups (``"1"`` selects friends). The response
envelope is identical to the friend scraper's (``data.basicResult.data``
+ ``data.basicResult.total``), so the shared ``_parse_total`` /
``_parse_rows`` helpers are reused.
The semicolon-joined ``memberIds`` field is carried both raw
(``member_ids``) and parsed (``member_ids_list``, empties dropped) so the
persistence layer can mirror :class:`app.db.models.GroupMember` rows
idempotently.
Each record is mapped to the ``ChatGroup`` column shape plus a verbatim
``raw_json`` copy. Member ids are NOT available from this endpoint —
``member_ids`` / ``member_ids_list`` are emitted as ``None`` / ``[]``.
Parameters
----------
@@ -968,11 +942,11 @@ async def scrape_wechat_groups(
"""
async for record in paginate_by_page(
client,
CHATROOM_LIST_PATH,
FRIEND_LIST_PATH,
page_size=page_size,
parse_total=_parse_chatroom_total,
parse_rows=_parse_chatroom_rows,
build_body=_build_chatroom_body,
parse_total=_parse_total,
parse_rows=_parse_rows,
build_body=_build_group_body,
):
yield _normalize_chat_group(record)
yunke-server/tests/cassettes/chatroom_fixture.py
-137
View File
@@ -1,137 +0,0 @@
"""Fixture payloads for ``POST /wechatweb/chatroom/getChatroomList`` (M2).
These records mirror the real field names captured from the live CRM (see the
data dictionary §2: ``chatRoomId(@chatroom), chatRoomName, memberCount,
memberIds(分号分隔全部成员wxid), headImageUrl, wechatId/wechatNickName(归属),
userName/userPhone(群主员工), messageCount/ownMessageCount, lossMembers/
addMembers, lastChatTime/updateTime``; total=1515).
The envelope is the same flat shape as ``wxUser/list`` (no ``basicResult``
wrapper): pagination fields sit directly under ``data``::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "page":.., "pageCount":..}}
The cassette ships three records across two pages so a test can drive the
page-number paginator across the empty-page stop signal:
* page 1 -> 2 of the 3 fixture records, ``total=3``
* page 2 -> the remaining 1 record, ``total=3``
* (page 3 -> empty ``data`` array, served inline by the test)
Records carry every field with a target ``ChatGroup`` column so the test can
assert a full mapping end-to-end. The ``memberIds`` field is the
semicolon-joined member wxid list; record 2 deliberately includes a trailing
empty segment (``"...;"``) so the parser's drop-empties path is exercised.
"""
# The module name mirrors the upstream endpoint name (``chatroom``); the M2
# spec mandates this exact filename.
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped chatroom records (3 total across 2 pages).
# ---------------------------------------------------------------------------
CHATROOM_RECORDS: list[dict[str, Any]] = [
{
"chatRoomId": "12345678901@chatroom",
"chatRoomName": "VIP客户交流群",
"memberCount": 3,
# 3 members, no empties -> member_ids_list length 3.
"memberIds": "wxid_member_a1;wxid_member_a2;csf7169708",
"headImageUrl": "https://wx.qlogo.cn/room_12345678901.png",
"wechatId": "wxid_staff_alice01",
"wechatNickName": "Alice WeChat",
"userId": "staff_user_001",
"userName": "Alice Wang",
"userPhone": "13800000001",
"messageCount": 4521,
"ownMessageCount": 1204,
"lossMembers": 2,
"addMembers": 5,
"lastChatTime": 1700000000123,
"updateTime": 1700000001000,
"version": "v1.20260731.room1",
},
{
"chatRoomId": "98765432109@chatroom",
"chatRoomName": "华东销售大群",
"memberCount": 2,
# Trailing empty segment -> member_ids_list length 2 (empties dropped).
"memberIds": "wxid_member_b1;wxid_member_b2;",
"headImageUrl": "https://wx.qlogo.cn/room_98765432109.png",
"wechatId": "wxid_staff_bob02",
"wechatNickName": "Bob WeChat",
"userId": "staff_user_002",
"userName": "Bob Chen",
"userPhone": "13800000002",
"messageCount": 9821,
"ownMessageCount": 211,
"lossMembers": 0,
"addMembers": 1,
"lastChatTime": 1700000000456,
"updateTime": 1700000000800,
"version": "v1.20260731.room2",
},
{
# A group whose owner has no resolved staff id (userId None) — the
# scraper must still yield it; the repo must still upsert its members.
"chatRoomId": "55555555555@chatroom",
"chatRoomName": "External Coordination",
"memberCount": 1,
"memberIds": "wxid_member_c1",
"headImageUrl": "https://wx.qlogo.cn/room_55555555555.png",
"wechatId": "wxid_staff_orphan04",
"wechatNickName": "Orphan WeChat",
"userId": None,
"userName": None,
"userPhone": None,
"messageCount": 12,
"ownMessageCount": 0,
"lossMembers": 0,
"addMembers": 0,
"lastChatTime": 1700000000789,
"updateTime": 1700000000900,
"version": "v1.20260731.room3",
},
]
# Default page size for the fixture (and the scraper). total=3 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(CHATROOM_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_chatroom_list_response(
records: list[dict[str, Any]],
*,
total: int,
page_num: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``getChatroomList`` response envelope.
Shape (no ``basicResult`` wrapper; pagination fields directly under
``data``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "page":.., "pageCount":..}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-{page_num}",
"data": {
"data": list(records),
"total": total,
"page": page_num,
"pageCount": (
(total + page_size - 1) // page_size if page_size else 1
),
},
}
yunke-server/tests/test_scrape_wechat_groups.py
+122 -74
View File
@@ -4,6 +4,13 @@ Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
Groups reuse the SAME upstream endpoint as friends
(``POST /wechatweb/wxFriends/friend_list/es``); the ``friendTypeYunke="2"``
body parameter selects groups (``"1"`` selects friends). The response
envelope is identical to the friend scraper's, so this test reuses
:func:`tests.cassettes.wxFriends_fixture.make_friend_list_response` to build
capture-shaped envelopes.
The fixture ships 3 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal for page-number endpoints):
@@ -12,33 +19,89 @@ empty page (the documented primary stop signal for page-number endpoints):
* page 3 -> empty ``data`` array (no more rows)
The scraper must yield exactly the 3 records (normalized to ``ChatGroup``
snake_case columns + ``raw_json``), parse ``memberIds`` into
``member_ids_list`` (dropping empty segments), make exactly 3 calls, and stop
without a 4th.
snake_case columns + ``raw_json``), make exactly 3 calls, and stop without a
4th. Member details are NOT surfaced by ``friend_list/es`` — ``member_ids``
is ``None`` and ``member_ids_list`` is ``[]`` on every yielded record.
"""
from __future__ import annotations
import json as _json
from typing import Any
import httpx
import respx
from app.scraper.wechat import scrape_wechat_groups
from app.session.yunke_client import YunkeClient
from tests.cassettes.chatroom_fixture import (
CHATROOM_RECORDS,
DEFAULT_PAGE_SIZE,
TOTAL,
make_chatroom_list_response,
)
from tests.cassettes.wxFriends_fixture import make_friend_list_response
BASE = "https://crm2.yunkecn.com"
CHATROOM_LIST_URL = f"{BASE}/wechatweb/chatroom/getChatroomList"
FRIEND_LIST_URL = f"{BASE}/wechatweb/wxFriends/friend_list/es"
USERID = "a" * 32
COMPANY = "8m0atd"
# Realistic, capture-shaped group records. Field names are EXACTLY what the
# live ``friend_list/es`` endpoint returns for ``friendTypeYunke="2"``
# (verified 2026-08-10 Playwright capture). Every field with a target
# ``ChatGroup`` column is populated so the test asserts a full mapping
# end-to-end. (Member ids are NOT in this payload — the endpoint surfaces
# only the ``friendNum`` count — so ``member_ids`` / ``member_ids_list`` are
# exercised as None / [] separately.)
GROUP_RECORDS: list[dict[str, Any]] = [
{
"friendWxId": "12345678901@chatroom",
"friendWxNickname": "VIP客户交流群",
"friendNum": 3,
"friendHeadImageUrl": "https://wx.qlogo.cn/room_12345678901.png",
"wxId": "wxid_staff_alice01",
"ownerName": "Alice WeChat",
"userId": "staff_user_001",
"phone": "13800000001",
"wechatPhone": "13800000011",
"messageCount": 4521,
"sendNum": 1204,
"lastChatTime": 1700000000123,
"version": "v1.20260731.room1",
},
{
"friendWxId": "98765432109@chatroom",
"friendWxNickname": "华东销售大群",
"friendNum": 2,
"friendHeadImageUrl": "https://wx.qlogo.cn/room_98765432109.png",
"wxId": "wxid_staff_bob02",
"ownerName": "Bob WeChat",
"userId": "staff_user_002",
"phone": "13800000002",
"wechatPhone": "13800000022",
"messageCount": 9821,
"sendNum": 211,
"lastChatTime": 1700000000456,
"version": "v1.20260731.room2",
},
{
# A group whose owner has no resolved staff id (userId None) — the
# scraper must still yield it; None-safe fields stay None.
"friendWxId": "55555555555@chatroom",
"friendWxNickname": "External Coordination",
"friendNum": 1,
"friendHeadImageUrl": "https://wx.qlogo.cn/room_55555555555.png",
"wxId": "wxid_staff_orphan04",
"ownerName": "Orphan WeChat",
"userId": None,
"phone": None,
"wechatPhone": None,
"messageCount": 12,
"sendNum": 0,
"lastChatTime": 1700000000789,
"version": "v1.20260731.room3",
},
]
TOTAL = len(GROUP_RECORDS)
DEFAULT_PAGE_SIZE = 2
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
@@ -46,24 +109,24 @@ def _primed_client() -> YunkeClient:
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + member_ids_list parsing
# End-to-end: pagination + field mapping
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 3 records, 3 calls."""
page1 = make_chatroom_list_response(
CHATROOM_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
page1 = make_friend_list_response(
GROUP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_chatroom_list_response(
CHATROOM_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
page2 = make_friend_list_response(
GROUP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_chatroom_list_response(
page3 = make_friend_list_response(
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(CHATROOM_LIST_URL).mock(
route = respx.post(FRIEND_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
@@ -85,39 +148,41 @@ async def test_scrape_wechat_groups_yields_records_and_paginates():
# Exactly the 3 fixture records (not duplicated, not dropped).
assert len(results) == 3
assert {r["room_id"] for r in results} == {
rec["chatRoomId"] for rec in CHATROOM_RECORDS
rec["friendWxId"] for rec in GROUP_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, CHATROOM_RECORDS, strict=True):
for result, record in zip(results, GROUP_RECORDS, strict=True):
# Identity + channel.
assert result["room_id"] == record["chatRoomId"]
assert result["room_id"] == record["friendWxId"]
assert result["channel"] == "wechat"
# Naming + ownership.
assert result["room_name"] == record["chatRoomName"]
assert result["owner_account_id"] == record["wechatId"]
assert result["owner_account_nickname"] == record["wechatNickName"]
assert result["room_name"] == record["friendWxNickname"]
assert result["owner_account_id"] == record["wxId"]
assert result["owner_account_nickname"] == record["ownerName"]
assert result["owner_user_id"] == record["userId"]
assert result["owner_user_phone"] == record["userPhone"]
# phone preferred, wechatPhone fallback.
assert result["owner_user_phone"] == (
record["phone"] or record["wechatPhone"]
)
# Counts.
assert result["member_count"] == record["memberCount"]
assert result["member_count"] == record["friendNum"]
assert result["message_count"] == record["messageCount"]
assert result["own_message_count"] == record["ownMessageCount"]
assert result["loss_members"] == record["lossMembers"]
assert result["add_members"] == record["addMembers"]
assert result["own_message_count"] == record["sendNum"]
# Avatar + timestamps.
assert result["head_image_url"] == record["headImageUrl"]
assert result["head_image_url"] == record["friendHeadImageUrl"]
assert result["last_chat_time"] == record["lastChatTime"]
assert result["update_time"] == record["updateTime"]
# member_ids carried raw; member_ids_list parsed (empties dropped).
assert result["member_ids"] == record["memberIds"]
assert result["member_ids_list"] == [
seg for seg in record["memberIds"].split(";") if seg
]
# Member details NOT surfaced by friend_list/es -> None / [].
assert result["member_ids"] is None
assert result["member_ids_list"] == []
# Fields the new endpoint does not carry.
assert result["loss_members"] is None
assert result["add_members"] is None
assert result["update_time"] is None
# Version carried verbatim; raw_json preserves the original record.
assert result["version"] == record["version"]
@@ -125,46 +190,26 @@ async def test_scrape_wechat_groups_yields_records_and_paginates():
# ---------------------------------------------------------------------------
# member_ids_list parsing edge cases
# member_ids behaviour: always None / [] (not surfaced by friend_list/es)
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_drops_empty_member_id_segments():
"""Trailing/empty ``memberIds`` segments are dropped from member_ids_list."""
# Record 2 has a trailing ";" -> one empty segment that must be dropped.
page1 = make_chatroom_list_response(
[CHATROOM_RECORDS[1]], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_chatroom_list_response([], total=1, page_num=2)
respx.post(CHATROOM_LIST_URL).mock(
async def test_scrape_wechat_groups_emits_empty_member_ids():
"""``friend_list/es`` does not surface member ids; they are always
``None`` / ``[]`` on yielded records (member mirroring deferred)."""
record = dict(GROUP_RECORDS[0])
page1 = make_friend_list_response([record], total=1, page_num=1)
page2 = make_friend_list_response([], total=1, page_num=2)
respx.post(FRIEND_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)]
finally:
await client.aclose()
assert len(results) == 1
# "wxid_member_b1;wxid_member_b2;" -> 2 ids (trailing empty dropped).
assert results[0]["member_ids_list"] == ["wxid_member_b1", "wxid_member_b2"]
@respx.mock
async def test_scrape_wechat_groups_handles_missing_member_ids():
"""A group with no ``memberIds`` field yields an empty member_ids_list."""
record = {"chatRoomId": "x@chatroom", "chatRoomName": "Empty"}
page1 = make_chatroom_list_response([record], total=1, page_num=1)
page2 = make_chatroom_list_response([], total=1, page_num=2)
respx.post(CHATROOM_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
client = _primed_client()
try:
results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)]
results = [
r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)
]
finally:
await client.aclose()
@@ -174,18 +219,18 @@ async def test_scrape_wechat_groups_handles_missing_member_ids():
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# Pagination body: pageNum/pageSize advanced + friendTypeYunke="2"
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_groups_sends_pagination_body():
"""Outgoing request body carries pageNum/pageSize; advances per page."""
page1 = make_chatroom_list_response(
[CHATROOM_RECORDS[0]], total=TOTAL, page_num=1, page_size=1
"""Outgoing body carries pageNum/pageSize and ``friendTypeYunke="2"``."""
page1 = make_friend_list_response(
[GROUP_RECORDS[0]], total=TOTAL, page_num=1, page_size=1
)
page2 = make_chatroom_list_response([], total=TOTAL, page_num=2, page_size=1)
route = respx.post(CHATROOM_LIST_URL).mock(
page2 = make_friend_list_response([], total=TOTAL, page_num=2, page_size=1)
route = respx.post(FRIEND_LIST_URL).mock(
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
)
@@ -202,10 +247,13 @@ async def test_scrape_wechat_groups_sends_pagination_body():
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
# The crucial switch: friendTypeYunke="2" selects GROUPS ("1" = friends).
assert body1["friendTypeYunke"] == "2"
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
assert body2["friendTypeYunke"] == "2"
# ---------------------------------------------------------------------------
@@ -216,8 +264,8 @@ async def test_scrape_wechat_groups_sends_pagination_body():
@respx.mock
async def test_scrape_wechat_groups_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_chatroom_list_response([], total=0, page_num=1)
route = respx.post(CHATROOM_LIST_URL).mock(
empty = make_friend_list_response([], total=0, page_num=1)
route = respx.post(FRIEND_LIST_URL).mock(
return_value=httpx.Response(200, json=empty)
)