From d220fa1a420671c8fab74371caa7d6537907dc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Mon, 10 Aug 2026 15:33:59 +0800 Subject: [PATCH] =?UTF-8?q?fix(scraper):=20=E7=BE=A4=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E6=94=B9=E7=94=A8=20friend=5Flist/es+friendTypeYunke=3D2(?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2=20getChatroomList)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getChatroomList 在云客真机返回 0 行(已 Playwright 2026-08-10 验证)。 群与好友共用 /wechatweb/wxFriends/friend_list/es,仅以 friendTypeYunke 区分 (1=好友 / 2=群),响应信封一致(basicResult),复用 _parse_total/_parse_rows。 - 删除 CHATROOM_LIST_PATH / _parse_chatroom_total / _parse_chatroom_rows / _build_chatroom_body / _parse_member_ids(改写后已无引用) - 新增 _build_group_body(pageNum/pageSize/friendTypeYunke="2") - 重写 _normalize_chat_group 映射 friend_list/es 字段 (friendWxId->room_id, friendWxNickname->room_name, friendNum->member_count, wxId/ownerName/userId, phone||wechatPhone, messageCount/sendNum/lastChatTime) - friend_list/es 不返回成员明细,member_ids/member_ids_list 固定 None/[] (群列表可用;成员镜像延后) - 删除 tests/cassettes/chatroom_fixture.py,改复用 wxFriends_fixture 信封 - 重写 test_scrape_wechat_groups:断言新端点 URL + friendTypeYunke="2" body + 新字段映射;member_ids 边界用例改为"始终 None/[]" scrape_wechat_groups 签名不变(client + page_size),orchestrator/scheduler 未改。 Co-Authored-By: Claude --- yunke-server/app/scraper/wechat.py | 164 ++++++--------- .../tests/cassettes/chatroom_fixture.py | 137 ------------ .../tests/test_scrape_wechat_groups.py | 196 +++++++++++------- 3 files changed, 191 insertions(+), 306 deletions(-) delete mode 100644 yunke-server/tests/cassettes/chatroom_fixture.py diff --git a/yunke-server/app/scraper/wechat.py b/yunke-server/app/scraper/wechat.py index c49e291..9bc04f5 100644 --- a/yunke-server/app/scraper/wechat.py +++ b/yunke-server/app/scraper/wechat.py @@ -19,11 +19,14 @@ Endpoints covered today: ``enddate`` or it answers ``code -90000`` "缺少..."; a recent ~7-day window works (the date affects per-account stats, not which accounts are returned). Page-number pagination (Task 2.1). -* :func:`scrape_wechat_groups` — wraps ``POST /wechatweb/chatroom/getChatroomList`` - and yields normalized :class:`app.db.models.ChatGroup` dicts, parsing the - semicolon-joined ``memberIds`` into a ``member_ids_list`` so the - persistence layer can mirror :class:`app.db.models.GroupMember` rows - (M2). Page-number pagination. +* :func:`scrape_wechat_groups` — wraps ``POST /wechatweb/wxFriends/friend_list/es`` + with ``friendTypeYunke="2"`` in the body (the SAME endpoint friends use; + ``"1"`` selects friends, ``"2"`` selects groups) and yields normalized + :class:`app.db.models.ChatGroup` dicts. The response envelope is identical + to the friend scraper's, so the shared ``_parse_total`` / ``_parse_rows`` + helpers are reused. Member details are NOT available from this endpoint — + ``member_ids`` / ``member_ids_list`` are emitted as ``None`` / ``[]`` + (group list works; member mirroring is deferred). Page-number pagination. * :func:`scrape_wechat_talkers` — wraps ``POST /wechatweb/wxFriends/talkerList`` and yields the per-staff-wechat conversation list (M2). Used by the sync orchestrator to decide which conversations have recent traffic to pull. @@ -84,7 +87,6 @@ FRIEND_LIST_PATH = "/wechatweb/wxFriends/friend_list/es" MSG_CHATS_PATH = "/wechatweb/msg/chats" DEPT_TREE_PATH = "/usercenter/dept/tree" WXUSER_LIST_PATH = "/wechatweb/wxUser/list" -CHATROOM_LIST_PATH = "/wechatweb/chatroom/getChatroomList" TALKER_LIST_PATH = "/wechatweb/wxFriends/talkerList" VOICE_LIST_PATH = "/wechatweb/callRecord/queryWeChatVoiceList" @@ -843,98 +845,69 @@ async def scrape_wechat_accounts( # =========================================================================== -# Chat-group scraper (M2) — POST /wechatweb/chatroom/getChatroomList +# Chat-group scraper (M2) — POST /wechatweb/wxFriends/friend_list/es +# (friendTypeYunke="2" selects groups; "1" selects friends) # =========================================================================== -def _parse_chatroom_total(resp: Any) -> int | None: - """Read the declared row count from the ``getChatroomList`` envelope. +def _build_group_body(page_num: int, page_size: int) -> dict[str, Any]: + """Build the ``friend_list/es`` body for chat-groups (friendTypeYunke=2). - Shape: ``data.total`` (an int). Returns ``None`` if the path is absent so - the paginator falls back to the empty-page stop signal. + Groups share the friend-list endpoint; the ``friendTypeYunke`` body + parameter is the switch (``"1"`` = friends, ``"2"`` = groups). Verified + against the live CRM (2026-08-10 Playwright capture) — the previous + ``POST /wechatweb/chatroom/getChatroomList`` endpoint returned 0 rows. """ - try: - body = resp.json() - except (AttributeError, ValueError): - return None - data = body.get("data") if isinstance(body, dict) else None - if not isinstance(data, dict): - return None - total = data.get("total") - return int(total) if isinstance(total, (int, float)) and not isinstance( - total, bool - ) else None - - -def _parse_chatroom_rows(resp: Any) -> list[dict[str, Any]]: - """Extract the chatroom list from the ``getChatroomList`` envelope. - - Shape: ``data.data`` (a list of records). Missing / non-list values - collapse to an empty list so the paginator sees "empty page". - """ - try: - body = resp.json() - except (AttributeError, ValueError): - return [] - data = body.get("data") if isinstance(body, dict) else None - if not isinstance(data, dict): - return [] - rows = data.get("data") - return rows if isinstance(rows, list) else [] - - -def _build_chatroom_body(page_num: int, page_size: int) -> dict[str, Any]: - """Build the ``getChatroomList`` request body for a given page. - - The live capture carries no filter parameters — just the page slice. - """ - return {"pageNum": page_num, "pageSize": page_size} - - -def _parse_member_ids(member_ids: Any) -> list[str]: - """Split a semicolon-joined member id list, dropping empty segments. - - Upstream ``memberIds`` looks like ``"wxid_a;wxid_b;csf7169708"`` and may - carry a trailing ``";"`` (yielding an empty segment). Empties are dropped - so each entry is a real wxid; a missing/None/non-string value yields ``[]``. - """ - if not isinstance(member_ids, str): - return [] - return [seg for seg in member_ids.split(";") if seg] + return {"pageNum": page_num, "pageSize": page_size, "friendTypeYunke": "2"} def _normalize_chat_group(record: dict[str, Any]) -> dict[str, Any]: - """Map one upstream ``getChatroomList`` record to the ``ChatGroup`` shape. + """Map one ``friend_list/es`` group record (friendTypeYunke=2) to ChatGroup. + + Field mapping (verified 2026-08-10 Playwright capture): + + * ``friendWxId`` -> ``room_id`` (the ``@chatroom`` id) + * ``friendWxNickname`` -> ``room_name`` (the group name) + * ``friendNum`` -> ``member_count`` + * ``friendHeadImageUrl`` -> ``head_image_url`` (group avatar) + * ``wxId`` -> ``owner_account_id`` (staff wechat account) + * ``ownerName`` -> ``owner_account_nickname`` (group owner nickname) + * ``userId`` -> ``owner_user_id`` (owning staff id) + * ``phone`` / ``wechatPhone`` -> ``owner_user_phone`` (phone preferred) + * ``messageCount`` -> ``message_count`` + * ``sendNum`` -> ``own_message_count`` (this account's sends) + * ``lastChatTime`` -> ``last_chat_time`` (ms-epoch) + * ``version`` -> ``version`` + + Member ids are NOT surfaced by ``friend_list/es`` (only the ``friendNum`` + count is); ``member_ids`` / ``member_ids_list`` are emitted as ``None`` / + ``[]`` so the column shape stays stable. Group-member mirroring is + deferred to a future endpoint. Likewise ``loss_members`` / ``add_members`` + / ``update_time`` are not carried by this endpoint and are emitted as + ``None``. Adds ``channel="wechat"`` and preserves the original record under - ``raw_json``. The raw ``memberIds`` string is carried verbatim as - ``member_ids`` (matching the ``ChatGroup.member_ids`` Text column) AND - parsed into ``member_ids_list`` (a plain list of wxids with empties - dropped) so :func:`app.storage.repos.upsert_chat_group` can mirror one - :class:`app.db.models.GroupMember` row per entry without re-parsing. - - ``None``-safe: missing fields become ``None`` rather than raising, so a - partial payload does not break the whole scrape. + ``raw_json``. ``None``-safe on every field: missing fields become ``None`` + rather than raising, so a partial payload does not break the scrape. """ - member_ids = record.get("memberIds") return { - "room_id": record.get("chatRoomId"), + "room_id": record.get("friendWxId"), "channel": "wechat", - "room_name": record.get("chatRoomName"), - "owner_account_id": record.get("wechatId"), - "owner_account_nickname": record.get("wechatNickName"), - "member_count": record.get("memberCount"), - "member_ids": member_ids, - "member_ids_list": _parse_member_ids(member_ids), - "head_image_url": record.get("headImageUrl"), + "room_name": record.get("friendWxNickname"), + "owner_account_id": record.get("wxId"), + "owner_account_nickname": record.get("ownerName"), + "member_count": record.get("friendNum"), + "member_ids": None, + "member_ids_list": [], + "head_image_url": record.get("friendHeadImageUrl"), "owner_user_id": record.get("userId"), - "owner_user_phone": record.get("userPhone"), + "owner_user_phone": record.get("phone") or record.get("wechatPhone"), "message_count": record.get("messageCount"), - "own_message_count": record.get("ownMessageCount"), - "loss_members": record.get("lossMembers"), - "add_members": record.get("addMembers"), + "own_message_count": record.get("sendNum"), + "loss_members": None, + "add_members": None, "last_chat_time": record.get("lastChatTime"), - "update_time": record.get("updateTime"), + "update_time": None, "version": record.get("version"), "raw_json": record, } @@ -945,17 +918,18 @@ async def scrape_wechat_groups( *, page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: - """Yield normalized WeChat chat-group records from ``getChatroomList``. + """Yield normalized WeChat chat-group records from ``friend_list/es``. - Walks ``POST /wechatweb/chatroom/getChatroomList`` page by page until the - upstream reports no more rows (empty ``data.data`` array — the primary - stop signal), mapping each record to the ``ChatGroup`` column shape plus - a verbatim ``raw_json`` copy. + Walks ``POST /wechatweb/wxFriends/friend_list/es`` page by page (the SAME + endpoint :func:`scrape_wechat_friends` uses) with ``friendTypeYunke="2"`` + in the body to select groups (``"1"`` selects friends). The response + envelope is identical to the friend scraper's (``data.basicResult.data`` + + ``data.basicResult.total``), so the shared ``_parse_total`` / + ``_parse_rows`` helpers are reused. - The semicolon-joined ``memberIds`` field is carried both raw - (``member_ids``) and parsed (``member_ids_list``, empties dropped) so the - persistence layer can mirror :class:`app.db.models.GroupMember` rows - idempotently. + Each record is mapped to the ``ChatGroup`` column shape plus a verbatim + ``raw_json`` copy. Member ids are NOT available from this endpoint — + ``member_ids`` / ``member_ids_list`` are emitted as ``None`` / ``[]``. Parameters ---------- @@ -968,11 +942,11 @@ async def scrape_wechat_groups( """ async for record in paginate_by_page( client, - CHATROOM_LIST_PATH, + FRIEND_LIST_PATH, page_size=page_size, - parse_total=_parse_chatroom_total, - parse_rows=_parse_chatroom_rows, - build_body=_build_chatroom_body, + parse_total=_parse_total, + parse_rows=_parse_rows, + build_body=_build_group_body, ): yield _normalize_chat_group(record) diff --git a/yunke-server/tests/cassettes/chatroom_fixture.py b/yunke-server/tests/cassettes/chatroom_fixture.py deleted file mode 100644 index 2e3c367..0000000 --- a/yunke-server/tests/cassettes/chatroom_fixture.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Fixture payloads for ``POST /wechatweb/chatroom/getChatroomList`` (M2). - -These records mirror the real field names captured from the live CRM (see the -data dictionary §2: ``chatRoomId(@chatroom), chatRoomName, memberCount, -memberIds(分号分隔全部成员wxid), headImageUrl, wechatId/wechatNickName(归属), -userName/userPhone(群主员工), messageCount/ownMessageCount, lossMembers/ -addMembers, lastChatTime/updateTime``; total=1515). - -The envelope is the same flat shape as ``wxUser/list`` (no ``basicResult`` -wrapper): pagination fields sit directly under ``data``:: - - {"message":"调用成功","code":"10000","success":true,"traceId":"...", - "data":{"data":[...], "total":.., "page":.., "pageCount":..}} - -The cassette ships three records across two pages so a test can drive the -page-number paginator across the empty-page stop signal: - -* page 1 -> 2 of the 3 fixture records, ``total=3`` -* page 2 -> the remaining 1 record, ``total=3`` -* (page 3 -> empty ``data`` array, served inline by the test) - -Records carry every field with a target ``ChatGroup`` column so the test can -assert a full mapping end-to-end. The ``memberIds`` field is the -semicolon-joined member wxid list; record 2 deliberately includes a trailing -empty segment (``"...;"``) so the parser's drop-empties path is exercised. -""" - -# The module name mirrors the upstream endpoint name (``chatroom``); the M2 -# spec mandates this exact filename. -from __future__ import annotations - -from typing import Any - -# --------------------------------------------------------------------------- -# Capture-shaped chatroom records (3 total across 2 pages). -# --------------------------------------------------------------------------- - -CHATROOM_RECORDS: list[dict[str, Any]] = [ - { - "chatRoomId": "12345678901@chatroom", - "chatRoomName": "VIP客户交流群", - "memberCount": 3, - # 3 members, no empties -> member_ids_list length 3. - "memberIds": "wxid_member_a1;wxid_member_a2;csf7169708", - "headImageUrl": "https://wx.qlogo.cn/room_12345678901.png", - "wechatId": "wxid_staff_alice01", - "wechatNickName": "Alice WeChat", - "userId": "staff_user_001", - "userName": "Alice Wang", - "userPhone": "13800000001", - "messageCount": 4521, - "ownMessageCount": 1204, - "lossMembers": 2, - "addMembers": 5, - "lastChatTime": 1700000000123, - "updateTime": 1700000001000, - "version": "v1.20260731.room1", - }, - { - "chatRoomId": "98765432109@chatroom", - "chatRoomName": "华东销售大群", - "memberCount": 2, - # Trailing empty segment -> member_ids_list length 2 (empties dropped). - "memberIds": "wxid_member_b1;wxid_member_b2;", - "headImageUrl": "https://wx.qlogo.cn/room_98765432109.png", - "wechatId": "wxid_staff_bob02", - "wechatNickName": "Bob WeChat", - "userId": "staff_user_002", - "userName": "Bob Chen", - "userPhone": "13800000002", - "messageCount": 9821, - "ownMessageCount": 211, - "lossMembers": 0, - "addMembers": 1, - "lastChatTime": 1700000000456, - "updateTime": 1700000000800, - "version": "v1.20260731.room2", - }, - { - # A group whose owner has no resolved staff id (userId None) — the - # scraper must still yield it; the repo must still upsert its members. - "chatRoomId": "55555555555@chatroom", - "chatRoomName": "External Coordination", - "memberCount": 1, - "memberIds": "wxid_member_c1", - "headImageUrl": "https://wx.qlogo.cn/room_55555555555.png", - "wechatId": "wxid_staff_orphan04", - "wechatNickName": "Orphan WeChat", - "userId": None, - "userName": None, - "userPhone": None, - "messageCount": 12, - "ownMessageCount": 0, - "lossMembers": 0, - "addMembers": 0, - "lastChatTime": 1700000000789, - "updateTime": 1700000000900, - "version": "v1.20260731.room3", - }, -] - -# Default page size for the fixture (and the scraper). total=3 means the -# fixture fills exactly 2 pages of size 2 plus a terminating empty page. -TOTAL = len(CHATROOM_RECORDS) -DEFAULT_PAGE_SIZE = 2 - - -def make_chatroom_list_response( - records: list[dict[str, Any]], - *, - total: int, - page_num: int = 1, - page_size: int = DEFAULT_PAGE_SIZE, - trace_id: str | None = None, -) -> dict[str, Any]: - """Build a capture-shaped ``getChatroomList`` response envelope. - - Shape (no ``basicResult`` wrapper; pagination fields directly under - ``data``):: - - {"message":"调用成功","code":"10000","success":true,"traceId":"...", - "data":{"data":[...], "total":.., "page":.., "pageCount":..}} - """ - return { - "message": "调用成功", - "code": "10000", - "success": True, - "traceId": trace_id or f"trace-{page_num}", - "data": { - "data": list(records), - "total": total, - "page": page_num, - "pageCount": ( - (total + page_size - 1) // page_size if page_size else 1 - ), - }, - } diff --git a/yunke-server/tests/test_scrape_wechat_groups.py b/yunke-server/tests/test_scrape_wechat_groups.py index 2d928f2..e19c7ff 100644 --- a/yunke-server/tests/test_scrape_wechat_groups.py +++ b/yunke-server/tests/test_scrape_wechat_groups.py @@ -4,6 +4,13 @@ Drives the real page-number pagination + normalization logic through ``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the login flow is skipped) against a mocked transport. +Groups reuse the SAME upstream endpoint as friends +(``POST /wechatweb/wxFriends/friend_list/es``); the ``friendTypeYunke="2"`` +body parameter selects groups (``"1"`` selects friends). The response +envelope is identical to the friend scraper's, so this test reuses +:func:`tests.cassettes.wxFriends_fixture.make_friend_list_response` to build +capture-shaped envelopes. + The fixture ships 3 records across 2 pages of size 2, plus a terminating empty page (the documented primary stop signal for page-number endpoints): @@ -12,33 +19,89 @@ empty page (the documented primary stop signal for page-number endpoints): * page 3 -> empty ``data`` array (no more rows) The scraper must yield exactly the 3 records (normalized to ``ChatGroup`` -snake_case columns + ``raw_json``), parse ``memberIds`` into -``member_ids_list`` (dropping empty segments), make exactly 3 calls, and stop -without a 4th. +snake_case columns + ``raw_json``), make exactly 3 calls, and stop without a +4th. Member details are NOT surfaced by ``friend_list/es`` — ``member_ids`` +is ``None`` and ``member_ids_list`` is ``[]`` on every yielded record. """ from __future__ import annotations import json as _json +from typing import Any import httpx import respx from app.scraper.wechat import scrape_wechat_groups from app.session.yunke_client import YunkeClient -from tests.cassettes.chatroom_fixture import ( - CHATROOM_RECORDS, - DEFAULT_PAGE_SIZE, - TOTAL, - make_chatroom_list_response, -) +from tests.cassettes.wxFriends_fixture import make_friend_list_response BASE = "https://crm2.yunkecn.com" -CHATROOM_LIST_URL = f"{BASE}/wechatweb/chatroom/getChatroomList" +FRIEND_LIST_URL = f"{BASE}/wechatweb/wxFriends/friend_list/es" USERID = "a" * 32 COMPANY = "8m0atd" +# Realistic, capture-shaped group records. Field names are EXACTLY what the +# live ``friend_list/es`` endpoint returns for ``friendTypeYunke="2"`` +# (verified 2026-08-10 Playwright capture). Every field with a target +# ``ChatGroup`` column is populated so the test asserts a full mapping +# end-to-end. (Member ids are NOT in this payload — the endpoint surfaces +# only the ``friendNum`` count — so ``member_ids`` / ``member_ids_list`` are +# exercised as None / [] separately.) +GROUP_RECORDS: list[dict[str, Any]] = [ + { + "friendWxId": "12345678901@chatroom", + "friendWxNickname": "VIP客户交流群", + "friendNum": 3, + "friendHeadImageUrl": "https://wx.qlogo.cn/room_12345678901.png", + "wxId": "wxid_staff_alice01", + "ownerName": "Alice WeChat", + "userId": "staff_user_001", + "phone": "13800000001", + "wechatPhone": "13800000011", + "messageCount": 4521, + "sendNum": 1204, + "lastChatTime": 1700000000123, + "version": "v1.20260731.room1", + }, + { + "friendWxId": "98765432109@chatroom", + "friendWxNickname": "华东销售大群", + "friendNum": 2, + "friendHeadImageUrl": "https://wx.qlogo.cn/room_98765432109.png", + "wxId": "wxid_staff_bob02", + "ownerName": "Bob WeChat", + "userId": "staff_user_002", + "phone": "13800000002", + "wechatPhone": "13800000022", + "messageCount": 9821, + "sendNum": 211, + "lastChatTime": 1700000000456, + "version": "v1.20260731.room2", + }, + { + # A group whose owner has no resolved staff id (userId None) — the + # scraper must still yield it; None-safe fields stay None. + "friendWxId": "55555555555@chatroom", + "friendWxNickname": "External Coordination", + "friendNum": 1, + "friendHeadImageUrl": "https://wx.qlogo.cn/room_55555555555.png", + "wxId": "wxid_staff_orphan04", + "ownerName": "Orphan WeChat", + "userId": None, + "phone": None, + "wechatPhone": None, + "messageCount": 12, + "sendNum": 0, + "lastChatTime": 1700000000789, + "version": "v1.20260731.room3", + }, +] + +TOTAL = len(GROUP_RECORDS) +DEFAULT_PAGE_SIZE = 2 + def _primed_client() -> YunkeClient: """A YunkeClient already in a logged-in state (login flow skipped).""" @@ -46,24 +109,24 @@ def _primed_client() -> YunkeClient: # --------------------------------------------------------------------------- -# End-to-end: pagination + field mapping + member_ids_list parsing +# End-to-end: pagination + field mapping # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_groups_yields_records_and_paginates(): """Two full pages + a terminating empty page -> 3 records, 3 calls.""" - page1 = make_chatroom_list_response( - CHATROOM_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE + page1 = make_friend_list_response( + GROUP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE ) - page2 = make_chatroom_list_response( - CHATROOM_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE + page2 = make_friend_list_response( + GROUP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE ) - page3 = make_chatroom_list_response( + page3 = make_friend_list_response( [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE ) - route = respx.post(CHATROOM_LIST_URL).mock( + route = respx.post(FRIEND_LIST_URL).mock( side_effect=[ httpx.Response(200, json=page1), httpx.Response(200, json=page2), @@ -85,39 +148,41 @@ async def test_scrape_wechat_groups_yields_records_and_paginates(): # Exactly the 3 fixture records (not duplicated, not dropped). assert len(results) == 3 assert {r["room_id"] for r in results} == { - rec["chatRoomId"] for rec in CHATROOM_RECORDS + rec["friendWxId"] for rec in GROUP_RECORDS } # Field mapping for each record (compare against the source record). - for result, record in zip(results, CHATROOM_RECORDS, strict=True): + for result, record in zip(results, GROUP_RECORDS, strict=True): # Identity + channel. - assert result["room_id"] == record["chatRoomId"] + assert result["room_id"] == record["friendWxId"] assert result["channel"] == "wechat" # Naming + ownership. - assert result["room_name"] == record["chatRoomName"] - assert result["owner_account_id"] == record["wechatId"] - assert result["owner_account_nickname"] == record["wechatNickName"] + assert result["room_name"] == record["friendWxNickname"] + assert result["owner_account_id"] == record["wxId"] + assert result["owner_account_nickname"] == record["ownerName"] assert result["owner_user_id"] == record["userId"] - assert result["owner_user_phone"] == record["userPhone"] + # phone preferred, wechatPhone fallback. + assert result["owner_user_phone"] == ( + record["phone"] or record["wechatPhone"] + ) # Counts. - assert result["member_count"] == record["memberCount"] + assert result["member_count"] == record["friendNum"] assert result["message_count"] == record["messageCount"] - assert result["own_message_count"] == record["ownMessageCount"] - assert result["loss_members"] == record["lossMembers"] - assert result["add_members"] == record["addMembers"] + assert result["own_message_count"] == record["sendNum"] # Avatar + timestamps. - assert result["head_image_url"] == record["headImageUrl"] + assert result["head_image_url"] == record["friendHeadImageUrl"] assert result["last_chat_time"] == record["lastChatTime"] - assert result["update_time"] == record["updateTime"] - # member_ids carried raw; member_ids_list parsed (empties dropped). - assert result["member_ids"] == record["memberIds"] - assert result["member_ids_list"] == [ - seg for seg in record["memberIds"].split(";") if seg - ] + # Member details NOT surfaced by friend_list/es -> None / []. + assert result["member_ids"] is None + assert result["member_ids_list"] == [] + # Fields the new endpoint does not carry. + assert result["loss_members"] is None + assert result["add_members"] is None + assert result["update_time"] is None # Version carried verbatim; raw_json preserves the original record. assert result["version"] == record["version"] @@ -125,46 +190,26 @@ async def test_scrape_wechat_groups_yields_records_and_paginates(): # --------------------------------------------------------------------------- -# member_ids_list parsing edge cases +# member_ids behaviour: always None / [] (not surfaced by friend_list/es) # --------------------------------------------------------------------------- @respx.mock -async def test_scrape_wechat_groups_drops_empty_member_id_segments(): - """Trailing/empty ``memberIds`` segments are dropped from member_ids_list.""" - # Record 2 has a trailing ";" -> one empty segment that must be dropped. - page1 = make_chatroom_list_response( - [CHATROOM_RECORDS[1]], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE - ) - page2 = make_chatroom_list_response([], total=1, page_num=2) - respx.post(CHATROOM_LIST_URL).mock( +async def test_scrape_wechat_groups_emits_empty_member_ids(): + """``friend_list/es`` does not surface member ids; they are always + ``None`` / ``[]`` on yielded records (member mirroring deferred).""" + record = dict(GROUP_RECORDS[0]) + page1 = make_friend_list_response([record], total=1, page_num=1) + page2 = make_friend_list_response([], total=1, page_num=2) + respx.post(FRIEND_LIST_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) client = _primed_client() try: - results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)] - finally: - await client.aclose() - - assert len(results) == 1 - # "wxid_member_b1;wxid_member_b2;" -> 2 ids (trailing empty dropped). - assert results[0]["member_ids_list"] == ["wxid_member_b1", "wxid_member_b2"] - - -@respx.mock -async def test_scrape_wechat_groups_handles_missing_member_ids(): - """A group with no ``memberIds`` field yields an empty member_ids_list.""" - record = {"chatRoomId": "x@chatroom", "chatRoomName": "Empty"} - page1 = make_chatroom_list_response([record], total=1, page_num=1) - page2 = make_chatroom_list_response([], total=1, page_num=2) - respx.post(CHATROOM_LIST_URL).mock( - side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] - ) - - client = _primed_client() - try: - results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)] + results = [ + r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE) + ] finally: await client.aclose() @@ -174,18 +219,18 @@ async def test_scrape_wechat_groups_handles_missing_member_ids(): # --------------------------------------------------------------------------- -# Pagination body: pageNum/pageSize advanced correctly +# Pagination body: pageNum/pageSize advanced + friendTypeYunke="2" # --------------------------------------------------------------------------- @respx.mock async def test_scrape_wechat_groups_sends_pagination_body(): - """Outgoing request body carries pageNum/pageSize; advances per page.""" - page1 = make_chatroom_list_response( - [CHATROOM_RECORDS[0]], total=TOTAL, page_num=1, page_size=1 + """Outgoing body carries pageNum/pageSize and ``friendTypeYunke="2"``.""" + page1 = make_friend_list_response( + [GROUP_RECORDS[0]], total=TOTAL, page_num=1, page_size=1 ) - page2 = make_chatroom_list_response([], total=TOTAL, page_num=2, page_size=1) - route = respx.post(CHATROOM_LIST_URL).mock( + page2 = make_friend_list_response([], total=TOTAL, page_num=2, page_size=1) + route = respx.post(FRIEND_LIST_URL).mock( side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)] ) @@ -202,10 +247,13 @@ async def test_scrape_wechat_groups_sends_pagination_body(): body1 = _json.loads(route.calls[0].request.content.decode()) assert body1["pageNum"] == 1 assert body1["pageSize"] == 1 + # The crucial switch: friendTypeYunke="2" selects GROUPS ("1" = friends). + assert body1["friendTypeYunke"] == "2" body2 = _json.loads(route.calls[1].request.content.decode()) assert body2["pageNum"] == 2 assert body2["pageSize"] == 1 + assert body2["friendTypeYunke"] == "2" # --------------------------------------------------------------------------- @@ -216,8 +264,8 @@ async def test_scrape_wechat_groups_sends_pagination_body(): @respx.mock async def test_scrape_wechat_groups_stops_on_empty_first_page(): """If the very first page is empty, scraper stops without over-fetching.""" - empty = make_chatroom_list_response([], total=0, page_num=1) - route = respx.post(CHATROOM_LIST_URL).mock( + empty = make_friend_list_response([], total=0, page_num=1) + route = respx.post(FRIEND_LIST_URL).mock( return_value=httpx.Response(200, json=empty) )