浅念
2a74d3888d
Add scrape_wechat_groups (POST /wechatweb/chatroom/getChatroomList) and scrape_wechat_talkers (POST /wechatweb/wxFriends/talkerList), both reusing the page-number paginator. Groups parse the semicolon-joined memberIds into a member_ids_list so upsert_chat_group can mirror GroupMember rows (composite PK channel/room_id/member_id) idempotently alongside the ChatGroup row. Talkers are a sync-planning input (member_count>0 marks a group conversation); no dedicated table. Co-Authored-By: Claude <noreply@anthropic.com>
232 lines
8.2 KiB
Python
232 lines
8.2 KiB
Python
"""Tests for the WeChat chat-group scraper (M2).
|
|
|
|
Drives the real page-number pagination + normalization logic through
|
|
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
|
|
login flow is skipped) against a mocked transport.
|
|
|
|
The fixture ships 3 records across 2 pages of size 2, plus a terminating
|
|
empty page (the documented primary stop signal for page-number endpoints):
|
|
|
|
* page 1 -> 2 records, ``total=3``
|
|
* page 2 -> 1 record, ``total=3``
|
|
* page 3 -> empty ``data`` array (no more rows)
|
|
|
|
The scraper must yield exactly the 3 records (normalized to ``ChatGroup``
|
|
snake_case columns + ``raw_json``), parse ``memberIds`` into
|
|
``member_ids_list`` (dropping empty segments), make exactly 3 calls, and stop
|
|
without a 4th.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json as _json
|
|
|
|
import httpx
|
|
import respx
|
|
|
|
from app.scraper.wechat import scrape_wechat_groups
|
|
from app.session.yunke_client import YunkeClient
|
|
from tests.cassettes.chatroom_fixture import (
|
|
CHATROOM_RECORDS,
|
|
DEFAULT_PAGE_SIZE,
|
|
TOTAL,
|
|
make_chatroom_list_response,
|
|
)
|
|
|
|
BASE = "https://crm2.yunkecn.com"
|
|
CHATROOM_LIST_URL = f"{BASE}/wechatweb/chatroom/getChatroomList"
|
|
|
|
USERID = "a" * 32
|
|
COMPANY = "8m0atd"
|
|
|
|
|
|
def _primed_client() -> YunkeClient:
|
|
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
|
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# End-to-end: pagination + field mapping + member_ids_list parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_groups_yields_records_and_paginates():
|
|
"""Two full pages + a terminating empty page -> 3 records, 3 calls."""
|
|
page1 = make_chatroom_list_response(
|
|
CHATROOM_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_chatroom_list_response(
|
|
CHATROOM_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page3 = make_chatroom_list_response(
|
|
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
|
|
route = respx.post(CHATROOM_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
httpx.Response(200, json=page3),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
|
|
assert route.call_count == 3
|
|
|
|
# Exactly the 3 fixture records (not duplicated, not dropped).
|
|
assert len(results) == 3
|
|
assert {r["room_id"] for r in results} == {
|
|
rec["chatRoomId"] for rec in CHATROOM_RECORDS
|
|
}
|
|
|
|
# Field mapping for each record (compare against the source record).
|
|
for result, record in zip(results, CHATROOM_RECORDS, strict=True):
|
|
# Identity + channel.
|
|
assert result["room_id"] == record["chatRoomId"]
|
|
assert result["channel"] == "wechat"
|
|
|
|
# Naming + ownership.
|
|
assert result["room_name"] == record["chatRoomName"]
|
|
assert result["owner_account_id"] == record["wechatId"]
|
|
assert result["owner_account_nickname"] == record["wechatNickName"]
|
|
assert result["owner_user_id"] == record["userId"]
|
|
assert result["owner_user_phone"] == record["userPhone"]
|
|
|
|
# Counts.
|
|
assert result["member_count"] == record["memberCount"]
|
|
assert result["message_count"] == record["messageCount"]
|
|
assert result["own_message_count"] == record["ownMessageCount"]
|
|
assert result["loss_members"] == record["lossMembers"]
|
|
assert result["add_members"] == record["addMembers"]
|
|
|
|
# Avatar + timestamps.
|
|
assert result["head_image_url"] == record["headImageUrl"]
|
|
assert result["last_chat_time"] == record["lastChatTime"]
|
|
assert result["update_time"] == record["updateTime"]
|
|
|
|
# member_ids carried raw; member_ids_list parsed (empties dropped).
|
|
assert result["member_ids"] == record["memberIds"]
|
|
assert result["member_ids_list"] == [
|
|
seg for seg in record["memberIds"].split(";") if seg
|
|
]
|
|
|
|
# Version carried verbatim; raw_json preserves the original record.
|
|
assert result["version"] == record["version"]
|
|
assert result["raw_json"] == record
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# member_ids_list parsing edge cases
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_groups_drops_empty_member_id_segments():
|
|
"""Trailing/empty ``memberIds`` segments are dropped from member_ids_list."""
|
|
# Record 2 has a trailing ";" -> one empty segment that must be dropped.
|
|
page1 = make_chatroom_list_response(
|
|
[CHATROOM_RECORDS[1]], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_chatroom_list_response([], total=1, page_num=2)
|
|
respx.post(CHATROOM_LIST_URL).mock(
|
|
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert len(results) == 1
|
|
# "wxid_member_b1;wxid_member_b2;" -> 2 ids (trailing empty dropped).
|
|
assert results[0]["member_ids_list"] == ["wxid_member_b1", "wxid_member_b2"]
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_groups_handles_missing_member_ids():
|
|
"""A group with no ``memberIds`` field yields an empty member_ids_list."""
|
|
record = {"chatRoomId": "x@chatroom", "chatRoomName": "Empty"}
|
|
page1 = make_chatroom_list_response([record], total=1, page_num=1)
|
|
page2 = make_chatroom_list_response([], total=1, page_num=2)
|
|
respx.post(CHATROOM_LIST_URL).mock(
|
|
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wechat_groups(client, page_size=DEFAULT_PAGE_SIZE)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert len(results) == 1
|
|
assert results[0]["member_ids"] is None
|
|
assert results[0]["member_ids_list"] == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pagination body: pageNum/pageSize advanced correctly
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_groups_sends_pagination_body():
|
|
"""Outgoing request body carries pageNum/pageSize; advances per page."""
|
|
page1 = make_chatroom_list_response(
|
|
[CHATROOM_RECORDS[0]], total=TOTAL, page_num=1, page_size=1
|
|
)
|
|
page2 = make_chatroom_list_response([], total=TOTAL, page_num=2, page_size=1)
|
|
route = respx.post(CHATROOM_LIST_URL).mock(
|
|
side_effect=[httpx.Response(200, json=page1), httpx.Response(200, json=page2)]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wechat_groups(client, page_size=1)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# One record on page 1, empty page 2 stops the loop.
|
|
assert len(results) == 1
|
|
assert route.call_count == 2
|
|
|
|
body1 = _json.loads(route.calls[0].request.content.decode())
|
|
assert body1["pageNum"] == 1
|
|
assert body1["pageSize"] == 1
|
|
|
|
body2 = _json.loads(route.calls[1].request.content.decode())
|
|
assert body2["pageNum"] == 2
|
|
assert body2["pageSize"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Empty first page -> no yields, single call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_groups_stops_on_empty_first_page():
|
|
"""If the very first page is empty, scraper stops without over-fetching."""
|
|
empty = make_chatroom_list_response([], total=0, page_num=1)
|
|
route = respx.post(CHATROOM_LIST_URL).mock(
|
|
return_value=httpx.Response(200, json=empty)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wechat_groups(client)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert results == []
|
|
assert route.call_count == 1
|