浅念
201 lines
7.1 KiB
Python
201 lines
7.1 KiB
Python
"""Tests for the WeChat friend-list scraper (Task 1.5).
|
|
|
|
Drives the *real* pagination + normalization logic through ``respx`` by wiring
|
|
a real ``YunkeClient`` (primed with a fake session so the login flow is
|
|
skipped) against a mocked transport. Two pages are served:
|
|
|
|
* page 1 -> total=2, both records (i.e. the full result set in one page),
|
|
* page 2 -> total=2, empty ``data`` array (the "no more rows" signal).
|
|
|
|
The paginator must yield exactly the 2 records (normalized), make exactly 2
|
|
calls, then stop without a 3rd call.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import respx
|
|
|
|
from app.scraper.wechat import scrape_wechat_friends
|
|
from app.session.yunke_client import YunkeClient
|
|
from tests.cassettes.wxFriends_fixture import (
|
|
FRIEND_RECORDS,
|
|
make_friend_list_response,
|
|
)
|
|
|
|
BASE = "https://crm2.yunkecn.com"
|
|
FRIEND_LIST_URL = f"{BASE}/wechatweb/wxFriends/friend_list/es"
|
|
|
|
USERID = "a" * 32
|
|
COMPANY = "8m0atd"
|
|
|
|
|
|
def _primed_client() -> YunkeClient:
|
|
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
|
c = YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
|
return c
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_friends_yields_normalized_records_and_paginates():
|
|
"""Two pages served; scraper yields 2 mapped dicts and makes exactly 2 calls."""
|
|
# Page 1: both records, total=2. Page 2: empty data, total still 2.
|
|
page1 = make_friend_list_response(FRIEND_RECORDS, total=len(FRIEND_RECORDS), page_num=1)
|
|
page2 = make_friend_list_response([], total=len(FRIEND_RECORDS), page_num=2)
|
|
|
|
route = respx.post(FRIEND_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wechat_friends(client)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# --- Pagination behaviour: exactly 2 calls, no speculative 3rd. ---
|
|
assert route.call_count == 2
|
|
|
|
# --- Yielded exactly the 2 fixture records (not duplicated, not dropped). ---
|
|
assert len(results) == 2
|
|
assert {r["friend_id"] for r in results} == {
|
|
rec["friendWxId"] for rec in FRIEND_RECORDS
|
|
}
|
|
|
|
# --- Field mapping for each record (compare against the source record). ---
|
|
for result, record in zip(results, FRIEND_RECORDS, strict=True):
|
|
# Identity / ownership / channel.
|
|
assert result["friend_id"] == record["friendWxId"]
|
|
assert result["channel"] == "wechat"
|
|
assert result["owner_account_id"] == record["wxId"]
|
|
assert result["user_id"] == record["userId"]
|
|
|
|
# Naming.
|
|
assert result["friend_nickname"] == record["friendWxNickname"]
|
|
assert result["friend_remark"] == record["friendWxRemark"]
|
|
assert result["real_name"] == record["realName"]
|
|
|
|
# Phones: `phone` <- "phone", `wechat_phone` <- "wechatPhone".
|
|
assert result["phone"] == record["phone"]
|
|
assert result["wechat_phone"] == record["wechatPhone"]
|
|
|
|
# Misc mapped fields.
|
|
assert result["gender"] == record["gender"]
|
|
assert result["region"] == record["region"]
|
|
assert result["description"] == record["description"]
|
|
assert result["sale_name"] == record["saleName"]
|
|
assert result["dept_route"] == record["deptRoute"]
|
|
assert result["dept_name"] == record["deptName"]
|
|
assert result["head_image_url"] == record["friendHeadImageUrl"]
|
|
assert result["last_chat_time"] == record["lastChatTime"]
|
|
assert result["send_num"] == record["sendNum"]
|
|
assert result["receive_num"] == record["receiveNum"]
|
|
assert result["message_count"] == record["messageCount"]
|
|
assert result["delete_flag"] == record["deleteFlag"]
|
|
assert result["is_block"] == record["isBlock"]
|
|
assert result["friend_state"] == record["friendState"]
|
|
|
|
# Version carried verbatim; raw_json preserves the original record.
|
|
assert result["version"] == record["version"]
|
|
assert result["raw_json"] == record
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_friends_sends_pagination_body():
|
|
"""Outgoing request carries pageNum/pageSize and optional filters."""
|
|
page1 = make_friend_list_response([], total=0, page_num=1)
|
|
route = respx.post(FRIEND_LIST_URL).mock(
|
|
return_value=httpx.Response(200, json=page1)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_friends(
|
|
client, department_id="dept-9", user_id="staff-1", page_size=7
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# Empty upstream -> no rows, exactly one call (the empty-page stop).
|
|
assert results == []
|
|
assert route.call_count == 1
|
|
|
|
sent = route.calls.last.request
|
|
import json as _json
|
|
|
|
body = _json.loads(sent.content.decode())
|
|
assert body["pageNum"] == 1
|
|
assert body["pageSize"] == 7
|
|
assert body["departmentId"] == "dept-9"
|
|
assert body["userId"] == "staff-1"
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_friends_stops_on_empty_first_page():
|
|
"""If the very first page is empty, the scraper stops without over-fetching."""
|
|
empty = make_friend_list_response([], total=0, page_num=1)
|
|
route = respx.post(FRIEND_LIST_URL).mock(
|
|
return_value=httpx.Response(200, json=empty)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wechat_friends(client)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert results == []
|
|
assert route.call_count == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# add_time mapping: addTime (string 'YYYY-MM-DD HH:mm:ss' OR ms-epoch int)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_normalize_maps_add_time_string_to_epoch():
|
|
"""addTime as a wall-clock string parses to a plausible recent ms epoch.
|
|
|
|
The CRM delivers wechat ``addTime`` as ``'YYYY-MM-DD HH:mm:ss'`` (confirmed
|
|
in smoke.db raw_json). ``parse_datetime_to_ms`` parses naive UTC; we assert
|
|
only that the result is an int in the right magnitude for a 2026 date
|
|
(any 2026 moment > 1.76e12) rather than pinning a specific epoch so the
|
|
test is not brittle to the exact format-resolution / tz convention.
|
|
"""
|
|
from app.scraper.wechat import _normalize
|
|
|
|
record = {
|
|
"friendWxId": "x",
|
|
"addTime": "2026-04-28 16:24:39",
|
|
"lastChatTime": 1700000000000,
|
|
}
|
|
out = _normalize(record)
|
|
assert isinstance(out["add_time"], int)
|
|
assert out["add_time"] > 1_760_000_000_000
|
|
|
|
|
|
def test_normalize_maps_add_time_int_passthrough():
|
|
"""addTime as a numeric ms-epoch passes through verbatim (int or float)."""
|
|
from app.scraper.wechat import _normalize
|
|
|
|
record = {"friendWxId": "x", "addTime": 1690000000000, "lastChatTime": None}
|
|
out = _normalize(record)
|
|
assert out["add_time"] == 1690000000000
|
|
|
|
|
|
def test_normalize_add_time_none_when_missing():
|
|
"""Missing / empty addTime maps to None (None-safe field)."""
|
|
from app.scraper.wechat import _normalize
|
|
|
|
out = _normalize({"friendWxId": "x"})
|
|
assert out["add_time"] is None
|
|
|
|
out_empty = _normalize({"friendWxId": "x", "addTime": ""})
|
|
assert out_empty["add_time"] is None
|