feat: wechat friend scraper

Add the first list scraper for the yunke data-mirror system, establishing the reusable pagination pattern all later list scrapers will build on. - app/scraper/base.py: generic `paginate_by_page` async generator that walks page-numbered endpoints until a page comes back empty (the yunke CRM's primary "no more rows" signal), with a defensive over-fetch ceiling keyed off the declared total (strict `>` so an honest source yields exactly `total`). Plus a tiny `RateLimiter` enforcing a min interval between calls. - app/scraper/wechat.py: `scrape_wechat_friends` wraps `POST /wechatweb/wxFriends/friend_list/es`, maps each record to the `Customer` column shape (snake_case) with `channel="wechat"` and a verbatim `raw_json` copy. Optional `department_id`/`user_id` filters pass through to the request body. - tests/cassettes/wxFriends_fixture.py: two realistic, capture-shaped friend records + a response-envelope builder, reused by the scraper tests. - tests/test_scrape_wechat_friends.py: drives the real pagination + normalization logic through respx (page1 returns both records, page2 empty -> stop). Asserts exact field mappings, `raw_json` fidelity, and exactly 2 upstream calls. - tests/test_scrape_base.py: smoke tests for the RateLimiter spacing. Persistence (upsert into `Customer`) is deliberately deferred to Task 1.7; scrapers remain pure producers of normalized dicts. Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-01 11:31:28 +08:00
parent 8243e9c6d2
commit 4a47e824c7
7 changed files with 650 additions and 0 deletions
@@ -0,0 +1,6 @@
+"""Test fixtures for yunke scrapers.
+
+Each module in this package holds realistic, capture-shaped payloads for one
+upstream endpoint so the scraper test suite can drive the real pagination /
+normalization code through ``respx`` without touching the live CRM.
+"""
@@ -0,0 +1,122 @@
+"""Fixture payloads for ``POST /wechatweb/wxFriends/friend_list/es``.
+
+These records mirror the real field names captured from the live CRM (see the
+Task 1.5 spec). Two realistic friend records are provided so a test can drive
+the paginator across two pages: page 1 returns both records, page 2 returns an
+empty ``data`` array to signal "no more" (the documented primary stop signal).
+"""
+# The module name uses camelCase deliberately: it mirrors the upstream endpoint
+# name (``wxFriends``) and the Task 1.5 spec mandates this exact filename.
+# ruff: noqa: N999
+
+from __future__ import annotations
+
+from typing import Any
+
+# Realistic, capture-shaped friend records. Field names are EXACTLY what the
+# live endpoint returns — the scraper maps these to ``Customer`` snake_case
+# columns. Every field that has a target column is populated so the test can
+# assert a full mapping end-to-end.
+FRIEND_RECORDS: list[dict[str, Any]] = [
+    {
+        "companyCode": "8m0atd",
+        "wxId": "wxid_staff_alice01",
+        "friendWxId": "wxid_friend_0001",
+        "friendWxAlias": "alice_alias",
+        "friendWxNickname": "Alice Wang",
+        "friendWxRemark": "Alice-VIP",
+        "realName": "Wang Ai Li",
+        "friendWxPhone": "13800000001",
+        "phone": "13800000001",
+        "wechatPhone": "13800000011",
+        "gender": 2,
+        "region": "Shanghai",
+        "description": "VIP client, prefers wechat",
+        "saleName": "Tom Sale",
+        "userId": "staff_user_001",
+        "deptRoute": "1.2.3",
+        "deptRouteName": "Sales/East/TeamA",
+        "deptName": "Sales-East-TeamA",
+        "friendHeadImageUrl": "https://cdn.example.com/avatar_alice.png",
+        "headImg": "https://cdn.example.com/avatar_alice_legacy.png",
+        "lastChatTime": 1700000000123,
+        "sendNum": 12,
+        "receiveNum": 34,
+        "messageCount": 46,
+        "deleteFlag": 0,
+        "isBlock": 0,
+        "friendState": 1,
+        "version": "v1.20260731.alice",
+        "addTime": 1690000000000,
+        "createTime": 1690000000000,
+        "fromType": "qrcode",
+        "addType": "scan",
+    },
+    {
+        "companyCode": "8m0atd",
+        "wxId": "wxid_staff_bob02",
+        "friendWxId": "wxid_friend_0002",
+        "friendWxAlias": "bob_alias",
+        "friendWxNickname": "Bob Chen",
+        "friendWxRemark": "Bob-Lead",
+        "realName": "Chen Bo",
+        "friendWxPhone": "13800000002",
+        "phone": "13800000002",
+        "wechatPhone": "13800000022",
+        "gender": 1,
+        "region": "Beijing",
+        "description": "New lead from expo",
+        "saleName": "Jerry Sale",
+        "userId": "staff_user_002",
+        "deptRoute": "1.2.4",
+        "deptRouteName": "Sales/North/TeamB",
+        "deptName": "Sales-North-TeamB",
+        "friendHeadImageUrl": "https://cdn.example.com/avatar_bob.png",
+        "headImg": "https://cdn.example.com/avatar_bob_legacy.png",
+        "lastChatTime": 1700000000456,
+        "sendNum": 3,
+        "receiveNum": 9,
+        "messageCount": 12,
+        "deleteFlag": 0,
+        "isBlock": 1,
+        "friendState": 2,
+        "version": "v1.20260731.bob",
+        "addTime": 1691000000000,
+        "createTime": 1691000000000,
+        "fromType": "phone",
+        "addType": "import",
+    },
+]
+
+
+def make_friend_list_response(
+    records: list[dict[str, Any]],
+    *,
+    total: int,
+    page_num: int = 1,
+    page_size: int = 50,
+) -> dict[str, Any]:
+    """Build a capture-shaped ``friend_list/es`` response envelope.
+
+    The shape matches the live API exactly:
+
+        {"message":"调用成功","code":"10000","success":true,"traceId":"...",
+         "data":{"displayStatus":"0",
+                 "basicResult":{"pageNum":..,"pageSize":..,"total":..,
+                                "data":[ {<friend record>}, ... ]}}}
+    """
+    return {
+        "message": "调用成功",
+        "code": "10000",
+        "success": True,
+        "traceId": f"trace-{page_num}",
+        "data": {
+            "displayStatus": "0",
+            "basicResult": {
+                "pageNum": page_num,
+                "pageSize": page_size,
+                "total": total,
+                "data": list(records),
+            },
+        },
+    }
@@ -0,0 +1,48 @@
+"""Smoke tests for the reusable scraper building blocks in :mod:`app.scraper.base`.
+
+The generic paginator is exercised end-to-end by the WeChat friend-scraper
+tests (it is the only caller today). Here we only sanity-check the
+:class:`RateLimiter`'s spacing behaviour so the helper is not shipped blind.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+from app.scraper.base import RateLimiter
+
+
+async def test_rate_limiter_no_interval_never_waits():
+    """``min_interval=0`` is a passthrough — acquire returns immediately."""
+    limiter = RateLimiter(min_interval=0.0)
+    start = time.monotonic()
+    for _ in range(5):
+        await limiter.acquire()
+    elapsed = time.monotonic() - start
+    # 5 acquires with no spacing should complete well under a real interval.
+    assert elapsed < 0.1
+
+
+async def test_rate_limiter_enforces_min_interval():
+    """Two acquires are spaced by at least ``min_interval`` seconds."""
+    limiter = RateLimiter(min_interval=0.05)
+    await limiter.acquire()  # first call never waits; primes ``_last``
+    start = time.monotonic()
+    await limiter.acquire()  # must wait >= 0.05s since the prior call
+    elapsed = time.monotonic() - start
+    assert elapsed >= 0.04  # allow a small scheduler slack margin
+
+
+async def test_rate_limiter_concurrent_acquire_is_safe():
+    """Concurrent acquires do not raise; each just waits its turn."""
+    limiter = RateLimiter(min_interval=0.01)
+    await asyncio.gather(*(limiter.acquire() for _ in range(8)))
+    # No assertion on timing here — the contract is "no crash, polite spacing".
+    # We only assert the gather completed without raising.
+
+
+def test_rate_limiter_is_instantiable_with_negative_interval():
+    """Negative intervals collapse to the passthrough branch (no surprising raise)."""
+    limiter = RateLimiter(min_interval=-1.0)
+    assert limiter._min_interval <= 0  # sanity check the guard
@@ -0,0 +1,154 @@
+"""Tests for the WeChat friend-list scraper (Task 1.5).
+
+Drives the *real* pagination + normalization logic through ``respx`` by wiring
+a real ``YunkeClient`` (primed with a fake session so the login flow is
+skipped) against a mocked transport. Two pages are served:
+
+* page 1 -> total=2, both records (i.e. the full result set in one page),
+* page 2 -> total=2, empty ``data`` array (the "no more rows" signal).
+
+The paginator must yield exactly the 2 records (normalized), make exactly 2
+calls, then stop without a 3rd call.
+"""
+
+from __future__ import annotations
+
+import httpx
+import respx
+
+from app.scraper.wechat import scrape_wechat_friends
+from app.session.yunke_client import YunkeClient
+from tests.cassettes.wxFriends_fixture import (
+    FRIEND_RECORDS,
+    make_friend_list_response,
+)
+
+BASE = "https://crm2.yunkecn.com"
+FRIEND_LIST_URL = f"{BASE}/wechatweb/wxFriends/friend_list/es"
+
+USERID = "a" * 32
+COMPANY = "8m0atd"
+
+
+def _primed_client() -> YunkeClient:
+    """A YunkeClient already in a logged-in state (login flow skipped)."""
+    c = YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
+    return c
+
+
+@respx.mock
+async def test_scrape_wechat_friends_yields_normalized_records_and_paginates():
+    """Two pages served; scraper yields 2 mapped dicts and makes exactly 2 calls."""
+    # Page 1: both records, total=2. Page 2: empty data, total still 2.
+    page1 = make_friend_list_response(FRIEND_RECORDS, total=len(FRIEND_RECORDS), page_num=1)
+    page2 = make_friend_list_response([], total=len(FRIEND_RECORDS), page_num=2)
+
+    route = respx.post(FRIEND_LIST_URL).mock(
+        side_effect=[
+            httpx.Response(200, json=page1),
+            httpx.Response(200, json=page2),
+        ]
+    )
+
+    client = _primed_client()
+    try:
+        results = [r async for r in scrape_wechat_friends(client)]
+    finally:
+        await client.aclose()
+
+    # --- Pagination behaviour: exactly 2 calls, no speculative 3rd. ---
+    assert route.call_count == 2
+
+    # --- Yielded exactly the 2 fixture records (not duplicated, not dropped). ---
+    assert len(results) == 2
+    assert {r["friend_id"] for r in results} == {
+        rec["friendWxId"] for rec in FRIEND_RECORDS
+    }
+
+    # --- Field mapping for each record (compare against the source record). ---
+    for result, record in zip(results, FRIEND_RECORDS, strict=True):
+        # Identity / ownership / channel.
+        assert result["friend_id"] == record["friendWxId"]
+        assert result["channel"] == "wechat"
+        assert result["owner_account_id"] == record["wxId"]
+        assert result["user_id"] == record["userId"]
+
+        # Naming.
+        assert result["friend_nickname"] == record["friendWxNickname"]
+        assert result["friend_remark"] == record["friendWxRemark"]
+        assert result["real_name"] == record["realName"]
+
+        # Phones: `phone` <- "phone", `wechat_phone` <- "wechatPhone".
+        assert result["phone"] == record["phone"]
+        assert result["wechat_phone"] == record["wechatPhone"]
+
+        # Misc mapped fields.
+        assert result["gender"] == record["gender"]
+        assert result["region"] == record["region"]
+        assert result["description"] == record["description"]
+        assert result["sale_name"] == record["saleName"]
+        assert result["dept_route"] == record["deptRoute"]
+        assert result["dept_name"] == record["deptName"]
+        assert result["head_image_url"] == record["friendHeadImageUrl"]
+        assert result["last_chat_time"] == record["lastChatTime"]
+        assert result["send_num"] == record["sendNum"]
+        assert result["receive_num"] == record["receiveNum"]
+        assert result["message_count"] == record["messageCount"]
+        assert result["delete_flag"] == record["deleteFlag"]
+        assert result["is_block"] == record["isBlock"]
+        assert result["friend_state"] == record["friendState"]
+
+        # Version carried verbatim; raw_json preserves the original record.
+        assert result["version"] == record["version"]
+        assert result["raw_json"] == record
+
+
+@respx.mock
+async def test_scrape_wechat_friends_sends_pagination_body():
+    """Outgoing request carries pageNum/pageSize and optional filters."""
+    page1 = make_friend_list_response([], total=0, page_num=1)
+    route = respx.post(FRIEND_LIST_URL).mock(
+        return_value=httpx.Response(200, json=page1)
+    )
+
+    client = _primed_client()
+    try:
+        results = [
+            r
+            async for r in scrape_wechat_friends(
+                client, department_id="dept-9", user_id="staff-1", page_size=7
+            )
+        ]
+    finally:
+        await client.aclose()
+
+    # Empty upstream -> no rows, exactly one call (the empty-page stop).
+    assert results == []
+    assert route.call_count == 1
+
+    sent = route.calls.last.request
+    import json as _json
+
+    body = _json.loads(sent.content.decode())
+    assert body["pageNum"] == 1
+    assert body["pageSize"] == 7
+    assert body["departmentId"] == "dept-9"
+    assert body["userId"] == "staff-1"
+
+
+@respx.mock
+async def test_scrape_wechat_friends_stops_on_empty_first_page():
+    """If the very first page is empty, the scraper stops without over-fetching."""
+    empty = make_friend_list_response([], total=0, page_num=1)
+    route = respx.post(FRIEND_LIST_URL).mock(
+        return_value=httpx.Response(200, json=empty)
+    )
+
+    client = _primed_client()
+    try:
+        results = [r async for r in scrape_wechat_friends(client)]
+    finally:
+        await client.aclose()
+
+    assert results == []
+    assert route.call_count == 1