浅念
4a47e824c7
Add the first list scraper for the yunke data-mirror system, establishing the reusable pagination pattern all later list scrapers will build on. - app/scraper/base.py: generic `paginate_by_page` async generator that walks page-numbered endpoints until a page comes back empty (the yunke CRM's primary "no more rows" signal), with a defensive over-fetch ceiling keyed off the declared total (strict `>` so an honest source yields exactly `total`). Plus a tiny `RateLimiter` enforcing a min interval between calls. - app/scraper/wechat.py: `scrape_wechat_friends` wraps `POST /wechatweb/wxFriends/friend_list/es`, maps each record to the `Customer` column shape (snake_case) with `channel="wechat"` and a verbatim `raw_json` copy. Optional `department_id`/`user_id` filters pass through to the request body. - tests/cassettes/wxFriends_fixture.py: two realistic, capture-shaped friend records + a response-envelope builder, reused by the scraper tests. - tests/test_scrape_wechat_friends.py: drives the real pagination + normalization logic through respx (page1 returns both records, page2 empty -> stop). Asserts exact field mappings, `raw_json` fidelity, and exactly 2 upstream calls. - tests/test_scrape_base.py: smoke tests for the RateLimiter spacing. Persistence (upsert into `Customer`) is deliberately deferred to Task 1.7; scrapers remain pure producers of normalized dicts. Co-Authored-By: Claude <noreply@anthropic.com>
123 lines
4.1 KiB
Python
123 lines
4.1 KiB
Python
"""Fixture payloads for ``POST /wechatweb/wxFriends/friend_list/es``.
|
|
|
|
These records mirror the real field names captured from the live CRM (see the
|
|
Task 1.5 spec). Two realistic friend records are provided so a test can drive
|
|
the paginator across two pages: page 1 returns both records, page 2 returns an
|
|
empty ``data`` array to signal "no more" (the documented primary stop signal).
|
|
"""
|
|
# The module name uses camelCase deliberately: it mirrors the upstream endpoint
|
|
# name (``wxFriends``) and the Task 1.5 spec mandates this exact filename.
|
|
# ruff: noqa: N999
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
# Realistic, capture-shaped friend records. Field names are EXACTLY what the
|
|
# live endpoint returns — the scraper maps these to ``Customer`` snake_case
|
|
# columns. Every field that has a target column is populated so the test can
|
|
# assert a full mapping end-to-end.
|
|
FRIEND_RECORDS: list[dict[str, Any]] = [
|
|
{
|
|
"companyCode": "8m0atd",
|
|
"wxId": "wxid_staff_alice01",
|
|
"friendWxId": "wxid_friend_0001",
|
|
"friendWxAlias": "alice_alias",
|
|
"friendWxNickname": "Alice Wang",
|
|
"friendWxRemark": "Alice-VIP",
|
|
"realName": "Wang Ai Li",
|
|
"friendWxPhone": "13800000001",
|
|
"phone": "13800000001",
|
|
"wechatPhone": "13800000011",
|
|
"gender": 2,
|
|
"region": "Shanghai",
|
|
"description": "VIP client, prefers wechat",
|
|
"saleName": "Tom Sale",
|
|
"userId": "staff_user_001",
|
|
"deptRoute": "1.2.3",
|
|
"deptRouteName": "Sales/East/TeamA",
|
|
"deptName": "Sales-East-TeamA",
|
|
"friendHeadImageUrl": "https://cdn.example.com/avatar_alice.png",
|
|
"headImg": "https://cdn.example.com/avatar_alice_legacy.png",
|
|
"lastChatTime": 1700000000123,
|
|
"sendNum": 12,
|
|
"receiveNum": 34,
|
|
"messageCount": 46,
|
|
"deleteFlag": 0,
|
|
"isBlock": 0,
|
|
"friendState": 1,
|
|
"version": "v1.20260731.alice",
|
|
"addTime": 1690000000000,
|
|
"createTime": 1690000000000,
|
|
"fromType": "qrcode",
|
|
"addType": "scan",
|
|
},
|
|
{
|
|
"companyCode": "8m0atd",
|
|
"wxId": "wxid_staff_bob02",
|
|
"friendWxId": "wxid_friend_0002",
|
|
"friendWxAlias": "bob_alias",
|
|
"friendWxNickname": "Bob Chen",
|
|
"friendWxRemark": "Bob-Lead",
|
|
"realName": "Chen Bo",
|
|
"friendWxPhone": "13800000002",
|
|
"phone": "13800000002",
|
|
"wechatPhone": "13800000022",
|
|
"gender": 1,
|
|
"region": "Beijing",
|
|
"description": "New lead from expo",
|
|
"saleName": "Jerry Sale",
|
|
"userId": "staff_user_002",
|
|
"deptRoute": "1.2.4",
|
|
"deptRouteName": "Sales/North/TeamB",
|
|
"deptName": "Sales-North-TeamB",
|
|
"friendHeadImageUrl": "https://cdn.example.com/avatar_bob.png",
|
|
"headImg": "https://cdn.example.com/avatar_bob_legacy.png",
|
|
"lastChatTime": 1700000000456,
|
|
"sendNum": 3,
|
|
"receiveNum": 9,
|
|
"messageCount": 12,
|
|
"deleteFlag": 0,
|
|
"isBlock": 1,
|
|
"friendState": 2,
|
|
"version": "v1.20260731.bob",
|
|
"addTime": 1691000000000,
|
|
"createTime": 1691000000000,
|
|
"fromType": "phone",
|
|
"addType": "import",
|
|
},
|
|
]
|
|
|
|
|
|
def make_friend_list_response(
|
|
records: list[dict[str, Any]],
|
|
*,
|
|
total: int,
|
|
page_num: int = 1,
|
|
page_size: int = 50,
|
|
) -> dict[str, Any]:
|
|
"""Build a capture-shaped ``friend_list/es`` response envelope.
|
|
|
|
The shape matches the live API exactly:
|
|
|
|
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
|
"data":{"displayStatus":"0",
|
|
"basicResult":{"pageNum":..,"pageSize":..,"total":..,
|
|
"data":[ {<friend record>}, ... ]}}}
|
|
"""
|
|
return {
|
|
"message": "调用成功",
|
|
"code": "10000",
|
|
"success": True,
|
|
"traceId": f"trace-{page_num}",
|
|
"data": {
|
|
"displayStatus": "0",
|
|
"basicResult": {
|
|
"pageNum": page_num,
|
|
"pageSize": page_size,
|
|
"total": total,
|
|
"data": list(records),
|
|
},
|
|
},
|
|
}
|