feat: wechat friend scraper
Add the first list scraper for the yunke data-mirror system, establishing the reusable pagination pattern all later list scrapers will build on. - app/scraper/base.py: generic `paginate_by_page` async generator that walks page-numbered endpoints until a page comes back empty (the yunke CRM's primary "no more rows" signal), with a defensive over-fetch ceiling keyed off the declared total (strict `>` so an honest source yields exactly `total`). Plus a tiny `RateLimiter` enforcing a min interval between calls. - app/scraper/wechat.py: `scrape_wechat_friends` wraps `POST /wechatweb/wxFriends/friend_list/es`, maps each record to the `Customer` column shape (snake_case) with `channel="wechat"` and a verbatim `raw_json` copy. Optional `department_id`/`user_id` filters pass through to the request body. - tests/cassettes/wxFriends_fixture.py: two realistic, capture-shaped friend records + a response-envelope builder, reused by the scraper tests. - tests/test_scrape_wechat_friends.py: drives the real pagination + normalization logic through respx (page1 returns both records, page2 empty -> stop). Asserts exact field mappings, `raw_json` fidelity, and exactly 2 upstream calls. - tests/test_scrape_base.py: smoke tests for the RateLimiter spacing. Persistence (upsert into `Customer`) is deliberately deferred to Task 1.7; scrapers remain pure producers of normalized dicts. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,122 @@
|
||||
"""Fixture payloads for ``POST /wechatweb/wxFriends/friend_list/es``.
|
||||
|
||||
These records mirror the real field names captured from the live CRM (see the
|
||||
Task 1.5 spec). Two realistic friend records are provided so a test can drive
|
||||
the paginator across two pages: page 1 returns both records, page 2 returns an
|
||||
empty ``data`` array to signal "no more" (the documented primary stop signal).
|
||||
"""
|
||||
# The module name uses camelCase deliberately: it mirrors the upstream endpoint
|
||||
# name (``wxFriends``) and the Task 1.5 spec mandates this exact filename.
|
||||
# ruff: noqa: N999
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Realistic, capture-shaped friend records. Field names are EXACTLY what the
|
||||
# live endpoint returns — the scraper maps these to ``Customer`` snake_case
|
||||
# columns. Every field that has a target column is populated so the test can
|
||||
# assert a full mapping end-to-end.
|
||||
FRIEND_RECORDS: list[dict[str, Any]] = [
|
||||
{
|
||||
"companyCode": "8m0atd",
|
||||
"wxId": "wxid_staff_alice01",
|
||||
"friendWxId": "wxid_friend_0001",
|
||||
"friendWxAlias": "alice_alias",
|
||||
"friendWxNickname": "Alice Wang",
|
||||
"friendWxRemark": "Alice-VIP",
|
||||
"realName": "Wang Ai Li",
|
||||
"friendWxPhone": "13800000001",
|
||||
"phone": "13800000001",
|
||||
"wechatPhone": "13800000011",
|
||||
"gender": 2,
|
||||
"region": "Shanghai",
|
||||
"description": "VIP client, prefers wechat",
|
||||
"saleName": "Tom Sale",
|
||||
"userId": "staff_user_001",
|
||||
"deptRoute": "1.2.3",
|
||||
"deptRouteName": "Sales/East/TeamA",
|
||||
"deptName": "Sales-East-TeamA",
|
||||
"friendHeadImageUrl": "https://cdn.example.com/avatar_alice.png",
|
||||
"headImg": "https://cdn.example.com/avatar_alice_legacy.png",
|
||||
"lastChatTime": 1700000000123,
|
||||
"sendNum": 12,
|
||||
"receiveNum": 34,
|
||||
"messageCount": 46,
|
||||
"deleteFlag": 0,
|
||||
"isBlock": 0,
|
||||
"friendState": 1,
|
||||
"version": "v1.20260731.alice",
|
||||
"addTime": 1690000000000,
|
||||
"createTime": 1690000000000,
|
||||
"fromType": "qrcode",
|
||||
"addType": "scan",
|
||||
},
|
||||
{
|
||||
"companyCode": "8m0atd",
|
||||
"wxId": "wxid_staff_bob02",
|
||||
"friendWxId": "wxid_friend_0002",
|
||||
"friendWxAlias": "bob_alias",
|
||||
"friendWxNickname": "Bob Chen",
|
||||
"friendWxRemark": "Bob-Lead",
|
||||
"realName": "Chen Bo",
|
||||
"friendWxPhone": "13800000002",
|
||||
"phone": "13800000002",
|
||||
"wechatPhone": "13800000022",
|
||||
"gender": 1,
|
||||
"region": "Beijing",
|
||||
"description": "New lead from expo",
|
||||
"saleName": "Jerry Sale",
|
||||
"userId": "staff_user_002",
|
||||
"deptRoute": "1.2.4",
|
||||
"deptRouteName": "Sales/North/TeamB",
|
||||
"deptName": "Sales-North-TeamB",
|
||||
"friendHeadImageUrl": "https://cdn.example.com/avatar_bob.png",
|
||||
"headImg": "https://cdn.example.com/avatar_bob_legacy.png",
|
||||
"lastChatTime": 1700000000456,
|
||||
"sendNum": 3,
|
||||
"receiveNum": 9,
|
||||
"messageCount": 12,
|
||||
"deleteFlag": 0,
|
||||
"isBlock": 1,
|
||||
"friendState": 2,
|
||||
"version": "v1.20260731.bob",
|
||||
"addTime": 1691000000000,
|
||||
"createTime": 1691000000000,
|
||||
"fromType": "phone",
|
||||
"addType": "import",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def make_friend_list_response(
|
||||
records: list[dict[str, Any]],
|
||||
*,
|
||||
total: int,
|
||||
page_num: int = 1,
|
||||
page_size: int = 50,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a capture-shaped ``friend_list/es`` response envelope.
|
||||
|
||||
The shape matches the live API exactly:
|
||||
|
||||
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
||||
"data":{"displayStatus":"0",
|
||||
"basicResult":{"pageNum":..,"pageSize":..,"total":..,
|
||||
"data":[ {<friend record>}, ... ]}}}
|
||||
"""
|
||||
return {
|
||||
"message": "调用成功",
|
||||
"code": "10000",
|
||||
"success": True,
|
||||
"traceId": f"trace-{page_num}",
|
||||
"data": {
|
||||
"displayStatus": "0",
|
||||
"basicResult": {
|
||||
"pageNum": page_num,
|
||||
"pageSize": page_size,
|
||||
"total": total,
|
||||
"data": list(records),
|
||||
},
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user