feat: wechat message scraper
Adds Task 1.6: a backward-cursor paginator in base.py, a message-type normalizer in parsers.py, and the WeChat chat-message scraper in wechat.py. - app/scraper/parsers.py: normalize_wechat_type() maps 1 -> "text", -2 -> "divider", everything else -> "unknown". Total (never raises); unknown codes pass through so no real message is silently dropped. - app/scraper/base.py: paginate_by_cursor() generic async generator. Takes a fetch(cursor) -> (rows, next_cursor, has_more) coroutine; advances the cursor and stops on has_more=False OR an empty page (defensive). Same RateLimiter integration as paginate_by_page. - app/scraper/wechat.py: scrape_wechat_messages() wraps POST /wechatweb/msg/chats. Body is form-encoded (NOT JSON) — content-type is overridden per request to application/x-www-form-urlencoded via the YunkeClient.post(headers=...) override hook. Pages backward by setting end = data.end - 1 each batch, repeats until hasLast=false. Drops any record whose type/messageType is -2 (date divider UI chrome). Yields normalized dicts (snake_case Message columns + channel="wechat" + raw_json); the raw type code is preserved on raw_json for the persistence layer (Task 1.7). Tests: 13 new (10 parser parametrized + 3 scraper). respx-mocked msg/chats with two backward pages — asserts [m1, m2, m3] yielded (the -2 divider dropped), full field mapping vs the live-capture names, the content-type is form-encoded (not JSON), and the 2nd request body's end == E1 - 1 (proving the backward cursor advance). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
"""Fixture payloads for ``POST /wechatweb/msg/chats`` (Task 1.6).
|
||||
|
||||
These records mirror the real field names captured from the live CRM (see the
|
||||
Task 1.6 spec). The cassette is shaped to drive the backward-cursor paginator
|
||||
across two pages:
|
||||
|
||||
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
|
||||
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
|
||||
|
||||
The ``-2`` record is a date divider that the scraper must DROP. ``E1`` is the
|
||||
oldest timestamp in page 1's batch, so the backward-cursor advance must set
|
||||
the next request's ``end`` to ``E1 - 1``. Page 2 reports ``hasLast=false`` so
|
||||
the scraper stops without a third fetch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Oldest-timestamp-in-batch returned by each page; the scraper uses these to
|
||||
# advance the backward cursor (next_request_end = page_end - 1).
|
||||
E1 = 1784517338999
|
||||
E2 = 1784517000000
|
||||
|
||||
# Page 1 records, EXACT live-capture field names. m1 is text/direct/incoming,
|
||||
# m_datebar is a -2 divider (must be filtered), m2 is image/chatroom/outgoing.
|
||||
MESSAGE_RECORDS_PAGE1: list[dict[str, Any]] = [
|
||||
{ # m1 — text message, incoming, direct (empty roomid).
|
||||
"msgSvrId": "1000001",
|
||||
"type": 1,
|
||||
"messageType": 1,
|
||||
"text": "hi alice",
|
||||
"wechatId": "wxid_staff_alice",
|
||||
"talker": "wxid_friend_0001",
|
||||
"oriTalker": "wxid_friend_0001",
|
||||
"roomid": "",
|
||||
"mine": 0,
|
||||
"isSend": 0,
|
||||
"timestamp": E1 + 600000,
|
||||
"createTime": E1 + 600000,
|
||||
"name": "Alice Wang",
|
||||
"headUrl": "https://cdn.example.com/alice.png",
|
||||
"isDel": 0,
|
||||
"version": "v1.0",
|
||||
},
|
||||
{ # date divider — type == -2; scraper MUST drop this, never yield it.
|
||||
"msgSvrId": "1000000",
|
||||
"type": -2,
|
||||
"messageType": -2,
|
||||
"text": "2026-01-01",
|
||||
"wechatId": "wxid_staff_alice",
|
||||
"talker": "wxid_friend_0001",
|
||||
"oriTalker": "wxid_friend_0001",
|
||||
"roomid": "",
|
||||
"mine": 0,
|
||||
"isSend": 0,
|
||||
"timestamp": E1 + 300000,
|
||||
"createTime": E1 + 300000,
|
||||
"name": "",
|
||||
"headUrl": "",
|
||||
"isDel": 0,
|
||||
"version": "v1.0",
|
||||
},
|
||||
{ # m2 — image message, outgoing, in a chatroom.
|
||||
"msgSvrId": "1000002",
|
||||
"type": 3,
|
||||
"messageType": 3,
|
||||
"text": "[image]",
|
||||
"file": "https://cdn.example.com/img2.jpg",
|
||||
"fileTh": "https://cdn.example.com/img2_th.jpg",
|
||||
"fileHd": "https://cdn.example.com/img2_hd.jpg",
|
||||
"md5": "md5img2",
|
||||
"md5Th": "md5img2th",
|
||||
"md5Hd": "md5img2hd",
|
||||
"imgPath": "/tmp/img2.jpg",
|
||||
"originalImage": "/tmp/img2_orig.jpg",
|
||||
"picType": 1,
|
||||
"wechatId": "wxid_staff_alice",
|
||||
"talker": "wxid_friend_0001",
|
||||
"oriTalker": "room12345@chatroom",
|
||||
"roomid": "room12345@chatroom",
|
||||
"mine": 1,
|
||||
"isSend": 1,
|
||||
"timestamp": E1 + 100,
|
||||
"createTime": E1 + 100,
|
||||
"name": "Bob Chen",
|
||||
"headUrl": "https://cdn.example.com/bob.png",
|
||||
"isDel": 0,
|
||||
"version": "v1.0",
|
||||
},
|
||||
]
|
||||
|
||||
# Page 2 record — voice message, oldest in the history window.
|
||||
MESSAGE_RECORDS_PAGE2: list[dict[str, Any]] = [
|
||||
{
|
||||
"msgSvrId": "999999",
|
||||
"type": 34,
|
||||
"messageType": 34,
|
||||
"text": "[voice]",
|
||||
"file": "https://cdn.example.com/voice3.amr",
|
||||
"md5": "md5voice3",
|
||||
"duration": 5,
|
||||
"transcriptionText": "hello from yesterday",
|
||||
"wechatId": "wxid_staff_alice",
|
||||
"talker": "wxid_friend_0001",
|
||||
"oriTalker": "wxid_friend_0001",
|
||||
"roomid": "",
|
||||
"mine": 1,
|
||||
"isSend": 1,
|
||||
"timestamp": E2,
|
||||
"createTime": E2,
|
||||
"name": "Alice Wang",
|
||||
"headUrl": "https://cdn.example.com/alice.png",
|
||||
"isDel": 0,
|
||||
"version": "v1.0",
|
||||
"paymsgid": "pay-abc-123",
|
||||
"referMsg": {"text": "earlier msg", "msgSvrId": "888888"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def make_msg_chats_response(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
end: int,
|
||||
has_last: bool,
|
||||
trace_id: str = "trace-msg-1",
|
||||
) -> dict[str, Any]:
|
||||
"""Build a capture-shaped ``msg/chats`` response envelope.
|
||||
|
||||
Shape (EXACT, per Task 1.6 spec)::
|
||||
|
||||
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
||||
"data":{"messages":[...], "end":<oldest ts in batch>,
|
||||
"hasLast":<bool — are there older messages?>}}
|
||||
"""
|
||||
return {
|
||||
"message": "调用成功",
|
||||
"code": "10000",
|
||||
"success": True,
|
||||
"traceId": trace_id,
|
||||
"data": {
|
||||
"messages": list(messages),
|
||||
"end": end,
|
||||
"hasLast": has_last,
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user