浅念
903286f379
Adds Task 1.6: a backward-cursor paginator in base.py, a message-type normalizer in parsers.py, and the WeChat chat-message scraper in wechat.py. - app/scraper/parsers.py: normalize_wechat_type() maps 1 -> "text", -2 -> "divider", everything else -> "unknown". Total (never raises); unknown codes pass through so no real message is silently dropped. - app/scraper/base.py: paginate_by_cursor() generic async generator. Takes a fetch(cursor) -> (rows, next_cursor, has_more) coroutine; advances the cursor and stops on has_more=False OR an empty page (defensive). Same RateLimiter integration as paginate_by_page. - app/scraper/wechat.py: scrape_wechat_messages() wraps POST /wechatweb/msg/chats. Body is form-encoded (NOT JSON) — content-type is overridden per request to application/x-www-form-urlencoded via the YunkeClient.post(headers=...) override hook. Pages backward by setting end = data.end - 1 each batch, repeats until hasLast=false. Drops any record whose type/messageType is -2 (date divider UI chrome). Yields normalized dicts (snake_case Message columns + channel="wechat" + raw_json); the raw type code is preserved on raw_json for the persistence layer (Task 1.7). Tests: 13 new (10 parser parametrized + 3 scraper). respx-mocked msg/chats with two backward pages — asserts [m1, m2, m3] yielded (the -2 divider dropped), full field mapping vs the live-capture names, the content-type is form-encoded (not JSON), and the 2nd request body's end == E1 - 1 (proving the backward cursor advance). Co-Authored-By: Claude <noreply@anthropic.com>
148 lines
4.6 KiB
Python
148 lines
4.6 KiB
Python
"""Fixture payloads for ``POST /wechatweb/msg/chats`` (Task 1.6).
|
|
|
|
These records mirror the real field names captured from the live CRM (see the
|
|
Task 1.6 spec). The cassette is shaped to drive the backward-cursor paginator
|
|
across two pages:
|
|
|
|
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
|
|
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
|
|
|
|
The ``-2`` record is a date divider that the scraper must DROP. ``E1`` is the
|
|
oldest timestamp in page 1's batch, so the backward-cursor advance must set
|
|
the next request's ``end`` to ``E1 - 1``. Page 2 reports ``hasLast=false`` so
|
|
the scraper stops without a third fetch.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
# Oldest-timestamp-in-batch returned by each page; the scraper uses these to
|
|
# advance the backward cursor (next_request_end = page_end - 1).
|
|
E1 = 1784517338999
|
|
E2 = 1784517000000
|
|
|
|
# Page 1 records, EXACT live-capture field names. m1 is text/direct/incoming,
|
|
# m_datebar is a -2 divider (must be filtered), m2 is image/chatroom/outgoing.
|
|
MESSAGE_RECORDS_PAGE1: list[dict[str, Any]] = [
|
|
{ # m1 — text message, incoming, direct (empty roomid).
|
|
"msgSvrId": "1000001",
|
|
"type": 1,
|
|
"messageType": 1,
|
|
"text": "hi alice",
|
|
"wechatId": "wxid_staff_alice",
|
|
"talker": "wxid_friend_0001",
|
|
"oriTalker": "wxid_friend_0001",
|
|
"roomid": "",
|
|
"mine": 0,
|
|
"isSend": 0,
|
|
"timestamp": E1 + 600000,
|
|
"createTime": E1 + 600000,
|
|
"name": "Alice Wang",
|
|
"headUrl": "https://cdn.example.com/alice.png",
|
|
"isDel": 0,
|
|
"version": "v1.0",
|
|
},
|
|
{ # date divider — type == -2; scraper MUST drop this, never yield it.
|
|
"msgSvrId": "1000000",
|
|
"type": -2,
|
|
"messageType": -2,
|
|
"text": "2026-01-01",
|
|
"wechatId": "wxid_staff_alice",
|
|
"talker": "wxid_friend_0001",
|
|
"oriTalker": "wxid_friend_0001",
|
|
"roomid": "",
|
|
"mine": 0,
|
|
"isSend": 0,
|
|
"timestamp": E1 + 300000,
|
|
"createTime": E1 + 300000,
|
|
"name": "",
|
|
"headUrl": "",
|
|
"isDel": 0,
|
|
"version": "v1.0",
|
|
},
|
|
{ # m2 — image message, outgoing, in a chatroom.
|
|
"msgSvrId": "1000002",
|
|
"type": 3,
|
|
"messageType": 3,
|
|
"text": "[image]",
|
|
"file": "https://cdn.example.com/img2.jpg",
|
|
"fileTh": "https://cdn.example.com/img2_th.jpg",
|
|
"fileHd": "https://cdn.example.com/img2_hd.jpg",
|
|
"md5": "md5img2",
|
|
"md5Th": "md5img2th",
|
|
"md5Hd": "md5img2hd",
|
|
"imgPath": "/tmp/img2.jpg",
|
|
"originalImage": "/tmp/img2_orig.jpg",
|
|
"picType": 1,
|
|
"wechatId": "wxid_staff_alice",
|
|
"talker": "wxid_friend_0001",
|
|
"oriTalker": "room12345@chatroom",
|
|
"roomid": "room12345@chatroom",
|
|
"mine": 1,
|
|
"isSend": 1,
|
|
"timestamp": E1 + 100,
|
|
"createTime": E1 + 100,
|
|
"name": "Bob Chen",
|
|
"headUrl": "https://cdn.example.com/bob.png",
|
|
"isDel": 0,
|
|
"version": "v1.0",
|
|
},
|
|
]
|
|
|
|
# Page 2 record — voice message, oldest in the history window.
|
|
MESSAGE_RECORDS_PAGE2: list[dict[str, Any]] = [
|
|
{
|
|
"msgSvrId": "999999",
|
|
"type": 34,
|
|
"messageType": 34,
|
|
"text": "[voice]",
|
|
"file": "https://cdn.example.com/voice3.amr",
|
|
"md5": "md5voice3",
|
|
"duration": 5,
|
|
"transcriptionText": "hello from yesterday",
|
|
"wechatId": "wxid_staff_alice",
|
|
"talker": "wxid_friend_0001",
|
|
"oriTalker": "wxid_friend_0001",
|
|
"roomid": "",
|
|
"mine": 1,
|
|
"isSend": 1,
|
|
"timestamp": E2,
|
|
"createTime": E2,
|
|
"name": "Alice Wang",
|
|
"headUrl": "https://cdn.example.com/alice.png",
|
|
"isDel": 0,
|
|
"version": "v1.0",
|
|
"paymsgid": "pay-abc-123",
|
|
"referMsg": {"text": "earlier msg", "msgSvrId": "888888"},
|
|
},
|
|
]
|
|
|
|
|
|
def make_msg_chats_response(
|
|
messages: list[dict[str, Any]],
|
|
*,
|
|
end: int,
|
|
has_last: bool,
|
|
trace_id: str = "trace-msg-1",
|
|
) -> dict[str, Any]:
|
|
"""Build a capture-shaped ``msg/chats`` response envelope.
|
|
|
|
Shape (EXACT, per Task 1.6 spec)::
|
|
|
|
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
|
"data":{"messages":[...], "end":<oldest ts in batch>,
|
|
"hasLast":<bool — are there older messages?>}}
|
|
"""
|
|
return {
|
|
"message": "调用成功",
|
|
"code": "10000",
|
|
"success": True,
|
|
"traceId": trace_id,
|
|
"data": {
|
|
"messages": list(messages),
|
|
"end": end,
|
|
"hasLast": has_last,
|
|
},
|
|
}
|