xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: wechat message scraper

Browse Source 
Adds Task 1.6: a backward-cursor paginator in base.py, a message-type
normalizer in parsers.py, and the WeChat chat-message scraper in wechat.py.

- app/scraper/parsers.py: normalize_wechat_type() maps 1 -> "text",
  -2 -> "divider", everything else -> "unknown". Total (never raises);
  unknown codes pass through so no real message is silently dropped.
- app/scraper/base.py: paginate_by_cursor() generic async generator. Takes
  a fetch(cursor) -> (rows, next_cursor, has_more) coroutine; advances the
  cursor and stops on has_more=False OR an empty page (defensive). Same
  RateLimiter integration as paginate_by_page.
- app/scraper/wechat.py: scrape_wechat_messages() wraps
  POST /wechatweb/msg/chats. Body is form-encoded (NOT JSON) — content-type
  is overridden per request to application/x-www-form-urlencoded via the
  YunkeClient.post(headers=...) override hook. Pages backward by setting
  end = data.end - 1 each batch, repeats until hasLast=false. Drops any
  record whose type/messageType is -2 (date divider UI chrome). Yields
  normalized dicts (snake_case Message columns + channel="wechat" +
  raw_json); the raw type code is preserved on raw_json for the
  persistence layer (Task 1.7).

Tests: 13 new (10 parser parametrized + 3 scraper). respx-mocked
msg/chats with two backward pages — asserts [m1, m2, m3] yielded (the -2
divider dropped), full field mapping vs the live-capture names, the
content-type is form-encoded (not JSON), and the 2nd request body's
end == E1 - 1 (proving the backward cursor advance).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 11:39:55 +08:00
parent 4a47e824c7
commit 903286f379
6 changed files with 744 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/tests/cassettes/msg_chats_fixture.py
+147
View File
@@ -0,0 +1,147 @@
"""Fixture payloads for ``POST /wechatweb/msg/chats`` (Task 1.6).
These records mirror the real field names captured from the live CRM (see the
Task 1.6 spec). The cassette is shaped to drive the backward-cursor paginator
across two pages:
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
The ``-2`` record is a date divider that the scraper must DROP. ``E1`` is the
oldest timestamp in page 1's batch, so the backward-cursor advance must set
the next request's ``end`` to ``E1 - 1``. Page 2 reports ``hasLast=false`` so
the scraper stops without a third fetch.
"""
from __future__ import annotations
from typing import Any
# Oldest-timestamp-in-batch returned by each page; the scraper uses these to
# advance the backward cursor (next_request_end = page_end - 1).
E1 = 1784517338999
E2 = 1784517000000
# Page 1 records, EXACT live-capture field names. m1 is text/direct/incoming,
# m_datebar is a -2 divider (must be filtered), m2 is image/chatroom/outgoing.
MESSAGE_RECORDS_PAGE1: list[dict[str, Any]] = [
{ # m1 — text message, incoming, direct (empty roomid).
"msgSvrId": "1000001",
"type": 1,
"messageType": 1,
"text": "hi alice",
"wechatId": "wxid_staff_alice",
"talker": "wxid_friend_0001",
"oriTalker": "wxid_friend_0001",
"roomid": "",
"mine": 0,
"isSend": 0,
"timestamp": E1 + 600000,
"createTime": E1 + 600000,
"name": "Alice Wang",
"headUrl": "https://cdn.example.com/alice.png",
"isDel": 0,
"version": "v1.0",
},
{ # date divider — type == -2; scraper MUST drop this, never yield it.
"msgSvrId": "1000000",
"type": -2,
"messageType": -2,
"text": "2026-01-01",
"wechatId": "wxid_staff_alice",
"talker": "wxid_friend_0001",
"oriTalker": "wxid_friend_0001",
"roomid": "",
"mine": 0,
"isSend": 0,
"timestamp": E1 + 300000,
"createTime": E1 + 300000,
"name": "",
"headUrl": "",
"isDel": 0,
"version": "v1.0",
},
{ # m2 — image message, outgoing, in a chatroom.
"msgSvrId": "1000002",
"type": 3,
"messageType": 3,
"text": "[image]",
"file": "https://cdn.example.com/img2.jpg",
"fileTh": "https://cdn.example.com/img2_th.jpg",
"fileHd": "https://cdn.example.com/img2_hd.jpg",
"md5": "md5img2",
"md5Th": "md5img2th",
"md5Hd": "md5img2hd",
"imgPath": "/tmp/img2.jpg",
"originalImage": "/tmp/img2_orig.jpg",
"picType": 1,
"wechatId": "wxid_staff_alice",
"talker": "wxid_friend_0001",
"oriTalker": "room12345@chatroom",
"roomid": "room12345@chatroom",
"mine": 1,
"isSend": 1,
"timestamp": E1 + 100,
"createTime": E1 + 100,
"name": "Bob Chen",
"headUrl": "https://cdn.example.com/bob.png",
"isDel": 0,
"version": "v1.0",
},
]
# Page 2 record — voice message, oldest in the history window.
MESSAGE_RECORDS_PAGE2: list[dict[str, Any]] = [
{
"msgSvrId": "999999",
"type": 34,
"messageType": 34,
"text": "[voice]",
"file": "https://cdn.example.com/voice3.amr",
"md5": "md5voice3",
"duration": 5,
"transcriptionText": "hello from yesterday",
"wechatId": "wxid_staff_alice",
"talker": "wxid_friend_0001",
"oriTalker": "wxid_friend_0001",
"roomid": "",
"mine": 1,
"isSend": 1,
"timestamp": E2,
"createTime": E2,
"name": "Alice Wang",
"headUrl": "https://cdn.example.com/alice.png",
"isDel": 0,
"version": "v1.0",
"paymsgid": "pay-abc-123",
"referMsg": {"text": "earlier msg", "msgSvrId": "888888"},
},
]
def make_msg_chats_response(
messages: list[dict[str, Any]],
*,
end: int,
has_last: bool,
trace_id: str = "trace-msg-1",
) -> dict[str, Any]:
"""Build a capture-shaped ``msg/chats`` response envelope.
Shape (EXACT, per Task 1.6 spec)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"messages":[...], "end":<oldest ts in batch>,
"hasLast":<bool — are there older messages?>}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id,
"data": {
"messages": list(messages),
"end": end,
"hasLast": has_last,
},
}
yunke-server/tests/test_parsers.py
+44
View File
@@ -0,0 +1,44 @@
"""Tests for the message-type normalization parser (Task 1.6).
The parser must be total — it never raises on weird input — because every
incoming WeChat record runs through it during a scrape, and one malformed
type code must not abort the whole pull. Unknown codes pass through as
``"unknown"`` so the message is still ingested (and recoverable via
``raw_json``); only the ``-2`` date-divider code maps to ``"divider"``,
which the scraper uses as its drop signal.
"""
from __future__ import annotations
import pytest
from app.scraper.parsers import normalize_wechat_type
@pytest.mark.parametrize(
"raw,expected",
[
# Documented codes.
(1, "text"),
(-2, "divider"),
# Phase-0 unknown codes — pass through as "unknown", never dropped.
(3, "unknown"), # image
(34, "unknown"), # voice
(49, "unknown"), # file / link / transfer
(10000, "unknown"),
# Stringified ints (the upstream API may send either form).
("1", "text"),
("-2", "divider"),
("34", "unknown"),
],
)
def test_normalize_wechat_type_maps_known_and_unknown(raw, expected):
assert normalize_wechat_type(raw) == expected
def test_normalize_wechat_type_never_raises():
"""Whatever we throw at it, it returns a string — never raises."""
for raw in (None, "", "junk", "1.5", [], {}, object(), float("nan")):
out = normalize_wechat_type(raw)
assert isinstance(out, str)
assert out == "unknown"
yunke-server/tests/test_scrape_wechat_messages.py
+217
View File
@@ -0,0 +1,217 @@
"""Tests for the WeChat message scraper (Task 1.6).
Drives the real backward-cursor pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport. Two pages are served:
* page 1 -> ``messages=[m1, m_datebar(-2), m2]``, ``end=E1``, ``hasLast=true``
* page 2 -> ``messages=[m3]``, ``end=E2``, ``hasLast=false``
The scraper must yield exactly ``[m1, m2, m3]`` (the ``-2`` divider dropped),
map fields per the live-capture names, AND the 2nd request body's ``end`` must
equal ``E1 - 1`` (proving the backward cursor advances by one ms).
"""
from __future__ import annotations
import urllib.parse
import httpx
import respx
from app.scraper.wechat import scrape_wechat_messages
from app.session.yunke_client import YunkeClient
from tests.cassettes.msg_chats_fixture import (
E1,
E2,
MESSAGE_RECORDS_PAGE1,
MESSAGE_RECORDS_PAGE2,
make_msg_chats_response,
)
BASE = "https://crm2.yunkecn.com"
MSG_CHATS_URL = f"{BASE}/wechatweb/msg/chats"
USERID = "a" * 32
COMPANY = "8m0atd"
WECHAT_ID = "wxid_staff_alice"
FRIEND_ID = "wxid_friend_0001"
START = 1
END = 1784518000000 # the initial upper-bound cursor
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
def _form(sent: httpx.Request) -> dict[str, str]:
"""Parse the form-encoded body of an outgoing request into a flat dict.
``keep_blank_values=True`` so an empty form field (e.g. ``types=``) is
preserved — the real API contract treats the empty ``types`` field as
"no filter", so its presence in the body is part of what we assert.
"""
parsed = urllib.parse.parse_qs(sent.content.decode(), keep_blank_values=True)
return {k: v[0] for k, v in parsed.items()}
# ---------------------------------------------------------------------------
# Test A: end-to-end backward-cursor pagination + divider drop + field mapping
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_messages_drops_divider_and_paginates_backward():
page1 = make_msg_chats_response(
MESSAGE_RECORDS_PAGE1, end=E1, has_last=True, trace_id="trace-1"
)
page2 = make_msg_chats_response(
MESSAGE_RECORDS_PAGE2, end=E2, has_last=False, trace_id="trace-2"
)
route = respx.post(MSG_CHATS_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_wechat_messages(
client, WECHAT_ID, FRIEND_ID, START, END
)
]
finally:
await client.aclose()
# --- Exactly 2 calls: hasLast=false on page 2 means no 3rd fetch. ---
assert route.call_count == 2
# --- Exactly [m1, m2, m3] yielded (the -2 divider dropped). ---
assert [r["msg_id"] for r in results] == ["1000001", "1000002", "999999"]
assert len(results) == 3
r1, r2, r3 = results
# m1 — text / direct / incoming.
assert r1["msg_id"] == "1000001"
assert r1["channel"] == "wechat"
assert r1["owner_account_id"] == WECHAT_ID
assert r1["talker_id"] == "wxid_friend_0001"
assert r1["ori_talker"] == "wxid_friend_0001"
assert r1["room_id"] == "" # raw value carried through
assert r1["is_send"] == 0 # pulled from `mine`
assert r1["type"] == "text"
assert r1["text"] == "hi alice"
assert r1["timestamp"] == E1 + 600000
assert r1["sender_name"] == "Alice Wang"
assert r1["sender_head_url"] == "https://cdn.example.com/alice.png"
assert r1["raw_json"] == MESSAGE_RECORDS_PAGE1[0]
# m2 — image / chatroom / outgoing.
assert r2["msg_id"] == "1000002"
assert r2["room_id"] == "room12345@chatroom"
assert r2["ori_talker"] == "room12345@chatroom"
assert r2["is_send"] == 1
assert r2["type"] == "unknown" # unmapped code passes through
assert r2["file_url"] == "https://cdn.example.com/img2.jpg"
assert r2["file_th_url"] == "https://cdn.example.com/img2_th.jpg"
assert r2["file_hd_url"] == "https://cdn.example.com/img2_hd.jpg"
assert r2["md5"] == "md5img2"
assert r2["md5_th"] == "md5img2th"
assert r2["md5_hd"] == "md5img2hd"
assert r2["img_path"] == "/tmp/img2.jpg"
assert r2["original_image"] == "/tmp/img2_orig.jpg"
assert r2["pic_type"] == 1
assert r2["raw_json"] == MESSAGE_RECORDS_PAGE1[2]
# m3 — voice / transcription.
assert r3["msg_id"] == "999999"
assert r3["type"] == "unknown"
assert r3["duration"] == 5
assert r3["transcription_text"] == "hello from yesterday"
assert r3["pay_msg_id"] == "pay-abc-123"
assert r3["refer_msg"] == {"text": "earlier msg", "msgSvrId": "888888"}
assert r3["raw_json"] == MESSAGE_RECORDS_PAGE2[0]
# --- Backward cursor advance: 2nd request body's end == E1 - 1. ---
first_req = route.calls[0].request
second_req = route.calls[1].request
assert _form(first_req)["end"] == str(END)
assert _form(second_req)["end"] == str(E1 - 1)
# --- Form-encoding is actually sent (NOT JSON). Content-type overridden. ---
assert (
first_req.headers["content-type"]
== "application/x-www-form-urlencoded"
)
form_first = _form(first_req)
assert form_first["wechatId"] == WECHAT_ID
assert form_first["friendId"] == FRIEND_ID
assert form_first["types"] == ""
assert form_first["start"] == str(START)
assert form_first["end"] == str(END)
# ---------------------------------------------------------------------------
# Test B: hasLast=false on first page -> stop after one call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_messages_stops_when_has_last_false_on_first_page():
"""A single page with hasLast=false: scraper yields the records and stops."""
page = make_msg_chats_response(
[MESSAGE_RECORDS_PAGE2[0]], end=E2, has_last=False
)
route = respx.post(MSG_CHATS_URL).mock(
return_value=httpx.Response(200, json=page)
)
client = _primed_client()
try:
results = [
r
async for r in scrape_wechat_messages(
client, WECHAT_ID, FRIEND_ID, START, END
)
]
finally:
await client.aclose()
assert [r["msg_id"] for r in results] == ["999999"]
assert route.call_count == 1
# ---------------------------------------------------------------------------
# Test C: empty messages array on the first page -> stop without a 3rd fetch
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_wechat_messages_empty_first_page_yields_nothing():
"""An empty messages array is a defensive stop signal — no rows, one call."""
page = make_msg_chats_response([], end=E1, has_last=True)
route = respx.post(MSG_CHATS_URL).mock(
return_value=httpx.Response(200, json=page)
)
client = _primed_client()
try:
results = [
r
async for r in scrape_wechat_messages(
client, WECHAT_ID, FRIEND_ID, START, END
)
]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1