feat: wechat message scraper
Adds Task 1.6: a backward-cursor paginator in base.py, a message-type normalizer in parsers.py, and the WeChat chat-message scraper in wechat.py. - app/scraper/parsers.py: normalize_wechat_type() maps 1 -> "text", -2 -> "divider", everything else -> "unknown". Total (never raises); unknown codes pass through so no real message is silently dropped. - app/scraper/base.py: paginate_by_cursor() generic async generator. Takes a fetch(cursor) -> (rows, next_cursor, has_more) coroutine; advances the cursor and stops on has_more=False OR an empty page (defensive). Same RateLimiter integration as paginate_by_page. - app/scraper/wechat.py: scrape_wechat_messages() wraps POST /wechatweb/msg/chats. Body is form-encoded (NOT JSON) — content-type is overridden per request to application/x-www-form-urlencoded via the YunkeClient.post(headers=...) override hook. Pages backward by setting end = data.end - 1 each batch, repeats until hasLast=false. Drops any record whose type/messageType is -2 (date divider UI chrome). Yields normalized dicts (snake_case Message columns + channel="wechat" + raw_json); the raw type code is preserved on raw_json for the persistence layer (Task 1.7). Tests: 13 new (10 parser parametrized + 3 scraper). respx-mocked msg/chats with two backward pages — asserts [m1, m2, m3] yielded (the -2 divider dropped), full field mapping vs the live-capture names, the content-type is form-encoded (not JSON), and the 2nd request body's end == E1 - 1 (proving the backward cursor advance). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -1,16 +1,24 @@
|
||||
"""WeChat friend-list scraper.
|
||||
"""WeChat scrapers.
|
||||
|
||||
Wraps ``POST /wechatweb/wxFriends/friend_list/es`` and yields normalized dicts
|
||||
ready to be upserted into :class:`app.db.models.Customer` (persistence is a
|
||||
LATER task — Task 1.7). Each dict carries:
|
||||
Two endpoints today:
|
||||
|
||||
* snake_case keys matching the ``Customer`` columns (see :func:`_normalize`),
|
||||
* :func:`scrape_wechat_friends` — wraps ``POST /wechatweb/wxFriends/friend_list/es``
|
||||
and yields normalized dicts ready to be upserted into
|
||||
:class:`app.db.models.Customer`.
|
||||
* :func:`scrape_wechat_messages` — wraps ``POST /wechatweb/msg/chats`` and
|
||||
yields normalized chat-message dicts (see Task 1.6).
|
||||
|
||||
Both yield dicts (not ORM). Persistence is a LATER task — Task 1.7. Each dict
|
||||
carries:
|
||||
|
||||
* snake_case keys matching the target ORM columns (see :func:`_normalize`
|
||||
for friends and :func:`_normalize_message` for messages),
|
||||
* ``channel == "wechat"``,
|
||||
* ``raw_json`` — the original upstream record, verbatim, so nothing captured
|
||||
is lost even when a column has no target field.
|
||||
|
||||
Field names are mapped from the real capture (see Task 1.5 spec). The mapping
|
||||
is a straight ``camelCase -> snake_case`` translation with a couple of
|
||||
Friend field names are mapped from the real capture (see Task 1.5 spec). The
|
||||
mapping is a straight ``camelCase -> snake_case`` translation with a couple of
|
||||
disambiguations where the source has two fields for one column:
|
||||
|
||||
* ``phone`` <- ``phone``
|
||||
@@ -24,12 +32,18 @@ from __future__ import annotations
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from app.scraper.base import paginate_by_page
|
||||
from app.scraper.base import paginate_by_cursor, paginate_by_page
|
||||
from app.scraper.parsers import normalize_wechat_type
|
||||
from app.session.yunke_client import YunkeClient
|
||||
|
||||
__all__ = ["scrape_wechat_friends"]
|
||||
__all__ = ["scrape_wechat_friends", "scrape_wechat_messages"]
|
||||
|
||||
FRIEND_LIST_PATH = "/wechatweb/wxFriends/friend_list/es"
|
||||
MSG_CHATS_PATH = "/wechatweb/msg/chats"
|
||||
|
||||
# Upstream sends form-encoded bodies for ``msg/chats`` (NOT JSON), so we have
|
||||
# to override the client's default JSON content-type on every request.
|
||||
_FORM_CONTENT_TYPE = "application/x-www-form-urlencoded"
|
||||
|
||||
|
||||
def _parse_total(resp: Any) -> int | None:
|
||||
@@ -169,3 +183,191 @@ async def scrape_wechat_friends(
|
||||
user_id=user_id,
|
||||
):
|
||||
yield _normalize(record)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Message scraper (Task 1.6) — POST /wechatweb/msg/chats
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
async def _fetch_msg_page(
|
||||
client: YunkeClient,
|
||||
*,
|
||||
wechat_id: str,
|
||||
friend_id: str,
|
||||
types: str,
|
||||
start: int,
|
||||
end_cursor: int,
|
||||
) -> tuple[list[dict[str, Any]], int, bool]:
|
||||
"""Fetch one backward page of ``msg/chats``.
|
||||
|
||||
Returns the ``(rows, next_cursor, has_more)`` triple expected by
|
||||
:func:`paginate_by_cursor`. ``next_cursor`` is ``data.end - 1`` so the
|
||||
next call's window is ``(end-1, ...]`` — i.e. strictly older than the
|
||||
oldest message in this batch.
|
||||
"""
|
||||
resp = await client.post(
|
||||
MSG_CHATS_PATH,
|
||||
data={
|
||||
"wechatId": wechat_id,
|
||||
"friendId": friend_id,
|
||||
"types": types,
|
||||
"start": str(start),
|
||||
"end": str(end_cursor),
|
||||
},
|
||||
headers={"content-type": _FORM_CONTENT_TYPE},
|
||||
)
|
||||
|
||||
try:
|
||||
body = resp.json()
|
||||
except (AttributeError, ValueError):
|
||||
# Malformed envelope — treat as empty + no-more so the paginator stops.
|
||||
return [], end_cursor, False
|
||||
|
||||
data = body.get("data") if isinstance(body, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
return [], end_cursor, False
|
||||
|
||||
rows = data.get("messages")
|
||||
if not isinstance(rows, list):
|
||||
rows = []
|
||||
|
||||
end = data.get("end")
|
||||
next_cursor = (
|
||||
int(end) - 1 if isinstance(end, (int, float)) and not isinstance(end, bool)
|
||||
else end_cursor
|
||||
)
|
||||
has_more = bool(data.get("hasLast"))
|
||||
return rows, next_cursor, has_more
|
||||
|
||||
|
||||
def _normalize_message(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Map one upstream ``msg/chats`` record to the ``Message`` column shape.
|
||||
|
||||
Adds ``channel="wechat"`` and preserves the original record under
|
||||
``raw_json``. The ``type`` field is normalized via
|
||||
:func:`normalize_wechat_type` to a stable enum string; the raw code is
|
||||
still recoverable from ``raw_json`` for the persistence layer.
|
||||
|
||||
``None``-safe: missing fields become ``None`` rather than raising, so a
|
||||
partial payload does not break the whole scrape. Value coercion
|
||||
(e.g. ``isSend``/``mine``/``isDel`` ``int`` -> ``bool``) is deferred to
|
||||
the persistence layer — here we carry the raw values through, except for
|
||||
``is_send`` which is pulled from ``mine`` (falling back to ``isSend``)
|
||||
since that disambiguation is straightforward.
|
||||
"""
|
||||
# `mine` and `isSend` carry the same semantic; the live capture uses both.
|
||||
# Prefer `mine`, fall back to `isSend`. Missing-key .get() returns None,
|
||||
# but `mine` may legitimately be the int 0 (falsy but present) — guard
|
||||
# explicitly with `is None`.
|
||||
mine = record.get("mine")
|
||||
is_send = mine if mine is not None else record.get("isSend")
|
||||
|
||||
raw_type = record.get("messageType")
|
||||
if raw_type is None:
|
||||
raw_type = record.get("type")
|
||||
|
||||
# `timestamp` is the canonical field; fall back to `createTime`.
|
||||
timestamp = record.get("timestamp")
|
||||
if timestamp is None:
|
||||
timestamp = record.get("createTime")
|
||||
|
||||
return {
|
||||
"msg_id": record.get("msgSvrId"),
|
||||
"channel": "wechat",
|
||||
"owner_account_id": record.get("wechatId"),
|
||||
"talker_id": record.get("talker"),
|
||||
"ori_talker": record.get("oriTalker"),
|
||||
"room_id": record.get("roomid"),
|
||||
"is_send": is_send,
|
||||
"type": normalize_wechat_type(raw_type),
|
||||
"text": record.get("text"),
|
||||
"file_url": record.get("file"),
|
||||
"file_th_url": record.get("fileTh"),
|
||||
"file_hd_url": record.get("fileHd"),
|
||||
"md5": record.get("md5"),
|
||||
"md5_th": record.get("md5Th"),
|
||||
"md5_hd": record.get("md5Hd"),
|
||||
"duration": record.get("duration"),
|
||||
"transcription_text": record.get("transcriptionText"),
|
||||
"img_path": record.get("imgPath"),
|
||||
"original_image": record.get("originalImage"),
|
||||
"pic_type": record.get("picType"),
|
||||
"pay_msg_id": record.get("paymsgid"),
|
||||
"refer_msg": record.get("referMsg"),
|
||||
"sender_name": record.get("name"),
|
||||
"sender_head_url": record.get("headUrl"),
|
||||
"timestamp": timestamp,
|
||||
"is_del": record.get("isDel"),
|
||||
"version": record.get("version"),
|
||||
"raw_json": record,
|
||||
}
|
||||
|
||||
|
||||
def _is_divider(record: dict[str, Any]) -> bool:
|
||||
"""True iff this record is a date-divider (``type``/``messageType`` == -2).
|
||||
|
||||
These are not real messages — the upstream inserts them as visual date
|
||||
bars in the chat UI — and must never be yielded or persisted. Per the
|
||||
Task 1.6 spec we drop a record when EITHER ``type`` or ``messageType`` is
|
||||
``-2``; all other unknown codes pass through.
|
||||
"""
|
||||
return normalize_wechat_type(record.get("messageType")) == "divider" or (
|
||||
normalize_wechat_type(record.get("type")) == "divider"
|
||||
)
|
||||
|
||||
|
||||
async def scrape_wechat_messages(
|
||||
client: YunkeClient,
|
||||
wechat_id: str,
|
||||
friend_id: str,
|
||||
start: int,
|
||||
end: int,
|
||||
*,
|
||||
types: str = "",
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield normalized WeChat chat messages, paging backward by timestamp.
|
||||
|
||||
Wraps ``POST /wechatweb/msg/chats`` (form-encoded body, NOT JSON). The
|
||||
endpoint pages BACKWARD: each response carries ``data.end`` (oldest
|
||||
timestamp in this batch) and ``data.hasLast`` (older messages exist?).
|
||||
The scraper advances the cursor to ``end - 1`` and repeats until
|
||||
``hasLast`` is false, then stops.
|
||||
|
||||
Date dividers (``type == -2``) are dropped — they are UI chrome, not real
|
||||
messages. All other type codes pass through (mapped to ``"unknown"`` when
|
||||
not in our small known-table) so no real message is silently lost.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client:
|
||||
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
|
||||
handled by the client.
|
||||
wechat_id:
|
||||
Staff WeChat id (``wechatId`` form field) — maps to
|
||||
``owner_account_id`` on each yielded record.
|
||||
friend_id:
|
||||
Friend ``wxid`` OR chatroom id (``friendId`` form field) — selects
|
||||
the conversation to pull.
|
||||
start, end:
|
||||
Time window in ms epoch. ``end`` is also the initial backward cursor;
|
||||
the first request queries the ``(start, end]`` window and the cursor
|
||||
walks toward ``start`` from there.
|
||||
types:
|
||||
Optional upstream message-type filter (passed through verbatim to the
|
||||
form body). Empty by default — pull all types.
|
||||
"""
|
||||
async for record in paginate_by_cursor(
|
||||
lambda end_cursor: _fetch_msg_page(
|
||||
client,
|
||||
wechat_id=wechat_id,
|
||||
friend_id=friend_id,
|
||||
types=types,
|
||||
start=start,
|
||||
end_cursor=end_cursor,
|
||||
),
|
||||
initial_cursor=end,
|
||||
):
|
||||
if _is_divider(record):
|
||||
continue
|
||||
yield _normalize_message(record)
|
||||
|
||||
Reference in New Issue
Block a user