feat(scraper): 微信群消息抓取(368778条/750群)+浏览器守护循环+SQLite忙等
- 新 group-messages 实体:真机探明群消息=同一 msg/chats 端点 friendId 传 roomId; talker=发送者wxid、msgId=null→owner:room:createTime:talker 合成稳定主键(重扫幂等) - 浏览器守护循环:超大群撑崩渲染进程时重启浏览器跳过毒群继续(实战2次自愈); 200页上限;恢复路径补 await - base.py SQLite connect_args timeout=60:并发写忙等而非撞锁报错 (双抓取器并发写实测 database is locked 崩任务) - 5个新单测,全量410绿 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
"""Unit tests for the WeChat GROUP message normalizer in
|
||||
:mod:`scripts.scrape_via_page` (msg/chats with friendId=roomId, confirmed
|
||||
live 2026-08-19). Group rows differ from 1:1: talker=sender, roomid=the room,
|
||||
msgId=null -> stable synthetic pk required for idempotent re-scrapes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from scripts.scrape_via_page import _normalize_group_message
|
||||
|
||||
OWNER = "wxid_wo97u1u2x11w22"
|
||||
ROOM = "56807599995@chatroom"
|
||||
|
||||
|
||||
def _row(**over):
|
||||
base = {
|
||||
"talker": "wxid_en2iklxtjxzz22",
|
||||
"oriTalker": ROOM,
|
||||
"name": "小**贸",
|
||||
"content": "hello",
|
||||
"type": 1,
|
||||
"roomid": ROOM,
|
||||
"createTime": 1787125769000,
|
||||
"msgId": None,
|
||||
}
|
||||
base.update(over)
|
||||
return base
|
||||
|
||||
|
||||
def test_group_row_maps_sender_and_room():
|
||||
rec = _normalize_group_message(_row(), OWNER, ROOM)
|
||||
assert rec["channel"] == "wechat"
|
||||
assert rec["owner_account_id"] == OWNER
|
||||
assert rec["room_id"] == ROOM
|
||||
assert rec["talker_id"] == "wxid_en2iklxtjxzz22" # 发送者(非会话对端)
|
||||
assert rec["sender_name"] == "小**贸"
|
||||
assert rec["type_raw"] == 1
|
||||
|
||||
|
||||
def test_synthetic_msg_id_stable_across_rescans():
|
||||
"""msgId=null 的群行两次归一化必须得到同一 msg_id(幂等 upsert 前提)。"""
|
||||
a = _normalize_group_message(_row(), OWNER, ROOM)
|
||||
b = _normalize_group_message(_row(), OWNER, ROOM)
|
||||
assert a["msg_id"] == b["msg_id"]
|
||||
assert a["msg_id"] == f"{OWNER}:{ROOM}:1787125769000:wxid_en2iklxtjxzz22"
|
||||
|
||||
|
||||
def test_real_msg_id_preferred_over_synthetic():
|
||||
rec = _normalize_group_message(_row(msgId="m999"), OWNER, ROOM)
|
||||
assert rec["msg_id"] == "m999"
|
||||
|
||||
|
||||
def test_divider_rows_dropped():
|
||||
assert _normalize_group_message(_row(type=-2, text="2026-08-19 15:49"), OWNER, ROOM) is None
|
||||
|
||||
|
||||
def test_roomid_falls_back_to_queried_room():
|
||||
rec = _normalize_group_message(_row(roomid=""), OWNER, ROOM)
|
||||
assert rec["room_id"] == ROOM
|
||||
Reference in New Issue
Block a user