feat(msg): 类型归一化扩展(spike判别+49拆红包/转账)+存量回填+API msg_type/q过滤

- _normalize_msg_type 升级 (raw_type, row): messageType 经典码优先 → 49/appmsg 行级拆分(红包/转账/聊天记录) → 云客码兜底 - spike 修正: 云客 type=3 实为语音(messageType=34, 8754/8955 行), 原 image_legacy 是误标; type_raw=3 的 201 行旧数据才是真图片 - 新映射: 5红包 6/7通话 9文件 10/18/20链接 11转账 13名片 14位置 15/10000系统 21引用回复→text 22/19 证据不足留 unknown - 回填 smoke.db: 620279 行重扫, 94268 行修正, unknown 87198→1885, 幂等 - GET /api/messages 加 msg_type(Literal 422)+q(LIKE 转义)参数 - 新增 test_scrape_msg_type.py(11) + test_api_messages 追加(8), 全量 466 绿 Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-25 15:17:21 +08:00
parent f6ddd0263a
commit 64577a1b02
5 changed files with 537 additions and 17 deletions
@@ -360,3 +360,119 @@ async def test_media_fields_default_none_when_unset(make_client):
    ):
        assert key in it
        assert it[key] is None
+
+
+# ---------------------------------------------------------------------------
+# msg_type / q 过滤(Task 4: 类型归一化扩展 + API 过滤, 对齐云客 10 类下拉)
+# ---------------------------------------------------------------------------
+
+
+async def _seed_types(session: AsyncSession) -> None:
+    """text / image / image_legacy / red_packet / transfer 各一条 + 关键词行。"""
+    session.add_all(
+        [
+            Message(
+                msg_id="ty_text", channel="wechat", owner_account_id="wx_owner_1",
+                talker_id="t_a", text="报价 135800 元", timestamp=1000, is_send=True,
+                type="text", type_raw=1,
+            ),
+            Message(
+                msg_id="ty_img", channel="wechat", owner_account_id="wx_owner_1",
+                talker_id="t_a", text="", timestamp=2000, is_send=False,
+                type="image", type_raw=2,
+            ),
+            Message(
+                msg_id="ty_img_legacy", channel="wechat", owner_account_id="wx_owner_1",
+                talker_id="t_a", text="", timestamp=2500, is_send=False,
+                type="image_legacy", type_raw=3,
+            ),
+            Message(
+                msg_id="ty_red", channel="wechat", owner_account_id="wx_owner_1",
+                talker_id="t_a", text="", timestamp=3000, is_send=True,
+                type="red_packet", type_raw=5,
+            ),
+            Message(
+                msg_id="ty_transfer", channel="wechat", owner_account_id="wx_owner_1",
+                talker_id="t_a", text="发送转账：￥100.00", timestamp=4000, is_send=True,
+                type="transfer", type_raw=11,
+            ),
+            Message(
+                msg_id="ty_text_2", channel="wechat", owner_account_id="wx_owner_1",
+                talker_id="t_a", text="含 100% 通配符字面量", timestamp=5000, is_send=True,
+                type="text", type_raw=1,
+            ),
+        ]
+    )
+    await session.commit()
+
+
+async def test_msg_type_filter_text(make_client):
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"msg_type": "text"})
+    assert r.status_code == 200
+    body = r.json()
+    assert {it["msg_id"] for it in body["items"]} == {"ty_text", "ty_text_2"}
+
+
+async def test_msg_type_filter_image_includes_legacy(make_client):
+    """msg_type=image 覆盖 image + image_legacy(过渡期兼容)。"""
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"msg_type": "image"})
+    assert r.status_code == 200
+    body = r.json()
+    assert {it["msg_id"] for it in body["items"]} == {"ty_img", "ty_img_legacy"}
+
+
+async def test_msg_type_filter_red_packet_only(make_client):
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"msg_type": "red_packet"})
+    assert r.status_code == 200
+    body = r.json()
+    assert [it["msg_id"] for it in body["items"]] == ["ty_red"]
+    assert body["total"] == 1
+
+
+async def test_msg_type_filter_no_rows_returns_empty(make_client):
+    """voice_call 等暂无样本类型: 过滤语法有效, 查空是数据现实。"""
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"msg_type": "voice_call"})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["items"] == []
+    assert body["total"] == 0
+
+
+async def test_msg_type_unknown_rejected_422(make_client):
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"msg_type": "bogus"})
+    assert r.status_code == 422
+
+
+async def test_q_substring_filter(make_client):
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"q": "报价"})
+    assert r.status_code == 200
+    body = r.json()
+    assert [it["msg_id"] for it in body["items"]] == ["ty_text"]
+    assert body["total"] == 1
+
+
+async def test_q_escapes_like_wildcards(make_client):
+    """q 里的 %/_ 按字面匹配, 不当通配符。"""
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"q": "100%"})
+    assert r.status_code == 200
+    body = r.json()
+    # 只命中字面含 "100%" 的行; 不因 % 通配漏扩到 "￥100.00" 之外的行
+    assert [it["msg_id"] for it in body["items"]] == ["ty_text_2"]
+    r2 = await client.get("/api/messages", params={"q": "100"})
+    ids2 = {it["msg_id"] for it in r2.json()["items"]}
+    assert ids2 == {"ty_transfer", "ty_text_2"}
+
+
+async def test_msg_type_and_q_combine(make_client):
+    client = await make_client(_seed_types)
+    r = await client.get("/api/messages", params={"msg_type": "transfer", "q": "转账"})
+    assert r.status_code == 200
+    body = r.json()
+    assert [it["msg_id"] for it in body["items"]] == ["ty_transfer"]
@@ -0,0 +1,144 @@
+"""Tests for :func:`scripts.scrape_via_page._normalize_msg_type` (extended).
+
+判别依据 smoke.db 2026-08-25 spike 实测(wechat 渠道): 云客 API 行里
+``messageType`` 是经典微信协议码(1文本/3图片/34语音/43视频/47表情/42名片/
+48位置/49appmsg/50VOIP通话/10000系统), ``type`` 是云客自编码, 两者同在
+raw_json 里。历史 DB 的 type_raw 列是二者混合(新抓取路径存 type, 旧
+orchestrator 路径存 messageType), 因此归一化以 messageType 优先、云客码
+兜底, 并对 49/appmsg 家族做行级拆分(红包/转账/聊天记录/链接/文件)。
+"""
+
+from __future__ import annotations
+
+from scripts.scrape_via_page import _normalize_msg_type
+
+
+# ---------------------------------------------------------------------------
+# 既有映射(1/2/34/43/47/49)保持不变 + None 透传
+# ---------------------------------------------------------------------------
+
+
+def test_basic_mapping_unchanged():
+    assert _normalize_msg_type("1") == "text"
+    assert _normalize_msg_type("2") == "image"
+    assert _normalize_msg_type("34") == "voice"
+    assert _normalize_msg_type("43") == "video"
+    assert _normalize_msg_type("47") == "emoji"
+    # 缺行级信息时 49 回退 file(现状)
+    assert _normalize_msg_type("49") == "file"
+    assert _normalize_msg_type(None) is None
+
+
+def test_type_3_is_voice_not_image_legacy():
+    """spike 修正: 云客 type=3 行 8754/8955 的 messageType='34'(经典语音码,
+    file=语音十六进制键) —— 原 "3"→image_legacy 是误标(作者把语音键当成了
+    旧版图片键)。messageType='3' 的 201 行(旧路径)才是真图片。"""
+    assert _normalize_msg_type("3", row={"messageType": "34", "type": 3}) == "voice"
+    assert _normalize_msg_type("3", row={"messageType": "3", "type": 3}) == "image"
+    # 无行信息: 3 的主体人群(98%)是语音
+    assert _normalize_msg_type("3") == "voice"
+
+
+# ---------------------------------------------------------------------------
+# spike 判别出的新映射(每个 code 至少 3 条 smoke.db 样本支撑)
+# ---------------------------------------------------------------------------
+
+
+def test_new_mappings_row_based():
+    # 云客码兜底表(raw_json.messageType 为合成 flag 码时靠 type 判)
+    assert _normalize_msg_type("4", row={"messageType": "43", "type": 4}) == "video"
+    assert _normalize_msg_type("5", row={"messageType": "436207665", "type": 5}) == "red_packet"
+    assert _normalize_msg_type("6", row={"messageType": "50", "type": 6}) == "voice_call"
+    assert _normalize_msg_type("7", row={"messageType": "50", "type": 7}) == "video_call"
+    assert _normalize_msg_type("8", row={"messageType": "47", "type": 8}) == "emoji"
+    assert _normalize_msg_type("9", row={"messageType": "49", "type": 9}) == "file"
+    assert _normalize_msg_type("10", row={"messageType": "49", "type": 10}) == "link"
+    assert _normalize_msg_type("11", row={"messageType": "419430449", "type": 11}) == "transfer"
+    assert _normalize_msg_type("13", row={"messageType": "42", "type": 13}) == "namecard"
+    assert _normalize_msg_type("14", row={"messageType": "48", "type": 14}) == "location"
+    assert _normalize_msg_type("15", row={"messageType": "10000", "type": 15}) == "system"
+    assert _normalize_msg_type("18", row={"messageType": "49", "type": 18}) == "link"
+    assert _normalize_msg_type("20", row={"messageType": "754974769", "type": 20}) == "link"
+    # 21 引用回复(messageType=822083633 合成码, referMsg 带 quoted 内容)
+    assert _normalize_msg_type("21", row={"messageType": "822083633", "type": 21}) == "text"
+
+
+def test_classic_message_type_wins():
+    """messageType(经典码)优先于云客码/传入 raw_type。"""
+    assert _normalize_msg_type(3, row={"messageType": "34"}) == "voice"
+    assert _normalize_msg_type(2, row={"messageType": "3", "type": 2}) == "image"
+    assert _normalize_msg_type(1, row={"messageType": "1", "type": 1}) == "text"
+    assert _normalize_msg_type(8, row={"messageType": "47", "type": 8}) == "emoji"
+    assert _normalize_msg_type(13, row={"messageType": "42", "type": 13}) == "namecard"
+    assert _normalize_msg_type(14, row={"messageType": "48", "type": 14}) == "location"
+    assert _normalize_msg_type(15, row={"messageType": "10000", "type": 15}) == "system"
+
+
+def test_voip_call_types():
+    """messageType=50(经典 VOIP): 云客 6=语音通话(2077 行) / 7=视频通话(77 行)。"""
+    assert _normalize_msg_type("6", row={"messageType": "50", "type": 6, "duration": 68}) == "voice_call"
+    assert _normalize_msg_type("7", row={"messageType": "50", "type": 7, "duration": 107}) == "video_call"
+
+
+def test_quote_reply_maps_to_text():
+    """引用回复(822083633)带 referMsg, 正文是纯文本 → text。"""
+    row = {"messageType": "822083633", "type": 21, "text": "你去咋没交上？",
+           "referMsg": {"text": "老子自己去也没交上"}}
+    assert _normalize_msg_type("21", row=row) == "text"
+
+
+def test_merged_forward_chat_record():
+    """appmsg 里的 <type>19</type> 聊天记录(合并转发, spike 1694+7 行)。"""
+    row = {
+        "messageType": "49", "type": 21,
+        "text": "<msg><appmsg><title>聊天记录</title><type>19</type><recorditem><![CDATA[...]]></recorditem></appmsg></msg>",
+    }
+    assert _normalize_msg_type("21", row=row) == "merged_forward"
+
+
+def test_evidence_insufficient_stays_unknown():
+    """无判别证据的类宁留 unknown: 云客 22(拍一拍类系统 flag, 1880 行全无 text
+    全无字段)与 19(仅 5 行空 text)不猜。"""
+    assert _normalize_msg_type("22", row={"messageType": "922746929", "type": 22}) == "unknown"
+    assert _normalize_msg_type("19", row={"messageType": "-1879048186", "type": 19}) == "unknown"
+    assert _normalize_msg_type("9999") == "unknown"
+
+
+# ---------------------------------------------------------------------------
+# 49 / appmsg 家族行级拆分(红包/转账)
+# ---------------------------------------------------------------------------
+
+
+def test_type_49_redpacket_vs_transfer():
+    # 拆分带行级信息: paymsgid 非空 → 按 text XML appmsg type 判, XML 无 type → transfer
+    assert _normalize_msg_type("49", row={"paymsgid": "abc"}) in ("red_packet", "transfer")
+    assert _normalize_msg_type("49", row={"paymsgid": "abc"}) == "transfer"
+    # text 含 >2001< → red_packet; >2000< → transfer
+    assert _normalize_msg_type("49", row={"text": "<msg><appmsg><type>2001</type>"}) == "red_packet"
+    assert _normalize_msg_type("49", row={"text": "<msg><appmsg><type>2000</type>"}) == "transfer"
+    assert _normalize_msg_type("49", row={"paymsgid": "abc", "text": "<msg><appmsg><type>2001</type>"}) == "red_packet"
+    # 子码直判(云客 type 字段)
+    assert _normalize_msg_type("49", row={"type": "10"}) == "link"
+    assert _normalize_msg_type("49", row={"type": "9"}) == "file"
+    assert _normalize_msg_type("49", row={"type": "18"}) == "link"
+    # 缺信息回退现有行为
+    assert _normalize_msg_type("49", row={}) == "file"
+    # 未知子码(无样本)不猜成 file 之外的语义
+    assert _normalize_msg_type("49", row={"type": "777"}) == "unknown"
+
+
+def test_legacy_classic_49_rows_use_yunke_subcode():
+    """DB 里 202 条 type_raw=49 旧行(orchestrator 路径写入) raw_json.type
+    已是云客子码(10/21/18/9) —— 子码优先拆分。"""
+    assert _normalize_msg_type("49", row={"messageType": "49", "type": 10}) == "link"
+    assert _normalize_msg_type("49", row={"messageType": "49", "type": 18}) == "link"
+    assert _normalize_msg_type("49", row={"messageType": "49", "type": 9}) == "file"
+    assert _normalize_msg_type("49", row={"messageType": "49", "type": 21,
+                                          "text": "<msg><appmsg><type>19</type>"}) == "merged_forward"
+
+
+def test_row_only_without_raw_type():
+    """raw_type 缺失时从 row.type 取(spike 后调用点都传整行)。"""
+    assert _normalize_msg_type(None, row={"type": 1, "messageType": "1"}) == "text"
+    assert _normalize_msg_type(None, row={"type": 5, "messageType": "436207665"}) == "red_packet"
+    assert _normalize_msg_type(None, row={}) is None