feat(msg): 类型归一化扩展(spike判别+49拆红包/转账)+存量回填+API msg_type/q过滤

- _normalize_msg_type 升级 (raw_type, row): messageType 经典码优先 → 49/appmsg 行级拆分(红包/转账/聊天记录) → 云客码兜底 - spike 修正: 云客 type=3 实为语音(messageType=34, 8754/8955 行), 原 image_legacy 是误标; type_raw=3 的 201 行旧数据才是真图片 - 新映射: 5红包 6/7通话 9文件 10/18/20链接 11转账 13名片 14位置 15/10000系统 21引用回复→text 22/19 证据不足留 unknown - 回填 smoke.db: 620279 行重扫, 94268 行修正, unknown 87198→1885, 幂等 - GET /api/messages 加 msg_type(Literal 422)+q(LIKE 转义)参数 - 新增 test_scrape_msg_type.py(11) + test_api_messages 追加(8), 全量 466 绿 Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-25 15:17:21 +08:00
parent f6ddd0263a
commit 64577a1b02
5 changed files with 537 additions and 17 deletions
@@ -28,6 +28,7 @@ import asyncio
 import json
 import logging
 import os
+import re
 from typing import Any
 from urllib.parse import urlparse

@@ -735,7 +736,7 @@ async def scrape_messages(cookie_str: str, max_conversations: int = 100, full_hi
                                bool(row["isSend"]) if row.get("isSend") is not None
                                else (bool(row["mine"]) if row.get("mine") is not None else None)
                            ),
-                            "type": _normalize_msg_type(row.get("type")),
+                            "type": _normalize_msg_type(row.get("type"), row=row),
                            "type_raw": row.get("type"),
                            "text": row.get("content") or row.get("text") or "",
                            "file_url": row.get("file") or row.get("fileUrl"),
@@ -827,7 +828,7 @@ def _normalize_group_message(row: dict, owner_id: str, room_id: str) -> dict | N
                                bool(row["isSend"]) if row.get("isSend") is not None
                                else (bool(row["mine"]) if row.get("mine") is not None else None)
                            ),
-        "type": _normalize_msg_type(row.get("type")),
+        "type": _normalize_msg_type(row.get("type"), row=row),
        "type_raw": row.get("type"),
        "text": row.get("content") or row.get("text") or "",
        "file_url": row.get("file") or row.get("fileUrl"),
@@ -972,25 +973,140 @@ async def scrape_group_messages(cookie_str: str, max_groups: int = 500, full_his
    return total_fetched, total_upserted


-def _normalize_msg_type(raw_type) -> str | None:
+def _normalize_msg_type(raw_type, row: dict | None = None) -> str | None:
    """Normalize a raw message type to our canonical type string.

-    微信 type=2 = 图片(新版,fileTh 带完整 OSS URL;真机 A**杰对话确认
-    2026-08-20——旧映射漏掉它导致图片被存成 unknown 前端不渲染)。
+    判别依据 smoke.db 2026-08-25 spike(每类 ≥3 条样本): 云客 API 行里
+    ``messageType`` 是经典微信协议码、``type`` 是云客自编码, 两者同在
+    raw_json。优先级: messageType 经典码直判 → 49/appmsg 家族行级拆分
+    (云客子码/paymsgid/text XML) → 云客码兜底(合成 flag 码时 type 才是
+    唯一可靠来源)。
+
+    spike 修正: 云客 type=3 的行 8754/8955 是 messageType='34'(经典语音
+    码, file=语音十六进制键) —— 旧 "3"→image_legacy 是误标, 现指向
+    voice; messageType='3' 的行(旧 orchestrator 路径写入)才是真图片。
    """
+    row = row or {}
    if raw_type is None:
-        return None
+        raw_type = row.get("type")
+        if raw_type is None:
+            return None
    t = str(raw_type)
-    mapping = {
-        "1": "text",
-        "2": "image",          # 新版图片(fileTh URL)
-        "3": "image_legacy",   # 旧版图片(md5/键)
-        "34": "voice",
-        "43": "video",
-        "47": "emoji",
-        "49": "file",
-    }
-    return mapping.get(t, "unknown")
+
+    mt_raw = row.get("messageType")
+    mt = str(mt_raw) if mt_raw is not None else None
+
+    # --- 1) appmsg 家族(经典码 49): 红包/转账/聊天记录/链接/文件 ---
+    if mt == "49" or t == "49":
+        return _normalize_appmsg_type(t, row)
+
+    # --- 2) 经典微信协议码直判 ---
+    if mt is not None:
+        if mt == "50":
+            # VOIP 通话: 云客 6=语音通话(2077 行)/7=视频通话(77 行, 频率推断)
+            return {"6": "voice_call", "7": "video_call"}.get(t, "voice_call")
+        classic = _CLASSIC_MSG_TYPE.get(mt)
+        if classic is not None:
+            return classic
+
+    # --- 3) 云客自编码兜底(messageType 是合成 flag 码或缺失); 无行信息时
+    #     也查经典码表(row-less 调用, 如单测里的 "34"/"42"/"10000") ---
+    for candidate in (row.get("type"), t):
+        if candidate is None:
+            continue
+        s = str(candidate)
+        if s in _YUNKE_MSG_TYPE:
+            return _YUNKE_MSG_TYPE[s]
+        if s in _CLASSIC_MSG_TYPE:
+            return _CLASSIC_MSG_TYPE[s]
+    return "unknown"
+
+
+# 经典微信协议码(raw_json.messageType)——spike 样本证实
+_CLASSIC_MSG_TYPE = {
+    "1": "text",        # 473922 行
+    "3": "image",       # 50355 行(md5+OSS fileTh URL)
+    "34": "voice",      # 云客 type=3 的主体人群(8754 行, 语音十六进制键)
+    "43": "video",      # 4835 行
+    "47": "emoji",      # 4242 行(tc.qq.com 表情 CDN)
+    "42": "namecard",   # 953 行(text=昵称, file=头像)
+    "48": "location",   # 651 行(text=地址, file="lat,lng")
+    "10000": "system",  # 13820 行(打招呼/已添加提示)
+}
+
+# 云客自编码(raw_json.type)——spike 样本证实; 22(1880 行无 text 无字段)/
+# 19(仅 5 行)证据不足, 宁留 unknown
+_YUNKE_MSG_TYPE = {
+    "1": "text",
+    "2": "image",        # messageType=3
+    "3": "voice",        # messageType=34(旧 image_legacy 为误标)
+    "4": "video",        # messageType=43
+    "5": "red_packet",   # messageType=436207665, file=wx.gtimg.com/hongbao 封面
+    "6": "voice_call",   # messageType=50 VOIP
+    "7": "video_call",   # messageType=50 VOIP
+    "8": "emoji",        # messageType=47
+    "9": "file",         # messageType=49, text=文件名(xlsx/pdf)
+    "10": "link",        # messageType=49, 92% 带封面图
+    "11": "transfer",    # messageType=419430449, text="发送转账：￥…"
+    "13": "namecard",    # messageType=42
+    "14": "location",    # messageType=48
+    "15": "system",      # messageType=10000/268445456/285222674
+    "18": "link",        # messageType=49, 99.8% 带封面图(公众号分享)
+    "20": "link",        # messageType=754974769, 视频号分享(封面+话题文本)
+    "21": "text",        # messageType=822083633 引用回复(referMsg)
+}
+
+# appmsg 家族内云客子码(row.type)——spike 样本: 9/10/18/21(+5/11 同语义码);
+# 不复用 _YUNKE_MSG_TYPE 全表, 避免假想组合(如 mt=49+type=6)错判成通话
+_APPMSG_SUB_TYPE = {
+    "5": "red_packet",
+    "9": "file",
+    "10": "link",
+    "11": "transfer",
+    "18": "link",
+}
+
+# appmsg text XML 里的 <type> 子码(仅 49 家族无云客子码时的老数据兜底)
+_APPMSG_XML_TYPE = {
+    "5": "link",
+    "6": "file",
+    "19": "merged_forward",   # 聊天记录(合并转发), spike 1694+7 行
+    "2000": "transfer",
+    "2001": "red_packet",
+}
+
+
+def _normalize_appmsg_type(t: str, row: dict) -> str:
+    """49/appmsg 家族的行级拆分(红包/转账/聊天记录/链接/文件)。
+
+    云客子码 row.type 优先(spike: 9=file 10/18=link 21+XML=聊天记录);
+    老数据缺子码时按 paymsgid(实测全空, 保留兼容)与 text XML 子码拆;
+    全缺 → file(现状回退)。未知子码不猜 → unknown。
+    """
+    text = str(row.get("text") or "")
+    sub = row.get("type")
+    if sub is not None and str(sub) != "49":
+        s = str(sub)
+        if s == "21":
+            # 云客 21 在 appmsg 家族里=聊天记录 XML(messageType=822083633
+            # 的引用回复不走本分支)
+            return "merged_forward"
+        return _APPMSG_SUB_TYPE.get(s, "unknown")
+    if row.get("paymsgid"):
+        if ">2001<" in text:
+            return "red_packet"
+        if ">2000<" in text:
+            return "transfer"
+        # paymsgid 有值但 XML 无 type → transfer(转账才有 paymsgid 概率高)
+        return "transfer"
+    if ">2001<" in text:
+        return "red_packet"
+    if ">2000<" in text:
+        return "transfer"
+    m = re.search(r"<type>(\d+)</type>", text)
+    if m:
+        return _APPMSG_XML_TYPE.get(m.group(1), "unknown")
+    return "file"


 # ---------------------------------------------------------------------------