feat(msg): 类型归一化扩展(spike判别+49拆红包/转账)+存量回填+API msg_type/q过滤
- _normalize_msg_type 升级 (raw_type, row): messageType 经典码优先 → 49/appmsg 行级拆分(红包/转账/聊天记录) → 云客码兜底 - spike 修正: 云客 type=3 实为语音(messageType=34, 8754/8955 行), 原 image_legacy 是误标; type_raw=3 的 201 行旧数据才是真图片 - 新映射: 5红包 6/7通话 9文件 10/18/20链接 11转账 13名片 14位置 15/10000系统 21引用回复→text 22/19 证据不足留 unknown - 回填 smoke.db: 620279 行重扫, 94268 行修正, unknown 87198→1885, 幂等 - GET /api/messages 加 msg_type(Literal 422)+q(LIKE 转义)参数 - 新增 test_scrape_msg_type.py(11) + test_api_messages 追加(8), 全量 466 绿 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,102 @@
|
||||
"""Backfill Message.type via the extended _normalize_msg_type (no re-scrape).
|
||||
|
||||
重扫范围: channel='wechat' AND type_raw IS NOT NULL 的**所有**行 —— 含
|
||||
type='unknown'/'file'(主要人群)与历史误标(如 type_raw=3 被标 image_legacy,
|
||||
实为 messageType=34 语音)。逐行用 raw_json(含 messageType 经典码 + type
|
||||
云客码)重算, 仅 UPDATE 与现值不同的行: 幂等, 重跑 0 改动; 已正确的类型
|
||||
不受影响。WhatsApp 渠道是另一套 type 编码(见 app/scraper/whatsapp.py),
|
||||
不在本脚本范围。
|
||||
|
||||
用法:
|
||||
cd yunke-server && .venv/Scripts/python.exe -m scripts.backfill_msg_types
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db import models # noqa: F401 (ensure tables are registered)
|
||||
from app.db.models import Message
|
||||
from scripts.scrape_via_page import _normalize_msg_type
|
||||
|
||||
BATCH = 5000 # 扫描批大小(按 msg_id 主键游标分页)
|
||||
UPDATE_CHUNK = 400 # 单条 IN (...) 的参数数, 低于 SQLite 999 变量上限
|
||||
|
||||
|
||||
async def main(db_url: str = "sqlite+aiosqlite:///./smoke.db") -> None:
|
||||
engine = create_async_engine(db_url)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
scanned = 0
|
||||
changed = 0
|
||||
by_new_type: Counter[str] = Counter()
|
||||
by_old_type: Counter[str] = Counter()
|
||||
async with sm() as s:
|
||||
last_id = ""
|
||||
while True:
|
||||
rows = (
|
||||
await s.execute(
|
||||
select(Message.msg_id, Message.type_raw, Message.type, Message.raw_json)
|
||||
.where(
|
||||
Message.channel == "wechat",
|
||||
Message.type_raw.is_not(None),
|
||||
Message.msg_id > last_id,
|
||||
)
|
||||
.order_by(Message.msg_id)
|
||||
.limit(BATCH)
|
||||
)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
pending: defaultdict[str, list[str]] = defaultdict(list)
|
||||
for mid, tr, cur, rj in rows:
|
||||
last_id = mid
|
||||
scanned += 1
|
||||
if isinstance(rj, dict):
|
||||
row = rj
|
||||
elif rj:
|
||||
try:
|
||||
row = json.loads(rj)
|
||||
except (ValueError, TypeError):
|
||||
continue # raw_json 损坏, 留给下次抓取刷新
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
else:
|
||||
row = {}
|
||||
new = _normalize_msg_type(tr, row=row)
|
||||
if new is None or new == cur:
|
||||
continue # 幂等: 已正确的行不动
|
||||
pending[new].append(mid)
|
||||
changed += 1
|
||||
by_new_type[new] += 1
|
||||
by_old_type[cur or "NULL"] += 1
|
||||
for new_type, ids in pending.items():
|
||||
for i in range(0, len(ids), UPDATE_CHUNK):
|
||||
await s.execute(
|
||||
update(Message)
|
||||
.where(Message.msg_id.in_(ids[i : i + UPDATE_CHUNK]))
|
||||
.values(type=new_type)
|
||||
)
|
||||
await s.commit()
|
||||
print(f"[{time.strftime('%H:%M:%S')}] scanned={scanned} changed={changed}...")
|
||||
# 回填后各 type 分布(wechat 渠道全景)
|
||||
final = (
|
||||
await s.execute(
|
||||
select(Message.type, func.count())
|
||||
.where(Message.channel == "wechat")
|
||||
.group_by(Message.type)
|
||||
)
|
||||
).all()
|
||||
print(f"DONE — scanned={scanned}, updated={changed}")
|
||||
print("updated -> new type:", dict(by_new_type.most_common()))
|
||||
print("updated <- old type:", dict(by_old_type.most_common()))
|
||||
print("wechat type distribution after backfill:", dict(final))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -28,6 +28,7 @@ import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -735,7 +736,7 @@ async def scrape_messages(cookie_str: str, max_conversations: int = 100, full_hi
|
||||
bool(row["isSend"]) if row.get("isSend") is not None
|
||||
else (bool(row["mine"]) if row.get("mine") is not None else None)
|
||||
),
|
||||
"type": _normalize_msg_type(row.get("type")),
|
||||
"type": _normalize_msg_type(row.get("type"), row=row),
|
||||
"type_raw": row.get("type"),
|
||||
"text": row.get("content") or row.get("text") or "",
|
||||
"file_url": row.get("file") or row.get("fileUrl"),
|
||||
@@ -827,7 +828,7 @@ def _normalize_group_message(row: dict, owner_id: str, room_id: str) -> dict | N
|
||||
bool(row["isSend"]) if row.get("isSend") is not None
|
||||
else (bool(row["mine"]) if row.get("mine") is not None else None)
|
||||
),
|
||||
"type": _normalize_msg_type(row.get("type")),
|
||||
"type": _normalize_msg_type(row.get("type"), row=row),
|
||||
"type_raw": row.get("type"),
|
||||
"text": row.get("content") or row.get("text") or "",
|
||||
"file_url": row.get("file") or row.get("fileUrl"),
|
||||
@@ -972,25 +973,140 @@ async def scrape_group_messages(cookie_str: str, max_groups: int = 500, full_his
|
||||
return total_fetched, total_upserted
|
||||
|
||||
|
||||
def _normalize_msg_type(raw_type) -> str | None:
|
||||
def _normalize_msg_type(raw_type, row: dict | None = None) -> str | None:
|
||||
"""Normalize a raw message type to our canonical type string.
|
||||
|
||||
微信 type=2 = 图片(新版,fileTh 带完整 OSS URL;真机 A**杰对话确认
|
||||
2026-08-20——旧映射漏掉它导致图片被存成 unknown 前端不渲染)。
|
||||
判别依据 smoke.db 2026-08-25 spike(每类 ≥3 条样本): 云客 API 行里
|
||||
``messageType`` 是经典微信协议码、``type`` 是云客自编码, 两者同在
|
||||
raw_json。优先级: messageType 经典码直判 → 49/appmsg 家族行级拆分
|
||||
(云客子码/paymsgid/text XML) → 云客码兜底(合成 flag 码时 type 才是
|
||||
唯一可靠来源)。
|
||||
|
||||
spike 修正: 云客 type=3 的行 8754/8955 是 messageType='34'(经典语音
|
||||
码, file=语音十六进制键) —— 旧 "3"→image_legacy 是误标, 现指向
|
||||
voice; messageType='3' 的行(旧 orchestrator 路径写入)才是真图片。
|
||||
"""
|
||||
row = row or {}
|
||||
if raw_type is None:
|
||||
return None
|
||||
raw_type = row.get("type")
|
||||
if raw_type is None:
|
||||
return None
|
||||
t = str(raw_type)
|
||||
mapping = {
|
||||
"1": "text",
|
||||
"2": "image", # 新版图片(fileTh URL)
|
||||
"3": "image_legacy", # 旧版图片(md5/键)
|
||||
"34": "voice",
|
||||
"43": "video",
|
||||
"47": "emoji",
|
||||
"49": "file",
|
||||
}
|
||||
return mapping.get(t, "unknown")
|
||||
|
||||
mt_raw = row.get("messageType")
|
||||
mt = str(mt_raw) if mt_raw is not None else None
|
||||
|
||||
# --- 1) appmsg 家族(经典码 49): 红包/转账/聊天记录/链接/文件 ---
|
||||
if mt == "49" or t == "49":
|
||||
return _normalize_appmsg_type(t, row)
|
||||
|
||||
# --- 2) 经典微信协议码直判 ---
|
||||
if mt is not None:
|
||||
if mt == "50":
|
||||
# VOIP 通话: 云客 6=语音通话(2077 行)/7=视频通话(77 行, 频率推断)
|
||||
return {"6": "voice_call", "7": "video_call"}.get(t, "voice_call")
|
||||
classic = _CLASSIC_MSG_TYPE.get(mt)
|
||||
if classic is not None:
|
||||
return classic
|
||||
|
||||
# --- 3) 云客自编码兜底(messageType 是合成 flag 码或缺失); 无行信息时
|
||||
# 也查经典码表(row-less 调用, 如单测里的 "34"/"42"/"10000") ---
|
||||
for candidate in (row.get("type"), t):
|
||||
if candidate is None:
|
||||
continue
|
||||
s = str(candidate)
|
||||
if s in _YUNKE_MSG_TYPE:
|
||||
return _YUNKE_MSG_TYPE[s]
|
||||
if s in _CLASSIC_MSG_TYPE:
|
||||
return _CLASSIC_MSG_TYPE[s]
|
||||
return "unknown"
|
||||
|
||||
|
||||
# 经典微信协议码(raw_json.messageType)——spike 样本证实
|
||||
_CLASSIC_MSG_TYPE = {
|
||||
"1": "text", # 473922 行
|
||||
"3": "image", # 50355 行(md5+OSS fileTh URL)
|
||||
"34": "voice", # 云客 type=3 的主体人群(8754 行, 语音十六进制键)
|
||||
"43": "video", # 4835 行
|
||||
"47": "emoji", # 4242 行(tc.qq.com 表情 CDN)
|
||||
"42": "namecard", # 953 行(text=昵称, file=头像)
|
||||
"48": "location", # 651 行(text=地址, file="lat,lng")
|
||||
"10000": "system", # 13820 行(打招呼/已添加提示)
|
||||
}
|
||||
|
||||
# 云客自编码(raw_json.type)——spike 样本证实; 22(1880 行无 text 无字段)/
|
||||
# 19(仅 5 行)证据不足, 宁留 unknown
|
||||
_YUNKE_MSG_TYPE = {
|
||||
"1": "text",
|
||||
"2": "image", # messageType=3
|
||||
"3": "voice", # messageType=34(旧 image_legacy 为误标)
|
||||
"4": "video", # messageType=43
|
||||
"5": "red_packet", # messageType=436207665, file=wx.gtimg.com/hongbao 封面
|
||||
"6": "voice_call", # messageType=50 VOIP
|
||||
"7": "video_call", # messageType=50 VOIP
|
||||
"8": "emoji", # messageType=47
|
||||
"9": "file", # messageType=49, text=文件名(xlsx/pdf)
|
||||
"10": "link", # messageType=49, 92% 带封面图
|
||||
"11": "transfer", # messageType=419430449, text="发送转账：￥…"
|
||||
"13": "namecard", # messageType=42
|
||||
"14": "location", # messageType=48
|
||||
"15": "system", # messageType=10000/268445456/285222674
|
||||
"18": "link", # messageType=49, 99.8% 带封面图(公众号分享)
|
||||
"20": "link", # messageType=754974769, 视频号分享(封面+话题文本)
|
||||
"21": "text", # messageType=822083633 引用回复(referMsg)
|
||||
}
|
||||
|
||||
# appmsg 家族内云客子码(row.type)——spike 样本: 9/10/18/21(+5/11 同语义码);
|
||||
# 不复用 _YUNKE_MSG_TYPE 全表, 避免假想组合(如 mt=49+type=6)错判成通话
|
||||
_APPMSG_SUB_TYPE = {
|
||||
"5": "red_packet",
|
||||
"9": "file",
|
||||
"10": "link",
|
||||
"11": "transfer",
|
||||
"18": "link",
|
||||
}
|
||||
|
||||
# appmsg text XML 里的 <type> 子码(仅 49 家族无云客子码时的老数据兜底)
|
||||
_APPMSG_XML_TYPE = {
|
||||
"5": "link",
|
||||
"6": "file",
|
||||
"19": "merged_forward", # 聊天记录(合并转发), spike 1694+7 行
|
||||
"2000": "transfer",
|
||||
"2001": "red_packet",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_appmsg_type(t: str, row: dict) -> str:
|
||||
"""49/appmsg 家族的行级拆分(红包/转账/聊天记录/链接/文件)。
|
||||
|
||||
云客子码 row.type 优先(spike: 9=file 10/18=link 21+XML=聊天记录);
|
||||
老数据缺子码时按 paymsgid(实测全空, 保留兼容)与 text XML 子码拆;
|
||||
全缺 → file(现状回退)。未知子码不猜 → unknown。
|
||||
"""
|
||||
text = str(row.get("text") or "")
|
||||
sub = row.get("type")
|
||||
if sub is not None and str(sub) != "49":
|
||||
s = str(sub)
|
||||
if s == "21":
|
||||
# 云客 21 在 appmsg 家族里=聊天记录 XML(messageType=822083633
|
||||
# 的引用回复不走本分支)
|
||||
return "merged_forward"
|
||||
return _APPMSG_SUB_TYPE.get(s, "unknown")
|
||||
if row.get("paymsgid"):
|
||||
if ">2001<" in text:
|
||||
return "red_packet"
|
||||
if ">2000<" in text:
|
||||
return "transfer"
|
||||
# paymsgid 有值但 XML 无 type → transfer(转账才有 paymsgid 概率高)
|
||||
return "transfer"
|
||||
if ">2001<" in text:
|
||||
return "red_packet"
|
||||
if ">2000<" in text:
|
||||
return "transfer"
|
||||
m = re.search(r"<type>(\d+)</type>", text)
|
||||
if m:
|
||||
return _APPMSG_XML_TYPE.get(m.group(1), "unknown")
|
||||
return "file"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user