浅念
64577a1b02
- _normalize_msg_type 升级 (raw_type, row): messageType 经典码优先 → 49/appmsg 行级拆分(红包/转账/聊天记录) → 云客码兜底 - spike 修正: 云客 type=3 实为语音(messageType=34, 8754/8955 行), 原 image_legacy 是误标; type_raw=3 的 201 行旧数据才是真图片 - 新映射: 5红包 6/7通话 9文件 10/18/20链接 11转账 13名片 14位置 15/10000系统 21引用回复→text 22/19 证据不足留 unknown - 回填 smoke.db: 620279 行重扫, 94268 行修正, unknown 87198→1885, 幂等 - GET /api/messages 加 msg_type(Literal 422)+q(LIKE 转义)参数 - 新增 test_scrape_msg_type.py(11) + test_api_messages 追加(8), 全量 466 绿 Co-Authored-By: Claude <noreply@anthropic.com>
479 lines
16 KiB
Python
479 lines
16 KiB
Python
"""Tests for GET /api/messages — chat timeline.
|
|
|
|
Seeds 3 messages (2 for talker t_a across times 1000/3000, 1 for talker t_b at
|
|
2000) and asserts filters (channel, owner, talker, time window) + ordering
|
|
(``timestamp asc``) + pagination.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.db.models import Message
|
|
|
|
|
|
async def _seed(session: AsyncSession) -> None:
|
|
session.add_all(
|
|
[
|
|
Message(
|
|
msg_id="m1",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id="t_a",
|
|
room_id=None,
|
|
text="hello",
|
|
timestamp=1000,
|
|
is_send=True,
|
|
type="text",
|
|
type_raw=1,
|
|
),
|
|
Message(
|
|
msg_id="m2",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id="t_b",
|
|
room_id=None,
|
|
text="hi",
|
|
timestamp=2000,
|
|
is_send=False,
|
|
type="text",
|
|
type_raw=1,
|
|
),
|
|
Message(
|
|
msg_id="m3",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id="t_a",
|
|
room_id=None,
|
|
text="bye",
|
|
timestamp=3000,
|
|
is_send=True,
|
|
type="text",
|
|
type_raw=1,
|
|
),
|
|
]
|
|
)
|
|
await session.commit()
|
|
|
|
|
|
async def test_default_page_returns_all_ordered_asc(make_client):
|
|
"""No filters -> 3 rows ordered by ``timestamp asc`` (timeline order)."""
|
|
client = await make_client(_seed)
|
|
r = await client.get("/api/messages")
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["total"] == 3
|
|
assert len(body["items"]) == 3
|
|
assert [it["msg_id"] for it in body["items"]] == ["m1", "m2", "m3"]
|
|
for key in (
|
|
"msg_id",
|
|
"channel",
|
|
"owner_account_id",
|
|
"talker_id",
|
|
"room_id",
|
|
"is_send",
|
|
"type",
|
|
"type_raw",
|
|
"text",
|
|
"file_url",
|
|
"sender_name",
|
|
"sender_head_url",
|
|
"timestamp",
|
|
"raw_json",
|
|
):
|
|
assert key in body["items"][0]
|
|
|
|
|
|
async def test_raw_json_serializes_verbatim(make_client):
|
|
"""``raw_json`` round-trips the verbatim upstream message record.
|
|
|
|
The seeded messages omit ``raw_json`` (NULL); this test seeds one with a
|
|
populated blob and asserts every camelCase key is preserved verbatim while
|
|
the existing rows still serialize as ``null``.
|
|
"""
|
|
|
|
async def seed(session: AsyncSession) -> None:
|
|
await _seed(session)
|
|
session.add(
|
|
Message(
|
|
msg_id="m_raw",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id="t_a",
|
|
room_id=None,
|
|
type="image",
|
|
type_raw=3,
|
|
timestamp=5000,
|
|
raw_json={
|
|
"msgSvrId": "m_raw",
|
|
"type": 3,
|
|
"file": "https://oss/full.jpg",
|
|
"fileTh": "https://oss/th.jpg",
|
|
"fileHd": "https://oss/hd.jpg",
|
|
"md5": "abc123",
|
|
"md5Th": "th123",
|
|
"md5Hd": "hd123",
|
|
"duration": 0,
|
|
"talker": "t_a",
|
|
"oriTalker": "t_a",
|
|
"name": "Alice",
|
|
"headUrl": "https://oss/alice.jpg",
|
|
"wechatId": "wx_owner_1",
|
|
"roomid": "",
|
|
"isSend": 1,
|
|
"paymsgid": "",
|
|
"referMsg": {"text": "原文"},
|
|
},
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
client = await make_client(seed)
|
|
r = await client.get("/api/messages", params={"talker_id": "t_a"})
|
|
assert r.status_code == 200
|
|
by_id = {it["msg_id"]: it for it in r.json()["items"]}
|
|
|
|
raw = by_id["m_raw"]["raw_json"]
|
|
assert raw == {
|
|
"msgSvrId": "m_raw",
|
|
"type": 3,
|
|
"file": "https://oss/full.jpg",
|
|
"fileTh": "https://oss/th.jpg",
|
|
"fileHd": "https://oss/hd.jpg",
|
|
"md5": "abc123",
|
|
"md5Th": "th123",
|
|
"md5Hd": "hd123",
|
|
"duration": 0,
|
|
"talker": "t_a",
|
|
"oriTalker": "t_a",
|
|
"name": "Alice",
|
|
"headUrl": "https://oss/alice.jpg",
|
|
"wechatId": "wx_owner_1",
|
|
"roomid": "",
|
|
"isSend": 1,
|
|
"paymsgid": "",
|
|
"referMsg": {"text": "原文"},
|
|
}
|
|
|
|
# Seeded messages without raw_json serialize as JSON null.
|
|
assert by_id["m1"]["raw_json"] is None
|
|
|
|
|
|
async def test_filter_by_talker_id(make_client):
|
|
"""``?talker_id=t_a`` -> 2 messages for that 1:1 conversation."""
|
|
client = await make_client(_seed)
|
|
r = await client.get("/api/messages", params={"talker_id": "t_a"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["total"] == 2
|
|
assert {it["msg_id"] for it in body["items"]} == {"m1", "m3"}
|
|
|
|
|
|
async def test_filter_by_owner_account_id(make_client):
|
|
"""``?owner_account_id=wx_owner_1`` -> all 3 (all owned by this account)."""
|
|
client = await make_client(_seed)
|
|
r = await client.get("/api/messages", params={"owner_account_id": "wx_owner_1"})
|
|
assert r.status_code == 200
|
|
assert r.json()["total"] == 3
|
|
|
|
|
|
async def test_filter_by_owner_account_id_no_match(make_client):
|
|
"""A non-existent owner account -> 0 rows."""
|
|
client = await make_client(_seed)
|
|
r = await client.get("/api/messages", params={"owner_account_id": "nobody"})
|
|
assert r.status_code == 200
|
|
assert r.json()["total"] == 0
|
|
|
|
|
|
async def test_time_window_filter(make_client):
|
|
"""``?start=1500&end=2500`` -> only m2 (timestamp 2000)."""
|
|
client = await make_client(_seed)
|
|
r = await client.get(
|
|
"/api/messages", params={"start": 1500, "end": 2500}
|
|
)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["total"] == 1
|
|
assert body["items"][0]["msg_id"] == "m2"
|
|
|
|
|
|
async def test_time_window_combined_with_talker(make_client):
|
|
"""``talker_id=t_a & start=1500`` -> only m3 (t_a's later message)."""
|
|
client = await make_client(_seed)
|
|
r = await client.get(
|
|
"/api/messages", params={"talker_id": "t_a", "start": 1500}
|
|
)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["total"] == 1
|
|
assert body["items"][0]["msg_id"] == "m3"
|
|
|
|
|
|
async def test_channel_filter(make_client):
|
|
"""``?channel=whatsapp`` -> 0 (all seeded are wechat)."""
|
|
client = await make_client(_seed)
|
|
r = await client.get("/api/messages", params={"channel": "whatsapp"})
|
|
assert r.status_code == 200
|
|
assert r.json()["total"] == 0
|
|
|
|
|
|
async def test_pagination_page_size_one(make_client):
|
|
"""``page_size=1&page=1`` -> 1 item, total still 3; page 2 -> next item."""
|
|
client = await make_client(_seed)
|
|
r1 = await client.get(
|
|
"/api/messages", params={"page": 1, "page_size": 1}
|
|
)
|
|
b1 = r1.json()
|
|
assert b1["total"] == 3
|
|
assert len(b1["items"]) == 1
|
|
assert b1["items"][0]["msg_id"] == "m1"
|
|
|
|
r2 = await client.get(
|
|
"/api/messages", params={"page": 2, "page_size": 1}
|
|
)
|
|
b2 = r2.json()
|
|
assert len(b2["items"]) == 1
|
|
assert b2["items"][0]["msg_id"] == "m2"
|
|
|
|
|
|
async def test_group_chat_filter_by_room_id(make_client):
|
|
"""``?room_id=room_x`` -> only the group message."""
|
|
async def seed(session: AsyncSession) -> None:
|
|
await _seed(session)
|
|
session.add(
|
|
Message(
|
|
msg_id="m4",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id=None,
|
|
room_id="room_x",
|
|
text="group hi",
|
|
timestamp=4000,
|
|
type="text",
|
|
type_raw=1,
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
client = await make_client(seed)
|
|
r = await client.get("/api/messages", params={"room_id": "room_x"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["total"] == 1
|
|
assert body["items"][0]["msg_id"] == "m4"
|
|
|
|
|
|
async def test_media_fields_serialize(make_client):
|
|
"""Image thumbnail/HD URLs, voice duration/transcription, md5, room_id and
|
|
sender avatar round-trip through MessageOut so the timeline can render
|
|
media + group messages without a second call.
|
|
"""
|
|
|
|
async def seed(session: AsyncSession) -> None:
|
|
session.add_all(
|
|
[
|
|
Message(
|
|
msg_id="img1",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id="t_a",
|
|
room_id=None,
|
|
type="image",
|
|
type_raw=3,
|
|
timestamp=1000,
|
|
file_url="https://oss/full.jpg",
|
|
file_th_url="https://oss/th.jpg",
|
|
file_hd_url="https://oss/hd.jpg",
|
|
md5="abc123",
|
|
sender_name="Alice",
|
|
sender_head_url="https://oss/alice.jpg",
|
|
),
|
|
Message(
|
|
msg_id="voice1",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id=None,
|
|
room_id="room_x",
|
|
type="voice",
|
|
type_raw=34,
|
|
timestamp=2000,
|
|
duration=12,
|
|
transcription_text="hello there",
|
|
sender_name="Bob",
|
|
sender_head_url="https://oss/bob.jpg",
|
|
),
|
|
]
|
|
)
|
|
await session.commit()
|
|
|
|
client = await make_client(seed)
|
|
r = await client.get("/api/messages", params={"owner_account_id": "wx_owner_1"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["total"] == 2
|
|
by_id = {it["msg_id"]: it for it in body["items"]}
|
|
|
|
img = by_id["img1"]
|
|
assert img["file_th_url"] == "https://oss/th.jpg"
|
|
assert img["file_hd_url"] == "https://oss/hd.jpg"
|
|
assert img["md5"] == "abc123"
|
|
assert img["room_id"] is None
|
|
assert img["sender_head_url"] == "https://oss/alice.jpg"
|
|
|
|
voice = by_id["voice1"]
|
|
assert voice["duration"] == 12
|
|
assert voice["transcription_text"] == "hello there"
|
|
assert voice["room_id"] == "room_x"
|
|
assert voice["sender_head_url"] == "https://oss/bob.jpg"
|
|
|
|
|
|
async def test_media_fields_default_none_when_unset(make_client):
|
|
"""A text message with no media columns set serializes the new keys as None."""
|
|
|
|
async def seed(session: AsyncSession) -> None:
|
|
session.add(
|
|
Message(
|
|
msg_id="t1",
|
|
channel="wechat",
|
|
owner_account_id="wx_owner_1",
|
|
talker_id="t_a",
|
|
type="text",
|
|
type_raw=1,
|
|
timestamp=1000,
|
|
text="plain",
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
client = await make_client(seed)
|
|
r = await client.get("/api/messages")
|
|
assert r.status_code == 200
|
|
it = r.json()["items"][0]
|
|
for key in (
|
|
"file_th_url",
|
|
"file_hd_url",
|
|
"duration",
|
|
"transcription_text",
|
|
"md5",
|
|
"room_id",
|
|
"sender_head_url",
|
|
):
|
|
assert key in it
|
|
assert it[key] is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# msg_type / q 过滤(Task 4: 类型归一化扩展 + API 过滤, 对齐云客 10 类下拉)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def _seed_types(session: AsyncSession) -> None:
|
|
"""text / image / image_legacy / red_packet / transfer 各一条 + 关键词行。"""
|
|
session.add_all(
|
|
[
|
|
Message(
|
|
msg_id="ty_text", channel="wechat", owner_account_id="wx_owner_1",
|
|
talker_id="t_a", text="报价 135800 元", timestamp=1000, is_send=True,
|
|
type="text", type_raw=1,
|
|
),
|
|
Message(
|
|
msg_id="ty_img", channel="wechat", owner_account_id="wx_owner_1",
|
|
talker_id="t_a", text="", timestamp=2000, is_send=False,
|
|
type="image", type_raw=2,
|
|
),
|
|
Message(
|
|
msg_id="ty_img_legacy", channel="wechat", owner_account_id="wx_owner_1",
|
|
talker_id="t_a", text="", timestamp=2500, is_send=False,
|
|
type="image_legacy", type_raw=3,
|
|
),
|
|
Message(
|
|
msg_id="ty_red", channel="wechat", owner_account_id="wx_owner_1",
|
|
talker_id="t_a", text="", timestamp=3000, is_send=True,
|
|
type="red_packet", type_raw=5,
|
|
),
|
|
Message(
|
|
msg_id="ty_transfer", channel="wechat", owner_account_id="wx_owner_1",
|
|
talker_id="t_a", text="发送转账：￥100.00", timestamp=4000, is_send=True,
|
|
type="transfer", type_raw=11,
|
|
),
|
|
Message(
|
|
msg_id="ty_text_2", channel="wechat", owner_account_id="wx_owner_1",
|
|
talker_id="t_a", text="含 100% 通配符字面量", timestamp=5000, is_send=True,
|
|
type="text", type_raw=1,
|
|
),
|
|
]
|
|
)
|
|
await session.commit()
|
|
|
|
|
|
async def test_msg_type_filter_text(make_client):
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"msg_type": "text"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert {it["msg_id"] for it in body["items"]} == {"ty_text", "ty_text_2"}
|
|
|
|
|
|
async def test_msg_type_filter_image_includes_legacy(make_client):
|
|
"""msg_type=image 覆盖 image + image_legacy(过渡期兼容)。"""
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"msg_type": "image"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert {it["msg_id"] for it in body["items"]} == {"ty_img", "ty_img_legacy"}
|
|
|
|
|
|
async def test_msg_type_filter_red_packet_only(make_client):
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"msg_type": "red_packet"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert [it["msg_id"] for it in body["items"]] == ["ty_red"]
|
|
assert body["total"] == 1
|
|
|
|
|
|
async def test_msg_type_filter_no_rows_returns_empty(make_client):
|
|
"""voice_call 等暂无样本类型: 过滤语法有效, 查空是数据现实。"""
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"msg_type": "voice_call"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["items"] == []
|
|
assert body["total"] == 0
|
|
|
|
|
|
async def test_msg_type_unknown_rejected_422(make_client):
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"msg_type": "bogus"})
|
|
assert r.status_code == 422
|
|
|
|
|
|
async def test_q_substring_filter(make_client):
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"q": "报价"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert [it["msg_id"] for it in body["items"]] == ["ty_text"]
|
|
assert body["total"] == 1
|
|
|
|
|
|
async def test_q_escapes_like_wildcards(make_client):
|
|
"""q 里的 %/_ 按字面匹配, 不当通配符。"""
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"q": "100%"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
# 只命中字面含 "100%" 的行; 不因 % 通配漏扩到 "￥100.00" 之外的行
|
|
assert [it["msg_id"] for it in body["items"]] == ["ty_text_2"]
|
|
r2 = await client.get("/api/messages", params={"q": "100"})
|
|
ids2 = {it["msg_id"] for it in r2.json()["items"]}
|
|
assert ids2 == {"ty_transfer", "ty_text_2"}
|
|
|
|
|
|
async def test_msg_type_and_q_combine(make_client):
|
|
client = await make_client(_seed_types)
|
|
r = await client.get("/api/messages", params={"msg_type": "transfer", "q": "转账"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert [it["msg_id"] for it in body["items"]] == ["ty_transfer"]
|