Files
yunke/yunke-server/tests/test_api_messages.py
T
浅念 9d3cc8e76e fix(messages): 聊天记录大对话超时——DESC首屏+keyset游标+群聊直达索引
用户报:聊天记录多时「聊天记录加载失败:请求失败 (ECONNABORTED)」(2026-09-07)。
服务器 EXPLAIN ANALYZE 实测两级根因:
- 单聊 1 万条:ASC 末页 OFFSET 10486 逐条回表 2515ms(DESC LIMIT 15ms,167 倍);
- 群聊 16,290 条:查询刻意不带 owner(chat_group 去重后 owner 错位),现有
  (channel,owner,room,timestamp) 索引被 owner 挡住,退化到 room 单列索引+
  filter+filesort——count 4.7s/首页 5.1s/末页 10.7s;真凶群 53132846289@chatroom
  40,863 条,三段串行 ~40s 必撞前端 axios 15s 超时。

修法(不动共享 MySQL 配置,沿用既往修复模式):
1. 迁移 e6f7a8b9c0d1:message 补 (channel, room_id, timestamp) 索引,群查询
   seek+排序直达+覆盖 count;模型侧双声明防 autogenerate 反 DROP。
2. /api/messages 加 order=asc|desc(默认 asc 不变)与 before_ts/before_id
   keyset 游标(成对,单传 422)——排序列 (timestamp,msg_id) 全序,平手按
   msg_id 决胜,游标页恒 O(page_size)。
3. 前端 ChatMain/ChatTimeline 首屏改「order=desc 取最新 50 反转」单请求
   (原 page1 拿 total+深 OFFSET 末页两请求),「加载更多」改游标上翻;
   hasMoreOlder=已加载<总数。SessionPane 搜索等 order=asc 默认路径不受影响。

本地实测(37,532 条群):desc 首页 0.39s、游标上翻 0.24s,浏览器真机
渲染+上翻 150 条无错。测试 8 增(desc/游标/平手边界/422/窗口叠加/
前端流程不变式),全量 601 绿。

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-09-07 16:59:04 +08:00

638 lines
22 KiB
Python

"""Tests for GET /api/messages — chat timeline.
Seeds 3 messages (2 for talker t_a across times 1000/3000, 1 for talker t_b at
2000) and asserts filters (channel, owner, talker, time window) + ordering
(``timestamp asc``) + pagination.
"""
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Message
async def _seed(session: AsyncSession) -> None:
session.add_all(
[
Message(
msg_id="m1",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_a",
room_id=None,
text="hello",
timestamp=1000,
is_send=True,
type="text",
type_raw=1,
),
Message(
msg_id="m2",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_b",
room_id=None,
text="hi",
timestamp=2000,
is_send=False,
type="text",
type_raw=1,
),
Message(
msg_id="m3",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_a",
room_id=None,
text="bye",
timestamp=3000,
is_send=True,
type="text",
type_raw=1,
),
]
)
await session.commit()
async def test_default_page_returns_all_ordered_asc(make_client):
"""No filters -> 3 rows ordered by ``timestamp asc`` (timeline order)."""
client = await make_client(_seed)
r = await client.get("/api/messages")
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert len(body["items"]) == 3
assert [it["msg_id"] for it in body["items"]] == ["m1", "m2", "m3"]
for key in (
"msg_id",
"channel",
"owner_account_id",
"talker_id",
"room_id",
"is_send",
"type",
"type_raw",
"text",
"file_url",
"sender_name",
"sender_head_url",
"timestamp",
"raw_json",
):
assert key in body["items"][0]
async def test_raw_json_serializes_verbatim(make_client):
"""``raw_json`` round-trips the verbatim upstream message record.
The seeded messages omit ``raw_json`` (NULL); this test seeds one with a
populated blob and asserts every camelCase key is preserved verbatim while
the existing rows still serialize as ``null``.
"""
async def seed(session: AsyncSession) -> None:
await _seed(session)
session.add(
Message(
msg_id="m_raw",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_a",
room_id=None,
type="image",
type_raw=3,
timestamp=5000,
raw_json={
"msgSvrId": "m_raw",
"type": 3,
"file": "https://oss/full.jpg",
"fileTh": "https://oss/th.jpg",
"fileHd": "https://oss/hd.jpg",
"md5": "abc123",
"md5Th": "th123",
"md5Hd": "hd123",
"duration": 0,
"talker": "t_a",
"oriTalker": "t_a",
"name": "Alice",
"headUrl": "https://oss/alice.jpg",
"wechatId": "wx_owner_1",
"roomid": "",
"isSend": 1,
"paymsgid": "",
"referMsg": {"text": "原文"},
},
)
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/messages", params={"talker_id": "t_a"})
assert r.status_code == 200
by_id = {it["msg_id"]: it for it in r.json()["items"]}
raw = by_id["m_raw"]["raw_json"]
assert raw == {
"msgSvrId": "m_raw",
"type": 3,
"file": "https://oss/full.jpg",
"fileTh": "https://oss/th.jpg",
"fileHd": "https://oss/hd.jpg",
"md5": "abc123",
"md5Th": "th123",
"md5Hd": "hd123",
"duration": 0,
"talker": "t_a",
"oriTalker": "t_a",
"name": "Alice",
"headUrl": "https://oss/alice.jpg",
"wechatId": "wx_owner_1",
"roomid": "",
"isSend": 1,
"paymsgid": "",
"referMsg": {"text": "原文"},
}
# Seeded messages without raw_json serialize as JSON null.
assert by_id["m1"]["raw_json"] is None
async def test_filter_by_talker_id(make_client):
"""``?talker_id=t_a`` -> 2 messages for that 1:1 conversation."""
client = await make_client(_seed)
r = await client.get("/api/messages", params={"talker_id": "t_a"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["msg_id"] for it in body["items"]} == {"m1", "m3"}
async def test_filter_by_owner_account_id(make_client):
"""``?owner_account_id=wx_owner_1`` -> all 3 (all owned by this account)."""
client = await make_client(_seed)
r = await client.get("/api/messages", params={"owner_account_id": "wx_owner_1"})
assert r.status_code == 200
assert r.json()["total"] == 3
async def test_filter_by_owner_account_id_no_match(make_client):
"""A non-existent owner account -> 0 rows."""
client = await make_client(_seed)
r = await client.get("/api/messages", params={"owner_account_id": "nobody"})
assert r.status_code == 200
assert r.json()["total"] == 0
async def test_time_window_filter(make_client):
"""``?start=1500&end=2500`` -> only m2 (timestamp 2000)."""
client = await make_client(_seed)
r = await client.get(
"/api/messages", params={"start": 1500, "end": 2500}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["msg_id"] == "m2"
async def test_time_window_combined_with_talker(make_client):
"""``talker_id=t_a & start=1500`` -> only m3 (t_a's later message)."""
client = await make_client(_seed)
r = await client.get(
"/api/messages", params={"talker_id": "t_a", "start": 1500}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["msg_id"] == "m3"
async def test_channel_filter(make_client):
"""``?channel=whatsapp`` -> 0 (all seeded are wechat)."""
client = await make_client(_seed)
r = await client.get("/api/messages", params={"channel": "whatsapp"})
assert r.status_code == 200
assert r.json()["total"] == 0
async def test_pagination_page_size_one(make_client):
"""``page_size=1&page=1`` -> 1 item, total still 3; page 2 -> next item."""
client = await make_client(_seed)
r1 = await client.get(
"/api/messages", params={"page": 1, "page_size": 1}
)
b1 = r1.json()
assert b1["total"] == 3
assert len(b1["items"]) == 1
assert b1["items"][0]["msg_id"] == "m1"
r2 = await client.get(
"/api/messages", params={"page": 2, "page_size": 1}
)
b2 = r2.json()
assert len(b2["items"]) == 1
assert b2["items"][0]["msg_id"] == "m2"
async def test_group_chat_filter_by_room_id(make_client):
"""``?room_id=room_x`` -> only the group message."""
async def seed(session: AsyncSession) -> None:
await _seed(session)
session.add(
Message(
msg_id="m4",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id=None,
room_id="room_x",
text="group hi",
timestamp=4000,
type="text",
type_raw=1,
)
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/messages", params={"room_id": "room_x"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["msg_id"] == "m4"
async def test_media_fields_serialize(make_client):
"""Image thumbnail/HD URLs, voice duration/transcription, md5, room_id and
sender avatar round-trip through MessageOut so the timeline can render
media + group messages without a second call.
"""
async def seed(session: AsyncSession) -> None:
session.add_all(
[
Message(
msg_id="img1",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_a",
room_id=None,
type="image",
type_raw=3,
timestamp=1000,
file_url="https://oss/full.jpg",
file_th_url="https://oss/th.jpg",
file_hd_url="https://oss/hd.jpg",
md5="abc123",
sender_name="Alice",
sender_head_url="https://oss/alice.jpg",
),
Message(
msg_id="voice1",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id=None,
room_id="room_x",
type="voice",
type_raw=34,
timestamp=2000,
duration=12,
transcription_text="hello there",
sender_name="Bob",
sender_head_url="https://oss/bob.jpg",
),
]
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/messages", params={"owner_account_id": "wx_owner_1"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
by_id = {it["msg_id"]: it for it in body["items"]}
img = by_id["img1"]
assert img["file_th_url"] == "https://oss/th.jpg"
assert img["file_hd_url"] == "https://oss/hd.jpg"
assert img["md5"] == "abc123"
assert img["room_id"] is None
assert img["sender_head_url"] == "https://oss/alice.jpg"
voice = by_id["voice1"]
assert voice["duration"] == 12
assert voice["transcription_text"] == "hello there"
assert voice["room_id"] == "room_x"
assert voice["sender_head_url"] == "https://oss/bob.jpg"
async def test_media_fields_default_none_when_unset(make_client):
"""A text message with no media columns set serializes the new keys as None."""
async def seed(session: AsyncSession) -> None:
session.add(
Message(
msg_id="t1",
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_a",
type="text",
type_raw=1,
timestamp=1000,
text="plain",
)
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/messages")
assert r.status_code == 200
it = r.json()["items"][0]
for key in (
"file_th_url",
"file_hd_url",
"duration",
"transcription_text",
"md5",
"room_id",
"sender_head_url",
):
assert key in it
assert it[key] is None
# ---------------------------------------------------------------------------
# msg_type / q 过滤(Task 4: 类型归一化扩展 + API 过滤, 对齐云客 10 类下拉)
# ---------------------------------------------------------------------------
async def _seed_types(session: AsyncSession) -> None:
"""text / image / image_legacy / red_packet / transfer 各一条 + 关键词行。"""
session.add_all(
[
Message(
msg_id="ty_text", channel="wechat", owner_account_id="wx_owner_1",
talker_id="t_a", text="报价 135800 元", timestamp=1000, is_send=True,
type="text", type_raw=1,
),
Message(
msg_id="ty_img", channel="wechat", owner_account_id="wx_owner_1",
talker_id="t_a", text="", timestamp=2000, is_send=False,
type="image", type_raw=2,
),
Message(
msg_id="ty_img_legacy", channel="wechat", owner_account_id="wx_owner_1",
talker_id="t_a", text="", timestamp=2500, is_send=False,
type="image_legacy", type_raw=3,
),
Message(
msg_id="ty_red", channel="wechat", owner_account_id="wx_owner_1",
talker_id="t_a", text="", timestamp=3000, is_send=True,
type="red_packet", type_raw=5,
),
Message(
msg_id="ty_transfer", channel="wechat", owner_account_id="wx_owner_1",
talker_id="t_a", text="发送转账:¥100.00", timestamp=4000, is_send=True,
type="transfer", type_raw=11,
),
Message(
msg_id="ty_text_2", channel="wechat", owner_account_id="wx_owner_1",
talker_id="t_a", text="含 100% 通配符字面量", timestamp=5000, is_send=True,
type="text", type_raw=1,
),
]
)
await session.commit()
async def test_msg_type_filter_text(make_client):
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"msg_type": "text"})
assert r.status_code == 200
body = r.json()
assert {it["msg_id"] for it in body["items"]} == {"ty_text", "ty_text_2"}
async def test_msg_type_filter_image_includes_legacy(make_client):
"""msg_type=image 覆盖 image + image_legacy(过渡期兼容)。"""
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"msg_type": "image"})
assert r.status_code == 200
body = r.json()
assert {it["msg_id"] for it in body["items"]} == {"ty_img", "ty_img_legacy"}
async def test_msg_type_filter_red_packet_only(make_client):
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"msg_type": "red_packet"})
assert r.status_code == 200
body = r.json()
assert [it["msg_id"] for it in body["items"]] == ["ty_red"]
assert body["total"] == 1
async def test_msg_type_filter_no_rows_returns_empty(make_client):
"""voice_call 等暂无样本类型: 过滤语法有效, 查空是数据现实。"""
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"msg_type": "voice_call"})
assert r.status_code == 200
body = r.json()
assert body["items"] == []
assert body["total"] == 0
async def test_msg_type_unknown_rejected_422(make_client):
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"msg_type": "bogus"})
assert r.status_code == 422
async def test_q_substring_filter(make_client):
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"q": "报价"})
assert r.status_code == 200
body = r.json()
assert [it["msg_id"] for it in body["items"]] == ["ty_text"]
assert body["total"] == 1
async def test_q_escapes_like_wildcards(make_client):
"""q 里的 %/_ 按字面匹配, 不当通配符。"""
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"q": "100%"})
assert r.status_code == 200
body = r.json()
# 只命中字面含 "100%" 的行; 不因 % 通配漏扩到 "¥100.00" 之外的行
assert [it["msg_id"] for it in body["items"]] == ["ty_text_2"]
r2 = await client.get("/api/messages", params={"q": "100"})
ids2 = {it["msg_id"] for it in r2.json()["items"]}
assert ids2 == {"ty_transfer", "ty_text_2"}
async def test_msg_type_and_q_combine(make_client):
client = await make_client(_seed_types)
r = await client.get("/api/messages", params={"msg_type": "transfer", "q": "转账"})
assert r.status_code == 200
body = r.json()
assert [it["msg_id"] for it in body["items"]] == ["ty_transfer"]
# ---------------------------------------------------------------------------
# order=desc + before_ts/before_id 游标(2026-09-07 深分页超时修复:
# 大对话 ASC 末页 OFFSET 数万行逐条回表 2.5s+,群聊 filesoort 10s+ 撞前端
# 15s 超时 ECONNABORTED。DESC 首页+游标上翻每页恒 O(page_size))
# ---------------------------------------------------------------------------
async def _seed_desc(session: AsyncSession) -> None:
"""t_a 会话 5 条,含两组同 timestamp 平手——验证 (timestamp, msg_id) 全序。
ASC 全序 = a1,a2,a3,a4,a5;DESC 全序 = a5,a4,a3,a2,a1。
"""
rows = [("a1", 1000), ("a2", 1000), ("a3", 2000), ("a4", 3000), ("a5", 3000)]
session.add_all(
[
Message(
msg_id=mid,
channel="wechat",
owner_account_id="wx_owner_1",
talker_id="t_a",
type="text",
type_raw=1,
is_send=True,
timestamp=ts,
)
for mid, ts in rows
]
)
await session.commit()
async def test_order_desc_returns_newest_first(make_client):
"""order=desc → 按 (timestamp, msg_id) 降序,平手时 msg_id 大者在前。"""
client = await make_client(_seed_desc)
r = await client.get(
"/api/messages", params={"talker_id": "t_a", "order": "desc"}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 5
assert [it["msg_id"] for it in body["items"]] == ["a5", "a4", "a3", "a2", "a1"]
async def test_order_desc_pagination_from_newest(make_client):
"""desc 的 page 从最新端数:page1=最新 slice(前端首屏即此页)。"""
client = await make_client(_seed_desc)
r1 = await client.get(
"/api/messages",
params={"talker_id": "t_a", "order": "desc", "page": 1, "page_size": 2},
)
assert [it["msg_id"] for it in r1.json()["items"]] == ["a5", "a4"]
r2 = await client.get(
"/api/messages",
params={"talker_id": "t_a", "order": "desc", "page": 2, "page_size": 2},
)
assert [it["msg_id"] for it in r2.json()["items"]] == ["a3", "a2"]
async def test_order_invalid_rejected_422(make_client):
client = await make_client(_seed_desc)
r = await client.get("/api/messages", params={"order": "bogus"})
assert r.status_code == 422
async def test_before_cursor_returns_preceding_slice(make_client):
"""before=(3000,a4) → 严格早于该游标的消息按 desc 取,平手边界不含游标本行。"""
client = await make_client(_seed_desc)
r = await client.get(
"/api/messages",
params={
"talker_id": "t_a",
"order": "desc",
"page_size": 2,
"before_ts": 3000,
"before_id": "a4",
},
)
assert r.status_code == 200
body = r.json()
# 游标后剩余 a1/a2/a3;desc 前 2 条
assert [it["msg_id"] for it in body["items"]] == ["a3", "a2"]
assert body["total"] == 3
async def test_before_cursor_tie_boundary_exclusive(make_client):
"""before=(3000,a5):同 ts=3000 的 a4 因 msg_id<a5 仍在界内。"""
client = await make_client(_seed_desc)
r = await client.get(
"/api/messages",
params={"talker_id": "t_a", "order": "desc", "before_ts": 3000, "before_id": "a5"},
)
body = r.json()
assert [it["msg_id"] for it in body["items"]] == ["a4", "a3", "a2", "a1"]
assert body["total"] == 4
async def test_before_requires_both_params_422(make_client):
"""游标两参必须成对:单传任一 422,防止静默忽略半个游标查错页。"""
client = await make_client(_seed_desc)
r1 = await client.get("/api/messages", params={"before_ts": 3000})
assert r1.status_code == 422
r2 = await client.get("/api/messages", params={"before_id": "a5"})
assert r2.status_code == 422
async def test_before_cursor_with_window_filter(make_client):
"""游标与 start/end 过滤叠加(loadOlder 带日期窗场景)。"""
client = await make_client(_seed_desc)
r = await client.get(
"/api/messages",
params={"talker_id": "t_a", "order": "desc", "start": 1500},
)
assert [it["msg_id"] for it in r.json()["items"]] == ["a5", "a4", "a3"]
r2 = await client.get(
"/api/messages",
params={
"talker_id": "t_a",
"order": "desc",
"start": 1500,
"before_ts": 2000,
"before_id": "a3",
},
)
body = r2.json()
assert body["items"] == []
assert body["total"] == 0
async def test_load_older_flow_reproduces_full_asc_order(make_client):
"""前端新流程端到端不变式:desc 首页反转 + 逐游标上翻反转,拼接 = 全量 ASC。
这正是 ChatMain/ChatTimeline 的首屏+「加载更多」序列——顺序若与旧
ASC 分页不一致,时间线就会错乱。
"""
client = await make_client(_seed_desc)
page_size = 2
async def fetch(**extra):
r = await client.get(
"/api/messages",
params={"talker_id": "t_a", "order": "desc", "page_size": page_size, **extra},
)
assert r.status_code == 200
return r.json()
displayed: list[str] = []
body = await fetch()
# 会话总数以首屏(无游标)响应为准;游标响应的 total 是「游标后剩余数」
conv_total = body["total"]
displayed = [it["msg_id"] for it in body["items"][::-1]]
while len(displayed) < conv_total:
oldest = body["items"][-1]
body = await fetch(before_ts=oldest["timestamp"], before_id=oldest["msg_id"])
assert body["items"], "游标还未走完 total 就断了"
displayed = [it["msg_id"] for it in body["items"][::-1]] + displayed
assert displayed == ["a1", "a2", "a3", "a4", "a5"]