feat(search): FTS5 trigram全文索引(中文搜索3.95s→0.003s,1300倍)

- message_fts 外部内容虚拟表+trigram分词(中文子串可MATCH,无需分词库)
- 3个触发器随ORM upsert自动同步索引(抓取器零改动)
- 搜索路由:≥3字走FTS MATCH,<3字/MySQL回退LIKE
- 实测:搜「大众途安」56条,LIKE全扫3.95s→FTS 0.003s;索引+300MB
- 7个新测试(建表幂等/触发器同步/中文匹配/短查询回退/未建索引回退),426绿

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-21 15:08:26 +08:00
parent 621c2cc929
commit e8c77412fa
4 changed files with 380 additions and 1 deletions
+85 -1
View File
@@ -57,7 +57,91 @@ async def _search_messages(
start: int | None,
end: int | None,
) -> Page[MessageOut]:
"""Run the message search. Caller validates ``q``."""
"""Run the message search. Caller validates ``q``.
**FTS 加速(2026-08-21)**: 建了 ``message_fts``(trigram) 索引且查询
≥3 字时走 MATCH(毫秒级);否则回退 LIKE 全扫(675k 行约 4s)。
两路径结果一致(FTS join 回 message 表后套同样的过滤/排序)。
"""
from app.db.fts import fts_available
use_fts = len(q) >= 3 and await fts_available(session)
if use_fts:
return await _search_messages_fts(session, q, page, page_size, channel, start, end)
return await _search_messages_like(session, q, page, page_size, channel, start, end)
async def _search_messages_fts(
session: AsyncSession,
q: str,
page: int,
page_size: int,
channel: str | None,
start: int | None,
end: int | None,
) -> Page[MessageOut]:
"""FTS trigram MATCH 路径 — join 回 message 表套过滤/排序/分页。"""
conds = ['message_fts MATCH :fts_q']
params: dict[str, Any] = {"fts_q": f'"{q}"'}
if channel is not None:
conds.append("m.channel = :ch")
params["ch"] = channel
if start is not None:
conds.append("m.timestamp >= :s")
params["s"] = start
if end is not None:
conds.append("m.timestamp <= :e")
params["e"] = end
where = " AND ".join(conds)
total = (
await session.execute(
sa.text(
f"SELECT count(*) FROM message m "
f"JOIN message_fts f ON m.rowid = f.rowid WHERE {where}"
),
params,
)
).scalar_one()
id_rows = (
await session.execute(
sa.text(
f"SELECT m.msg_id FROM message m "
f"JOIN message_fts f ON m.rowid = f.rowid WHERE {where} "
f"ORDER BY m.timestamp DESC, m.msg_id ASC "
f"LIMIT :lim OFFSET :off"
),
{**params, "lim": page_size, "off": (page - 1) * page_size},
)
).scalars().all()
items: list[MessageOut] = []
if id_rows:
rows = (
await session.execute(
sa.select(Message).where(Message.msg_id.in_(list(id_rows)))
)
).scalars().all()
by_id = {r.msg_id: r for r in rows}
items = [
MessageOut.model_validate(by_id[mid])
for mid in id_rows
if mid in by_id
]
return Page(items=items, total=total, page=page, page_size=page_size)
async def _search_messages_like(
session: AsyncSession,
q: str,
page: int,
page_size: int,
channel: str | None,
start: int | None,
end: int | None,
) -> Page[MessageOut]:
"""LIKE 全扫路径(未建 FTS 索引 / 查询 <3 字 / MySQL)。"""
conditions: list[Any] = [Message.text.like(f"%{q}%")]
if channel is not None:
conditions.append(Message.channel == channel)