feat(search): FTS5 trigram全文索引(中文搜索3.95s→0.003s,1300倍)
- message_fts 外部内容虚拟表+trigram分词(中文子串可MATCH,无需分词库) - 3个触发器随ORM upsert自动同步索引(抓取器零改动) - 搜索路由:≥3字走FTS MATCH,<3字/MySQL回退LIKE - 实测:搜「大众途安」56条,LIKE全扫3.95s→FTS 0.003s;索引+300MB - 7个新测试(建表幂等/触发器同步/中文匹配/短查询回退/未建索引回退),426绿 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -57,7 +57,91 @@ async def _search_messages(
|
||||
start: int | None,
|
||||
end: int | None,
|
||||
) -> Page[MessageOut]:
|
||||
"""Run the message search. Caller validates ``q``."""
|
||||
"""Run the message search. Caller validates ``q``.
|
||||
|
||||
**FTS 加速(2026-08-21)**: 建了 ``message_fts``(trigram) 索引且查询
|
||||
≥3 字时走 MATCH(毫秒级);否则回退 LIKE 全扫(675k 行约 4s)。
|
||||
两路径结果一致(FTS join 回 message 表后套同样的过滤/排序)。
|
||||
"""
|
||||
from app.db.fts import fts_available
|
||||
|
||||
use_fts = len(q) >= 3 and await fts_available(session)
|
||||
if use_fts:
|
||||
return await _search_messages_fts(session, q, page, page_size, channel, start, end)
|
||||
return await _search_messages_like(session, q, page, page_size, channel, start, end)
|
||||
|
||||
|
||||
async def _search_messages_fts(
|
||||
session: AsyncSession,
|
||||
q: str,
|
||||
page: int,
|
||||
page_size: int,
|
||||
channel: str | None,
|
||||
start: int | None,
|
||||
end: int | None,
|
||||
) -> Page[MessageOut]:
|
||||
"""FTS trigram MATCH 路径 — join 回 message 表套过滤/排序/分页。"""
|
||||
conds = ['message_fts MATCH :fts_q']
|
||||
params: dict[str, Any] = {"fts_q": f'"{q}"'}
|
||||
if channel is not None:
|
||||
conds.append("m.channel = :ch")
|
||||
params["ch"] = channel
|
||||
if start is not None:
|
||||
conds.append("m.timestamp >= :s")
|
||||
params["s"] = start
|
||||
if end is not None:
|
||||
conds.append("m.timestamp <= :e")
|
||||
params["e"] = end
|
||||
where = " AND ".join(conds)
|
||||
|
||||
total = (
|
||||
await session.execute(
|
||||
sa.text(
|
||||
f"SELECT count(*) FROM message m "
|
||||
f"JOIN message_fts f ON m.rowid = f.rowid WHERE {where}"
|
||||
),
|
||||
params,
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
id_rows = (
|
||||
await session.execute(
|
||||
sa.text(
|
||||
f"SELECT m.msg_id FROM message m "
|
||||
f"JOIN message_fts f ON m.rowid = f.rowid WHERE {where} "
|
||||
f"ORDER BY m.timestamp DESC, m.msg_id ASC "
|
||||
f"LIMIT :lim OFFSET :off"
|
||||
),
|
||||
{**params, "lim": page_size, "off": (page - 1) * page_size},
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
items: list[MessageOut] = []
|
||||
if id_rows:
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Message).where(Message.msg_id.in_(list(id_rows)))
|
||||
)
|
||||
).scalars().all()
|
||||
by_id = {r.msg_id: r for r in rows}
|
||||
items = [
|
||||
MessageOut.model_validate(by_id[mid])
|
||||
for mid in id_rows
|
||||
if mid in by_id
|
||||
]
|
||||
return Page(items=items, total=total, page=page, page_size=page_size)
|
||||
|
||||
|
||||
async def _search_messages_like(
|
||||
session: AsyncSession,
|
||||
q: str,
|
||||
page: int,
|
||||
page_size: int,
|
||||
channel: str | None,
|
||||
start: int | None,
|
||||
end: int | None,
|
||||
) -> Page[MessageOut]:
|
||||
"""LIKE 全扫路径(未建 FTS 索引 / 查询 <3 字 / MySQL)。"""
|
||||
conditions: list[Any] = [Message.text.like(f"%{q}%")]
|
||||
if channel is not None:
|
||||
conditions.append(Message.channel == channel)
|
||||
|
||||
Reference in New Issue
Block a user