feat(search): FTS5 trigram全文索引(中文搜索3.95s→0.003s,1300倍)

- message_fts 外部内容虚拟表+trigram分词(中文子串可MATCH,无需分词库) - 3个触发器随ORM upsert自动同步索引(抓取器零改动) - 搜索路由:≥3字走FTS MATCH,<3字/MySQL回退LIKE - 实测:搜「大众途安」56条,LIKE全扫3.95s→FTS 0.003s;索引+300MB - 7个新测试(建表幂等/触发器同步/中文匹配/短查询回退/未建索引回退),426绿 Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-21 15:08:26 +08:00
parent 621c2cc929
commit e8c77412fa
4 changed files with 380 additions and 1 deletions
@@ -313,3 +313,45 @@ async def test_customer_pagination(make_client):
    # All seeded phone numbers contain "1" -> all 3.
    assert custs["total"] == 3
    assert len(custs["items"]) == 1
+
+
+# ---------------------------------------------------------------------------
+# FTS 加速路径(2026-08-21): 建了 message_fts 索引时搜索走 trigram MATCH,
+# 结果与 LIKE 一致;未建索引回退 LIKE(现有测试全是这条路径)。
+# ---------------------------------------------------------------------------
+
+async def test_search_uses_fts_when_available(make_client):
+    """建 FTS 索引后 /api/search 结果与 LIKE 等价(中文≥3字)。"""
+    from sqlalchemy.ext.asyncio import async_sessionmaker
+
+    from app.db.fts import ensure_fts
+
+    async def seed(session):
+        await ensure_fts(session)
+        session.add_all([
+            Message(msg_id="f1", channel="wechat", owner_account_id="o", talker_id="t", text="24年大众途安 简约白", timestamp=1000),
+            Message(msg_id="f2", channel="wechat", owner_account_id="o", talker_id="t", text="丰田汉兰达 2958", timestamp=2000),
+        ])
+        await session.commit()
+
+    client = await make_client(seed)
+    r = await client.get("/api/search", params={"q": "大众途安", "type": "messages"})
+    assert r.status_code == 200
+    msgs = r.json()["messages"]
+    assert msgs["total"] == 1
+    assert msgs["items"][0]["msg_id"] == "f1"
+
+
+async def test_search_fts_short_query_falls_back(make_client):
+    """2 字查询(低于 trigram 最小 3 字)回退 LIKE,结果一致。"""
+    from app.db.fts import ensure_fts
+
+    async def seed(session):
+        await ensure_fts(session)
+        session.add(Message(msg_id="s1", channel="wechat", owner_account_id="o", talker_id="t", text="汉兰达", timestamp=1))
+        await session.commit()
+
+    client = await make_client(seed)
+    r = await client.get("/api/search", params={"q": "汉兰", "type": "messages"})
+    assert r.status_code == 200
+    assert r.json()["messages"]["total"] == 1
@@ -0,0 +1,83 @@
+"""Tests for the FTS5 message-search index (app.db.fts).
+
+SQLite FTS5 trigram tokenizer gives Chinese substring matching at index
+speed instead of a 4s LIKE scan over 675k rows (2026-08-21). The module:
+
+* ``ensure_fts`` creates the ``message_fts`` virtual table + sync triggers
+  (idempotent; no-op when already present, tolerant of MySQL where FTS5
+  doesn't exist — callers fall back to LIKE).
+* ORM inserts/updates keep the index in sync via the triggers.
+* ``fts_available`` reports whether the index exists on this engine.
+"""
+
+from __future__ import annotations
+
+import pytest
+import sqlalchemy as sa
+
+from app.db.base import Base
+from app.db.fts import ensure_fts, fts_available, fts_match_rows
+from app.db.models import Message
+
+
+@pytest.fixture
+async def engine():
+    from sqlalchemy.ext.asyncio import create_async_engine
+
+    eng = create_async_engine("sqlite+aiosqlite:///:memory:")
+    async with eng.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    try:
+        yield eng
+    finally:
+        await eng.dispose()
+
+
+async def test_ensure_fts_creates_index_and_triggers(engine):
+    async with engine.begin() as conn:
+        await ensure_fts(conn)
+    assert await fts_available(engine) is True
+
+
+async def test_ensure_fts_idempotent(engine):
+    async with engine.begin() as conn:
+        await ensure_fts(conn)
+        await ensure_fts(conn)  # 二次调用不报错
+    assert await fts_available(engine) is True
+
+
+async def test_orm_insert_syncs_fts(engine):
+    """ORM 插入消息 → 触发器自动进 FTS 索引 → 能 MATCH 到。"""
+    from sqlalchemy.ext.asyncio import async_sessionmaker
+
+    async with engine.begin() as conn:
+        await ensure_fts(conn)
+    maker = async_sessionmaker(engine, expire_on_commit=False)
+
+    async with maker() as session:
+        session.add(Message(msg_id="m1", channel="wechat", owner_account_id="o", talker_id="t", text="24年大众途安 简约白", timestamp=1000))
+        session.add(Message(msg_id="m2", channel="wechat", owner_account_id="o", talker_id="t", text="丰田汉兰达", timestamp=2000))
+        await session.commit()
+
+    rows = await fts_match_rows(engine, "大众途安")
+    assert [r.msg_id for r in rows] == ["m1"]
+
+
+async def test_fts_match_chinese_substring(engine):
+    """trigram 中文子串匹配:搜「途安」(2字,LIKE 回退)与「大众途安」(4字,FTS)。"""
+    from sqlalchemy.ext.asyncio import async_sessionmaker
+
+    async with engine.begin() as conn:
+        await ensure_fts(conn)
+    maker = async_sessionmaker(engine, expire_on_commit=False)
+    async with maker() as session:
+        session.add(Message(msg_id="m1", channel="wechat", owner_account_id="o", talker_id="t", text="推荐24年大众途安L", timestamp=1000))
+        await session.commit()
+
+    rows = await fts_match_rows(engine, "大众途安")
+    assert len(rows) == 1
+
+
+async def test_fts_absent_on_fresh_db(engine):
+    """未 ensure 过的库 → fts_available False(搜索走 LIKE 回退)。"""
+    assert await fts_available(engine) is False