feat(search): FTS5 trigram全文索引(中文搜索3.95s→0.003s,1300倍)

- message_fts 外部内容虚拟表+trigram分词(中文子串可MATCH,无需分词库) - 3个触发器随ORM upsert自动同步索引(抓取器零改动) - 搜索路由:≥3字走FTS MATCH,<3字/MySQL回退LIKE - 实测:搜「大众途安」56条,LIKE全扫3.95s→FTS 0.003s;索引+300MB - 7个新测试(建表幂等/触发器同步/中文匹配/短查询回退/未建索引回退),426绿 Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-21 15:08:26 +08:00
parent 621c2cc929
commit e8c77412fa
4 changed files with 380 additions and 1 deletions
@@ -0,0 +1,170 @@
+"""FTS5 message-search index — Chinese substring search at index speed.
+
+The plain ``LIKE '%q%'`` scan over 675k messages costs ~4s (measured
+2026-08-21). SQLite's FTS5 with the **trigram** tokenizer matches Chinese
+substrings (no word segmentation needed) using an index — same results,
+milliseconds instead of seconds.
+
+Design:
+
+* ``message_fts`` is an **external-content** FTS5 table over ``message``
+  (``content='message', content_rowid='rowid'``) so the text isn't stored
+  twice; only the trigram index adds size.
+* Three triggers keep the index in sync with ORM upserts transparently —
+  no scraper changes needed.
+* MySQL (prod target) has no FTS5: ``fts_available`` returns False there
+  and the search router falls back to the LIKE scan (``FULLTEXT`` index is
+  the MySQL-side upgrade path, deferred).
+* Trigram MATCH needs >= 3 characters; shorter queries fall back to LIKE
+  at the call site (see :mod:`app.api.routers.search`).
+"""
+
+from __future__ import annotations
+
+import logging
+
+import sqlalchemy as sa
+from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
+
+logger = logging.getLogger(__name__)
+
+# DDL — idempotent (IF NOT EXISTS). Triggers mirror the classic FTS5
+# external-content pattern from the SQLite docs.
+_FTS_DDL = [
+    # trigram 分词器:中文子串可 MATCH(≥3字),无需分词库
+    """
+    CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5(
+        text,
+        content='message',
+        content_rowid='rowid',
+        tokenize='trigram'
+    )
+    """,
+    """
+    CREATE TRIGGER IF NOT EXISTS message_fts_ai AFTER INSERT ON message BEGIN
+        INSERT INTO message_fts(rowid, text) VALUES (new.rowid, new.text);
+    END
+    """,
+    """
+    CREATE TRIGGER IF NOT EXISTS message_fts_au AFTER UPDATE ON message BEGIN
+        INSERT INTO message_fts(message_fts, rowid, text)
+            VALUES ('delete', old.rowid, old.text);
+        INSERT INTO message_fts(rowid, text) VALUES (new.rowid, new.text);
+    END
+    """,
+    """
+    CREATE TRIGGER IF NOT EXISTS message_fts_ad AFTER DELETE ON message BEGIN
+        INSERT INTO message_fts(message_fts, rowid, text)
+            VALUES ('delete', old.rowid, old.text);
+    END
+    """,
+]
+
+
+async def ensure_fts(holder) -> None:
+    """Create the FTS5 virtual table + sync triggers (idempotent, no-op on MySQL).
+
+    Accepts an ``AsyncSession`` / ``AsyncConnection`` (anything with async
+    ``execute``) or an ``AsyncEngine``. Raises whatever the dialect raises
+    for unsupported DDL — callers on non-SQLite engines should catch and
+    treat FTS as unavailable.
+    """
+    if hasattr(holder, "execute"):
+        for ddl in _FTS_DDL:
+            await holder.execute(sa.text(ddl))
+    else:  # AsyncEngine
+        async with holder.begin() as conn:
+            for ddl in _FTS_DDL:
+                await conn.execute(sa.text(ddl))
+
+
+async def rebuild_fts(holder) -> int:
+    """(Re)populate the index from the current ``message`` rows.
+
+    Returns the rowcount. One-time backfill for existing data — triggers
+    only cover writes made after ``ensure_fts``.
+    """
+    result = await holder.execute(
+        sa.text("INSERT INTO message_fts(message_fts) VALUES('rebuild')")
+    )
+    return result.rowcount or 0
+
+
+async def fts_available(holder) -> bool:
+    """True iff the ``message_fts`` virtual table exists (session/conn/engine)."""
+    try:
+        if hasattr(holder, "execute"):
+            row = (
+                await holder.execute(
+                    sa.text(
+                        "SELECT count(*) FROM sqlite_master "
+                        "WHERE type='table' AND name='message_fts'"
+                    )
+                )
+            ).scalar_one()
+            return bool(row)
+        async with holder.connect() as conn:
+            row = (
+                await conn.execute(
+                    sa.text(
+                        "SELECT count(*) FROM sqlite_master "
+                        "WHERE type='table' AND name='message_fts'"
+                    )
+                )
+            ).scalar_one()
+            return bool(row)
+    except Exception:
+        # MySQL 等 — sqlite_master 不存在
+        return False
+
+
+async def fts_match_rows(
+    engine: AsyncEngine,
+    query: str,
+    *,
+    limit: int = 200,
+) -> list:
+    """MATCH ``query`` against the trigram index, newest first.
+
+    Returns Message ORM rows (joined back from rowid via raw SQL — FTS
+    virtual tables can't join into ORM ``select()`` directly). Caller is
+    responsible for ensuring the index exists (``fts_available``) and
+    ``len(query) >= 3``.
+    """
+    from sqlalchemy.ext.asyncio import async_sessionmaker
+
+    from app.db.models import Message
+
+    maker = async_sessionmaker(engine, expire_on_commit=False)
+    async with maker() as session:
+        # 先取 MATCH 命中的 rowid(索引快),再按主键取 ORM 行
+        rowids = (
+            await session.execute(
+                sa.text(
+                    "SELECT rowid FROM message_fts WHERE message_fts MATCH :q "
+                    "ORDER BY rowid LIMIT :lim"
+                ),
+                {"q": f'"{query}"', "lim": limit * 2},
+            )
+        ).scalars().all()
+        if not rowids:
+            return []
+        # rowid → msg_id 再取行(消息表 rowid 与 ORM 无直接映射)
+        rows = (
+            await session.execute(
+                sa.text(
+                    "SELECT msg_id FROM message WHERE rowid IN :rids"
+                ).bindparams(sa.bindparam("rids", expanding=True)),
+                {"rids": list(rowids)},
+            )
+        ).scalars().all()
+        if not rows:
+            return []
+        stmt = (
+            sa.select(Message)
+            .where(Message.msg_id.in_(rows))
+            .order_by(sa.nullslast(Message.timestamp.desc()))
+            .limit(limit)
+        )
+        result = await session.execute(stmt)
+        return result.scalars().all()