浅念
abc97fe217
MySQL LIKE 全扫698k行13.5s撞axios 15s超时→前端ERR_ABORTED; MATCH...AGAINST(ngram)毫秒级;本地C盘不足索引待服务器建 (索引SQL: CREATE FULLTEXT INDEX ft_msg_text ON message(text) WITH PARSER ngram) Co-Authored-By: Claude <noreply@anthropic.com>
310 lines
10 KiB
Python
310 lines
10 KiB
Python
"""GET /api/search — full-text search over messages + customers.
|
|
|
|
Read-only ``LIKE '%q%'`` search across two mirrored tables:
|
|
|
|
* **Messages** — matches against ``message.text``. Filters: ``channel``,
|
|
``start``/``end`` (ms-epoch window compared against ``timestamp``). Ordered
|
|
by ``timestamp desc`` so the most recent matches surface first.
|
|
* **Customers** — matches against any of ``friend_nickname`` / ``real_name`` /
|
|
``phone`` / ``wechat_phone``. Filter: ``channel``. Ordered by
|
|
``last_chat_time desc`` (nulls last).
|
|
|
|
The ``type`` selector picks which section(s) to run:
|
|
|
|
* ``messages`` -> ``{"messages": Page[MessageOut]}``
|
|
* ``customers`` -> ``{"customers": Page[CustomerOut]}``
|
|
* ``all`` (default) -> ``{"messages": Page[MessageOut], "customers":
|
|
Page[CustomerOut]}``; both run with the supplied ``page`` / ``page_size``.
|
|
|
|
No external search dependencies (no FTS index, no Elasticsearch) — this is a
|
|
plain SQL ``LIKE`` scan over the stored mirror data. Sufficient for the
|
|
data-mirror's typical row counts and the same portability rules as the rest
|
|
of the API (MySQL in prod, in-memory SQLite in tests).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Annotated, Any, Literal
|
|
|
|
import sqlalchemy as sa
|
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.api.deps import get_session
|
|
from app.api.schemas import CustomerOut, MessageOut, Page
|
|
from app.db.models import Customer, Message
|
|
|
|
router = APIRouter(prefix="/api", tags=["search"])
|
|
|
|
SessionDep = Annotated[AsyncSession, Depends(get_session)]
|
|
PageParam = Annotated[int, Query(ge=1)]
|
|
PageSizeParam = Annotated[int, Query(ge=1, le=100)]
|
|
SearchType = Annotated[Literal["messages", "customers", "all"], Query()]
|
|
|
|
# Response shape: ``{"messages": Page[MessageOut], "customers": Page[CustomerOut]}``
|
|
# where each requested section is present and unrequested sections are omitted
|
|
# entirely (not serialized as null). Returning a plain dict keeps the key set
|
|
# explicit so the frontend can branch on ``section in payload``.
|
|
SearchPayload = dict[str, Page[MessageOut] | Page[CustomerOut]]
|
|
|
|
|
|
async def _search_messages(
|
|
session: AsyncSession,
|
|
q: str,
|
|
page: int,
|
|
page_size: int,
|
|
channel: str | None,
|
|
start: int | None,
|
|
end: int | None,
|
|
) -> Page[MessageOut]:
|
|
"""Run the message search. Caller validates ``q``.
|
|
|
|
**FTS 加速(2026-08-21)**: 建了 ``message_fts``(trigram) 索引且查询
|
|
≥3 字时走 MATCH(毫秒级);否则回退 LIKE 全扫(675k 行约 4s)。
|
|
两路径结果一致(FTS join 回 message 表后套同样的过滤/排序)。
|
|
"""
|
|
from app.db.fts import fts_available
|
|
|
|
if session.bind is not None and session.bind.dialect.name == "mysql":
|
|
# MySQL: ngram FULLTEXT 索引(LIKE 全扫 698k 行要 13.5s,2026-08-22 实测
|
|
# 撞 axios 15s 超时前端 ERR_ABORTED);索引不存在时自动回退 LIKE
|
|
return await _search_messages_mysql(
|
|
session, q, page, page_size, channel, start, end
|
|
)
|
|
use_fts = len(q) >= 3 and await fts_available(session)
|
|
if use_fts:
|
|
return await _search_messages_fts(session, q, page, page_size, channel, start, end)
|
|
return await _search_messages_like(session, q, page, page_size, channel, start, end)
|
|
|
|
|
|
async def _search_messages_fts(
|
|
session: AsyncSession,
|
|
q: str,
|
|
page: int,
|
|
page_size: int,
|
|
channel: str | None,
|
|
start: int | None,
|
|
end: int | None,
|
|
) -> Page[MessageOut]:
|
|
"""FTS trigram MATCH 路径 — join 回 message 表套过滤/排序/分页。"""
|
|
conds = ['message_fts MATCH :fts_q']
|
|
params: dict[str, Any] = {"fts_q": f'"{q}"'}
|
|
if channel is not None:
|
|
conds.append("m.channel = :ch")
|
|
params["ch"] = channel
|
|
if start is not None:
|
|
conds.append("m.timestamp >= :s")
|
|
params["s"] = start
|
|
if end is not None:
|
|
conds.append("m.timestamp <= :e")
|
|
params["e"] = end
|
|
where = " AND ".join(conds)
|
|
|
|
total = (
|
|
await session.execute(
|
|
sa.text(
|
|
f"SELECT count(*) FROM message m "
|
|
f"JOIN message_fts f ON m.rowid = f.rowid WHERE {where}"
|
|
),
|
|
params,
|
|
)
|
|
).scalar_one()
|
|
|
|
id_rows = (
|
|
await session.execute(
|
|
sa.text(
|
|
f"SELECT m.msg_id FROM message m "
|
|
f"JOIN message_fts f ON m.rowid = f.rowid WHERE {where} "
|
|
f"ORDER BY m.timestamp DESC, m.msg_id ASC "
|
|
f"LIMIT :lim OFFSET :off"
|
|
),
|
|
{**params, "lim": page_size, "off": (page - 1) * page_size},
|
|
)
|
|
).scalars().all()
|
|
|
|
items: list[MessageOut] = []
|
|
if id_rows:
|
|
rows = (
|
|
await session.execute(
|
|
sa.select(Message).where(Message.msg_id.in_(list(id_rows)))
|
|
)
|
|
).scalars().all()
|
|
by_id = {r.msg_id: r for r in rows}
|
|
items = [
|
|
MessageOut.model_validate(by_id[mid])
|
|
for mid in id_rows
|
|
if mid in by_id
|
|
]
|
|
return Page(items=items, total=total, page=page, page_size=page_size)
|
|
|
|
|
|
async def _search_messages_mysql(
|
|
session: AsyncSession,
|
|
q: str,
|
|
page: int,
|
|
page_size: int,
|
|
channel: str | None,
|
|
start: int | None,
|
|
end: int | None,
|
|
) -> Page[MessageOut]:
|
|
"""MySQL ngram FULLTEXT MATCH 路径;索引缺失自动回退 LIKE。"""
|
|
conds = ["MATCH(m.text) AGAINST(:ftq IN NATURAL LANGUAGE MODE)"]
|
|
params: dict[str, Any] = {"ftq": q}
|
|
if channel is not None:
|
|
conds.append("m.channel = :ch")
|
|
params["ch"] = channel
|
|
if start is not None:
|
|
conds.append("m.timestamp >= :s")
|
|
params["s"] = start
|
|
if end is not None:
|
|
conds.append("m.timestamp <= :e")
|
|
params["e"] = end
|
|
where = " AND ".join(conds)
|
|
|
|
try:
|
|
total = (
|
|
await session.execute(
|
|
sa.text(f"SELECT count(*) FROM message m WHERE {where}"), params
|
|
)
|
|
).scalar_one()
|
|
id_rows = (
|
|
await session.execute(
|
|
sa.text(
|
|
f"SELECT m.msg_id FROM message m WHERE {where} "
|
|
f"ORDER BY m.timestamp DESC, m.msg_id ASC LIMIT :lim OFFSET :off"
|
|
),
|
|
{**params, "lim": page_size, "off": (page - 1) * page_size},
|
|
)
|
|
).scalars().all()
|
|
except Exception:
|
|
# 索引不存在(部署早期) → 回退 LIKE
|
|
return await _search_messages_like(session, q, page, page_size, channel, start, end)
|
|
|
|
items: list[MessageOut] = []
|
|
if id_rows:
|
|
rows = (
|
|
await session.execute(
|
|
sa.select(Message).where(Message.msg_id.in_(list(id_rows)))
|
|
)
|
|
).scalars().all()
|
|
by_id = {r.msg_id: r for r in rows}
|
|
items = [MessageOut.model_validate(by_id[mid]) for mid in id_rows if mid in by_id]
|
|
return Page(items=items, total=total, page=page, page_size=page_size)
|
|
|
|
|
|
async def _search_messages_like(
|
|
session: AsyncSession,
|
|
q: str,
|
|
page: int,
|
|
page_size: int,
|
|
channel: str | None,
|
|
start: int | None,
|
|
end: int | None,
|
|
) -> Page[MessageOut]:
|
|
"""LIKE 全扫路径(未建 FTS 索引 / 查询 <3 字 / MySQL)。"""
|
|
conditions: list[Any] = [Message.text.like(f"%{q}%")]
|
|
if channel is not None:
|
|
conditions.append(Message.channel == channel)
|
|
if start is not None:
|
|
conditions.append(Message.timestamp >= start)
|
|
if end is not None:
|
|
conditions.append(Message.timestamp <= end)
|
|
|
|
total = (
|
|
await session.execute(
|
|
sa.select(sa.func.count())
|
|
.select_from(Message)
|
|
.where(*conditions)
|
|
)
|
|
).scalar_one()
|
|
|
|
list_stmt = (
|
|
sa.select(Message)
|
|
.where(*conditions)
|
|
.order_by(
|
|
sa.nullslast(Message.timestamp.desc()),
|
|
Message.msg_id.asc(),
|
|
)
|
|
.limit(page_size)
|
|
.offset((page - 1) * page_size)
|
|
)
|
|
rows = (await session.execute(list_stmt)).scalars().all()
|
|
items = [MessageOut.model_validate(row) for row in rows]
|
|
return Page(items=items, total=total, page=page, page_size=page_size)
|
|
|
|
|
|
async def _search_customers(
|
|
session: AsyncSession,
|
|
q: str,
|
|
page: int,
|
|
page_size: int,
|
|
channel: str | None,
|
|
) -> Page[CustomerOut]:
|
|
"""Run the customer search. Caller validates ``q``."""
|
|
like = f"%{q}%"
|
|
text_conditions = [
|
|
Customer.friend_nickname.like(like),
|
|
Customer.real_name.like(like),
|
|
Customer.phone.like(like),
|
|
Customer.wechat_phone.like(like),
|
|
]
|
|
conditions: list[Any] = [sa.or_(*text_conditions)]
|
|
if channel is not None:
|
|
conditions.append(Customer.channel == channel)
|
|
|
|
total = (
|
|
await session.execute(
|
|
sa.select(sa.func.count())
|
|
.select_from(Customer)
|
|
.where(*conditions)
|
|
)
|
|
).scalar_one()
|
|
|
|
list_stmt = (
|
|
sa.select(Customer)
|
|
.where(*conditions)
|
|
.order_by(
|
|
sa.nullslast(Customer.last_chat_time.desc()),
|
|
Customer.friend_id.asc(),
|
|
)
|
|
.limit(page_size)
|
|
.offset((page - 1) * page_size)
|
|
)
|
|
rows = (await session.execute(list_stmt)).scalars().all()
|
|
items = [CustomerOut.model_validate(row) for row in rows]
|
|
return Page(items=items, total=total, page=page, page_size=page_size)
|
|
|
|
|
|
@router.get("/search")
|
|
async def search(
|
|
session: SessionDep,
|
|
q: Annotated[str | None, Query()] = None,
|
|
type: SearchType = "all",
|
|
channel: Annotated[str | None, Query()] = None,
|
|
start: Annotated[int | None, Query(ge=0)] = None,
|
|
end: Annotated[int | None, Query(ge=0)] = None,
|
|
page: PageParam = 1,
|
|
page_size: PageSizeParam = 20,
|
|
) -> SearchPayload:
|
|
"""Search messages (by ``text``) and/or customers (by name/phone).
|
|
|
|
``q`` is required — missing / empty / whitespace-only all map to HTTP 400
|
|
so callers see a consistent validation response rather than the framework
|
|
default 422. Only the requested section(s) are present in the payload;
|
|
unrequested ones are omitted entirely.
|
|
"""
|
|
keyword = (q or "").strip()
|
|
if not keyword:
|
|
raise HTTPException(status_code=400, detail="q must not be empty")
|
|
|
|
out: SearchPayload = {}
|
|
if type in ("messages", "all"):
|
|
out["messages"] = await _search_messages(
|
|
session, keyword, page, page_size, channel, start, end
|
|
)
|
|
if type in ("customers", "all"):
|
|
out["customers"] = await _search_customers(
|
|
session, keyword, page, page_size, channel
|
|
)
|
|
return out
|