feat(analytics): C1 敏感词库 CRUD+加词预览——防一加就淹没

Co-Authored-By: Claude Code <noreply@anthropic.com>

yunke-server/app/analytics/sensitive/__init__.py

@@ -0,0 +1,5 @@
+"""C1 敏感词子系统(spec §3.6)——words(词库)/scanner(扫描)/hits(处置)。
+
+内容源抽象:scanner 目前扫 message.text;预留 refer_msg 引文与 D4 ASR
+转写文本接入位(二/三批)。日志禁打词内容与命中正文。
+"""

yunke-server/app/analytics/sensitive/words.py

@@ -0,0 +1,130 @@
+"""词库服务——CRUD + 加词前命中量预览(防「一加就淹没」,spec P3)。"""
+from __future__ import annotations
+
+import sqlalchemy as sa
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Message, SensitiveWord
+
+PREVIEW_WARN_THRESHOLD = 500  # 预览超此数警示收窄(前端弹提示)
+
+
+async def list_words(session: AsyncSession) -> list[SensitiveWord]:
+    return (
+        (await session.execute(sa.select(SensitiveWord).order_by(SensitiveWord.id)))
+        .scalars().all()
+    )
+
+
+async def create_word(
+    session: AsyncSession, *, word: str, category: str | None = None,
+    min_len: int = 2, exclude_words: list[str] | None = None,
+) -> SensitiveWord:
+    word = (word or "").strip()
+    if not word:
+        raise ValueError("词不能为空")
+    dup = (
+        await session.execute(
+            sa.select(SensitiveWord).where(SensitiveWord.word == word)
+        )
+    ).scalar_one_or_none()
+    if dup is not None:
+        raise ValueError(f"词已存在: id={dup.id}")  # 词是用户输入,回显 id 可
+    obj = SensitiveWord(word=word, category=category, min_len=min_len,
+                        exclude_words=exclude_words)
+    session.add(obj)
+    await session.flush()
+    return obj
+
+
+async def update_word(session: AsyncSession, word_id: int, **fields) -> SensitiveWord:
+    obj = await session.get(SensitiveWord, word_id)
+    if obj is None:
+        raise LookupError("word not found")
+    for k in ("enabled", "category", "min_len", "exclude_words"):
+        if k in fields and fields[k] is not None:
+            setattr(obj, k, fields[k])
+    await session.flush()
+    return obj
+
+
+async def delete_word(session: AsyncSession, word_id: int) -> None:
+    """删词 = 级联删命中(SQLite FK 不强制,显式删;UI 二次确认)。"""
+    obj = await session.get(SensitiveWord, word_id)
+    if obj is None:
+        raise LookupError("word not found")
+    from app.db.models import SensitiveHit
+
+    await session.execute(
+        sa.delete(SensitiveHit).where(SensitiveHit.word_id == word_id)
+    )
+    await session.delete(obj)
+    await session.flush()
+
+
+async def preview_count(
+    session: AsyncSession, word: str, exclude_words: list[str] | None = None
+) -> int:
+    """加词前 COUNT——MySQL 走 ngram MATCH,其他方言 LIKE(测试)。
+
+    口径注:MySQL MATCH 路径不扣排除词(预览=上限,排除词在扫描时精确
+    生效);LIKE 路径(测试库)精确扣。前端文案「预计命中」体现上限语义。
+    """
+    word = (word or "").strip()
+    if not word:
+        return 0
+    conds = [Message.text.is_not(None), Message.text != ""]
+    dialect = session.bind.dialect.name if session.bind else ""
+    if dialect == "mysql" and len(word) >= 2:
+        conds.append(
+            sa.text("MATCH(message.text) AGAINST (:w IN BOOLEAN MODE)")
+            .bindparams(w=f'"{word}"')
+        )
+        stmt = sa.select(sa.func.count()).select_from(Message).where(*conds)
+        n = (await session.execute(stmt)).scalar_one()
+    else:
+        escaped = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+        like = f"%{escaped}%"
+        stmt = sa.select(Message.msg_id, Message.text).where(
+            Message.text.is_not(None), Message.text.like(like, escape="\\")
+        )
+        rows = (await session.execute(stmt)).all()
+        n = sum(1 for _mid, t in rows if not _hit_exclude(t, word, exclude_words))
+    return int(n)
+
+
+def _hit_exclude(text: str | None, word: str, exclude_words: list[str] | None) -> bool:
+    """命中行含任一排除词 → 视为误报跳过。"""
+    if not exclude_words or not text:
+        return False
+    return any(ex in text for ex in exclude_words)
+
+
+# 内置起步词库(spec §3.6):幂等补种(CLI sensitive-scan 首跑时调),上线后
+# 管理员在词库页自补行业词。
+STARTER_WORDS: list[tuple[str, str]] = [
+    ("私单", "飞单"),
+    ("飞单", "飞单"),
+    ("加私人微信", "私联"),
+    ("红包", "资金"),
+    ("转账", "资金"),
+    ("离职", "人员风险"),
+]
+
+
+async def ensure_starter_words(session: AsyncSession) -> int:
+    """缺则补(按 word 去重幂等),返回新种入数。"""
+    existing = set(
+        (
+            await session.execute(sa.select(SensitiveWord.word))
+        ).scalars().all()
+    )
+    n = 0
+    for word, category in STARTER_WORDS:
+        if word in existing:
+            continue
+        session.add(SensitiveWord(word=word, category=category))
+        n += 1
+    if n:
+        await session.flush()
+    return n

yunke-server/app/api/routers/analytics.py

@@ -4,10 +4,12 @@ from __future__ import annotations
 from typing import Annotated, Any, Literal
 
 from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.analytics import health as health_svc
 from app.analytics import metrics as metrics_svc
+from app.analytics.sensitive import words as words_svc
 from app.api.deps import get_session
 
 router = APIRouter(prefix="/api/analytics", tags=["analytics"])
@@ -69,3 +71,68 @@ async def unreplied_list(
     return await metrics_svc.unreplied(
         session, channel=channel, hours=hours, page=page, page_size=page_size
     )
+
+
+class WordIn(BaseModel):
+    word: str
+    category: str | None = None
+    min_len: int = 2
+    exclude_words: list[str] | None = None
+
+
+class WordPreviewIn(BaseModel):
+    word: str
+    exclude_words: list[str] | None = None
+
+
+@router.get("/sensitive/words")
+async def list_words(session: SessionDep) -> list[dict]:
+    """C1 词库清单(enabled=false 的词 scanner 跳过)。"""
+    return [
+        {"id": w.id, "word": w.word, "category": w.category, "enabled": w.enabled,
+         "min_len": w.min_len, "exclude_words": w.exclude_words or []}
+        for w in await words_svc.list_words(session)
+    ]
+
+
+@router.post("/sensitive/words")
+async def create_word(session: SessionDep, body: WordIn) -> dict:
+    """加词(重复 409)。词内容是敏感数据,错误信息只回 id 不回词。"""
+    try:
+        w = await words_svc.create_word(
+            session, word=body.word, category=body.category,
+            min_len=body.min_len, exclude_words=body.exclude_words,
+        )
+        await session.commit()
+        return {"id": w.id, "word": w.word, "enabled": w.enabled}
+    except ValueError as exc:
+        raise HTTPException(409, str(exc)) from exc
+
+
+@router.post("/sensitive/words/preview")
+async def preview_word(session: SessionDep, body: WordPreviewIn) -> dict:
+    """加词前命中量预览(count 为 MySQL 上限语义;warn=超阈值建议收窄)。"""
+    n = await words_svc.preview_count(session, body.word, body.exclude_words)
+    return {"count": n, "warn": n >= words_svc.PREVIEW_WARN_THRESHOLD}
+
+
+@router.patch("/sensitive/words/{word_id}")
+async def update_word(session: SessionDep, word_id: int, body: dict) -> dict:
+    """改词(启停/分类/排除词);字段 None=不改。"""
+    try:
+        w = await words_svc.update_word(session, word_id, **body)
+        await session.commit()
+        return {"id": w.id, "enabled": w.enabled, "category": w.category}
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+
+
+@router.delete("/sensitive/words/{word_id}")
+async def delete_word(session: SessionDep, word_id: int) -> dict:
+    """删词=级联删命中(UI 二次确认)。"""
+    try:
+        await words_svc.delete_word(session, word_id)
+        await session.commit()
+        return {"ok": True}
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc

yunke-server/tests/test_sensitive.py

@@ -0,0 +1,61 @@
+"""C1 词库 CRUD + 预览。"""
+from __future__ import annotations
+
+import datetime as dt
+
+from app.analytics.sensitive import words as words_svc
+from app.db.models import Message
+
+
+async def test_create_and_duplicate(make_client):
+    async def seed(session):
+        w = await words_svc.create_word(
+            session, word="飞单", category="私单", exclude_words=["飞单群"]
+        )
+        assert w.id is not None and w.enabled is True
+        try:
+            await words_svc.create_word(session, word="飞单")
+            raise AssertionError("should raise")
+        except ValueError:
+            pass
+
+    await make_client(seed)
+
+
+async def test_preview_count_and_excludes(make_client):
+    async def seed(session):
+        ts = 1756900000000
+        for i, text in enumerate(["今天聊飞单的事", "飞单群公告", "正常聊天"]):
+            session.add(Message(msg_id=f"m{i}", channel="wechat",
+                                owner_account_id="wx1", talker_id="t1",
+                                timestamp=ts + i, is_send=False, type="text",
+                                text=text, fetched_at=dt.datetime(2026, 9, 4)))
+        await session.commit()
+        n1 = await words_svc.preview_count(session, "飞单", None)
+        assert n1 == 2  # SQLite 测试库走 LIKE 路径
+        n2 = await words_svc.preview_count(session, "飞单", ["飞单群"])
+        assert n2 == 1  # 排除词剔除「飞单群公告」
+
+    await make_client(seed)
+
+
+async def test_api_words_crud(make_client):
+    async def seed(session):
+        ...
+
+    client = await make_client(seed)
+    r = await client.post("/api/analytics/sensitive/words",
+                          json={"word": "私单", "category": "飞单"})
+    assert r.status_code == 200 and r.json()["word"] == "私单"
+    r = await client.post("/api/analytics/sensitive/words",
+                          json={"word": "私单"})
+    assert r.status_code == 409
+    r = await client.post("/api/analytics/sensitive/words/preview",
+                          json={"word": "私单"})
+    assert r.status_code == 200 and r.json()["count"] == 0
+    wid = 1
+    r = await client.patch(f"/api/analytics/sensitive/words/{wid}",
+                           json={"enabled": False})
+    assert r.status_code == 200 and r.json()["enabled"] is False
+    r = await client.delete(f"/api/analytics/sensitive/words/{wid}")
+    assert r.status_code == 200