"""词库服务——CRUD + 加词前命中量预览(防「一加就淹没」,spec P3)。""" from __future__ import annotations import sqlalchemy as sa from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import Message, SensitiveWord PREVIEW_WARN_THRESHOLD = 500 # 预览超此数警示收窄(前端弹提示) async def list_words(session: AsyncSession) -> list[SensitiveWord]: return ( (await session.execute(sa.select(SensitiveWord).order_by(SensitiveWord.id))) .scalars().all() ) async def create_word( session: AsyncSession, *, word: str, category: str | None = None, min_len: int = 2, exclude_words: list[str] | None = None, ) -> SensitiveWord: word = (word or "").strip() if not word: raise ValueError("词不能为空") dup = ( await session.execute( sa.select(SensitiveWord).where(SensitiveWord.word == word) ) ).scalar_one_or_none() if dup is not None: raise ValueError(f"词已存在: id={dup.id}") # 词是用户输入,回显 id 可 obj = SensitiveWord(word=word, category=category, min_len=min_len, exclude_words=exclude_words) session.add(obj) await session.flush() return obj async def update_word(session: AsyncSession, word_id: int, **fields) -> SensitiveWord: obj = await session.get(SensitiveWord, word_id) if obj is None: raise LookupError("word not found") for k in ("enabled", "category", "min_len", "exclude_words"): if k in fields and fields[k] is not None: setattr(obj, k, fields[k]) await session.flush() return obj async def delete_word(session: AsyncSession, word_id: int) -> None: """删词 = 级联删命中(SQLite FK 不强制,显式删;UI 二次确认)。""" obj = await session.get(SensitiveWord, word_id) if obj is None: raise LookupError("word not found") from app.db.models import SensitiveHit await session.execute( sa.delete(SensitiveHit).where(SensitiveHit.word_id == word_id) ) await session.delete(obj) await session.flush() async def preview_count( session: AsyncSession, word: str, exclude_words: list[str] | None = None ) -> int: """加词前 COUNT——MySQL 走 ngram MATCH,其他方言 LIKE(测试)。 口径注:MySQL MATCH 路径不扣排除词(预览=上限,排除词在扫描时精确 生效);LIKE 路径(测试库)精确扣。前端文案「预计命中」体现上限语义。 单字词在 MySQL 拒预览(ValueError)——ngram 对单字无用,落 LIKE 前导 通配全扫(86 万行实测 ~16s,b41e8fa0 迁移记录),添加后从命中列表观察。 """ word = (word or "").strip() if not word: return 0 dialect = session.bind.dialect.name if session.bind else "" if len(word) < 2 and dialect == "mysql": raise ValueError("单字词不做预览(MySQL 全扫);添加后从命中列表观察") conds = [Message.text.is_not(None), Message.text != ""] if dialect == "mysql": # 短语净化:剥 " 与 * —— BOOLEAN MODE 里裸引号/星号是语法符, # 混入会 1064 语法错;剥后包引号做整词短语匹配。 phrase = word.replace('"', "").replace("*", "") conds.append( sa.text("MATCH(message.text) AGAINST (:w IN BOOLEAN MODE)") .bindparams(w=f'"{phrase}"') ) stmt = sa.select(sa.func.count()).select_from(Message).where(*conds) n = (await session.execute(stmt)).scalar_one() else: escaped = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") like = f"%{escaped}%" stmt = sa.select(Message.msg_id, Message.text).where( Message.text.is_not(None), Message.text.like(like, escape="\\") ) rows = (await session.execute(stmt)).all() n = sum(1 for _mid, t in rows if not _hit_exclude(t, word, exclude_words)) return int(n) def _hit_exclude(text: str | None, word: str, exclude_words: list[str] | None) -> bool: """命中行含任一排除词 → 视为误报跳过。""" if not exclude_words or not text: return False return any(ex in text for ex in exclude_words) # 内置起步词库(spec §3.6):幂等补种(CLI sensitive-scan 首跑时调),上线后 # 管理员在词库页自补行业词。 STARTER_WORDS: list[tuple[str, str]] = [ ("私单", "飞单"), ("飞单", "飞单"), ("加私人微信", "私联"), ("红包", "资金"), ("转账", "资金"), ("离职", "人员风险"), ] async def ensure_starter_words(session: AsyncSession) -> int: """缺则补(按 word 去重幂等),返回新种入数。""" existing = set( ( await session.execute(sa.select(SensitiveWord.word)) ).scalars().all() ) n = 0 for word, category in STARTER_WORDS: if word in existing: continue session.add(SensitiveWord(word=word, category=category)) n += 1 if n: await session.flush() return n