浅念
211 lines
8.3 KiB
Python
211 lines
8.3 KiB
Python
"""敏感词扫描器——增量水位(fetched_at)+ 首次全量回填。
|
||
|
||
水位语义(spec §3.6 P2):last_fetched_at = MAX(message.fetched_at)。增量
|
||
补扫拉回的历史消息 timestamp 早于水位但 fetched_at 是新的,按 timestamp
|
||
做水位会永久漏扫。比较用 >=:fetched_at 是 MySQL DATETIME 秒精度,SELECT
|
||
快照后同秒 commit 的行 fetched_at == 水位,严格 > 会永久丢弃(只丢不重
|
||
扫)——>= 每 tick 重扫 ≤1 秒边界行,UNIQUE(word_id,msg_id)+已有命中
|
||
跳过去重,幂等(st.scanned 计数随边界重扫微膨胀,可接受)。
|
||
|
||
方言:MySQL ngram MATCH(≥2字,快);单字词与其他方言走 LIKE。语义差
|
||
异(MATCH 分词 vs LIKE 子串)可接受——命中可处置,误报治理闭环在。
|
||
|
||
分批:增量直接拉水位后行集(4h 增量仅数千);全量回填按词扫——MATCH 在
|
||
服务端把 86 万行预筛成命中行(每词仅数千),整批取回后按 500 行分片插
|
||
入/flush 控挂起对象数。不流式消费游标:MySQL 驱动(asyncmy)下结果未
|
||
取尽时同连接发下一条语句会 out-of-sync,全量路径无测试覆盖,取保守形。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import datetime as dt
|
||
import logging
|
||
|
||
import sqlalchemy as sa
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.analytics.sensitive.words import _hit_exclude
|
||
from app.db.models import Message, SensitiveHit, SensitiveScanState, SensitiveWord
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
ENTITY = "message_text"
|
||
_SNIPPET_RADIUS = 30 # 命中词前后各取 30 字
|
||
|
||
|
||
def _snippet(text: str, word: str) -> str:
|
||
i = text.find(word)
|
||
if i < 0:
|
||
return text[: _SNIPPET_RADIUS * 2]
|
||
lo, hi = max(0, i - _SNIPPET_RADIUS), min(len(text), i + len(word) + _SNIPPET_RADIUS)
|
||
return text[lo:hi]
|
||
|
||
|
||
def _text_match_cond(word: str, dialect: str) -> list:
|
||
"""全量回填路径:MySQL ≥2字 → ngram MATCH;否则 LIKE(单字 ngram 不可用)。
|
||
|
||
已知漏召:ngram 对嵌入标点的拉丁词可能分词不齐(如 "we-chat" 不含
|
||
"we-chat" 整词 token)——增量路径 Python 子串精确无此问题;起步词
|
||
全 CJK,当前无影响。
|
||
"""
|
||
if dialect == "mysql" and len(word) >= 2:
|
||
phrase = word.replace('"', "").replace("*", "")
|
||
return [
|
||
sa.text("MATCH(message.text) AGAINST (:w IN BOOLEAN MODE)")
|
||
.bindparams(w=f'"{phrase}"')
|
||
]
|
||
escaped = word.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||
return [Message.text.like(f"%{escaped}%", escape="\\")]
|
||
|
||
|
||
async def _get_state(session: AsyncSession) -> SensitiveScanState:
|
||
st = await session.get(SensitiveScanState, ENTITY)
|
||
if st is None:
|
||
st = SensitiveScanState(entity=ENTITY)
|
||
session.add(st)
|
||
await session.flush()
|
||
return st
|
||
|
||
|
||
async def _insert_hits(
|
||
session: AsyncSession, word: SensitiveWord, rows: list, existing: set[str]
|
||
) -> int:
|
||
n = 0
|
||
for msg_id, channel, owner, talker, room, ts, text in rows:
|
||
if msg_id in existing:
|
||
continue
|
||
if _hit_exclude(text, word.word, word.exclude_words or None):
|
||
continue
|
||
session.add(
|
||
SensitiveHit(
|
||
word_id=word.id, msg_id=msg_id, channel=channel,
|
||
owner_account_id=owner, talker_id=talker, room_id=room,
|
||
ts=ts, snippet=_snippet(text or "", word.word),
|
||
)
|
||
)
|
||
existing.add(msg_id)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
async def _scan_batch(
|
||
session: AsyncSession, word: SensitiveWord, rows: list
|
||
) -> int:
|
||
"""对一个词扫一批 (msg_id..text) 行:查已有命中、插新命中。"""
|
||
sel_rows = [tuple(r[:7]) for r in rows if word.word in (r[6] or "")]
|
||
if not sel_rows:
|
||
return 0
|
||
existing = set(
|
||
(
|
||
await session.execute(
|
||
sa.select(SensitiveHit.msg_id).where(
|
||
SensitiveHit.word_id == word.id,
|
||
SensitiveHit.msg_id.in_(
|
||
[r[0] for r in sel_rows] or ["-"]
|
||
),
|
||
)
|
||
)
|
||
).scalars().all()
|
||
)
|
||
return await _insert_hits(session, word, sel_rows, existing)
|
||
|
||
|
||
async def run_incremental(session: AsyncSession) -> dict:
|
||
"""增量扫描:fetched_at > 水位 的行 × 全部 enabled 词。
|
||
|
||
增量行少(4h 数千)——统一 Python 侧子串判定(=LIKE 语义,精确;MATCH
|
||
只在全量回填的 MySQL 路径用,见 _text_match_cond)。
|
||
"""
|
||
words = (
|
||
await session.execute(
|
||
sa.select(SensitiveWord).where(SensitiveWord.enabled.is_(True))
|
||
)
|
||
).scalars().all()
|
||
st = await _get_state(session)
|
||
|
||
# 无水位自卫:无水位首跑=全表(86 万行 MEDIUMTEXT)进 API 进程内存。
|
||
# 增量由调度 tick 直调,大表首跑必须走 CLI sensitive-scan --full 建水位。
|
||
if st.last_fetched_at is None:
|
||
n = (await session.execute(
|
||
sa.select(sa.func.count()).select_from(Message)
|
||
)).scalar_one()
|
||
if n > 20_000:
|
||
raise RuntimeError(
|
||
"message 表超阈值且无扫描水位——首跑请先 CLI sensitive-scan"
|
||
" --full 建水位"
|
||
)
|
||
|
||
conds = [Message.text.is_not(None), Message.text != ""]
|
||
if st.last_fetched_at is not None:
|
||
# >=(非 >):fetched_at 秒精度,快照后同秒 commit 的行等值——严格 >
|
||
# 永久丢弃;>= 每 tick 重扫 ≤1s 边界行,existing/UNIQUE 去重。
|
||
conds.append(Message.fetched_at >= st.last_fetched_at)
|
||
rows = (
|
||
await session.execute(
|
||
sa.select(
|
||
Message.msg_id, Message.channel, Message.owner_account_id,
|
||
Message.talker_id, Message.room_id, Message.timestamp,
|
||
Message.text, Message.fetched_at,
|
||
).where(*conds)
|
||
)
|
||
).all()
|
||
|
||
scanned = len(rows)
|
||
hits = 0
|
||
for w in words:
|
||
hits += await _scan_batch(session, w, rows)
|
||
|
||
new_wm = max((r[7] for r in rows), default=None) if rows else None
|
||
if new_wm is not None:
|
||
st.last_fetched_at = new_wm
|
||
st.last_run_at = dt.datetime.now(dt.UTC)
|
||
st.scanned += scanned
|
||
st.hits += hits
|
||
await session.flush()
|
||
return {"scanned": scanned, "hits": hits, "new_watermark": new_wm}
|
||
|
||
|
||
async def run_full(session: AsyncSession) -> dict:
|
||
"""全量回填(CLI 一次性):每词单独扫全库;水位在扫描前取(见内注)。
|
||
|
||
每词结果整批取回(MATCH/LIKE 已在服务端预筛成命中行)再分片复用
|
||
_scan_batch——不在未取尽的结果上发新语句/flush(MySQL 驱动会
|
||
out-of-sync)。日志只打数量,不打词与正文。
|
||
"""
|
||
dialect = session.bind.dialect.name if session.bind else ""
|
||
words = (
|
||
await session.execute(
|
||
sa.select(SensitiveWord).where(SensitiveWord.enabled.is_(True))
|
||
)
|
||
).scalars().all()
|
||
st = await _get_state(session)
|
||
# 水位在扫描前取——回填期间新入库的行 fetched_at > 水位,下轮增量自然
|
||
# 扫到;若在词循环后才取,回填窗口内 commit 的行 fetched_at ≤ 水位,
|
||
# >= 增量也扫不到(首部署夜间回填与 02:00 全量批写重叠时漏扫)。
|
||
# >= 边界重扫由 UNIQUE(word_id,msg_id) 去重。
|
||
new_wm = (
|
||
await session.execute(sa.select(sa.func.max(Message.fetched_at)))
|
||
).scalar_one_or_none()
|
||
if new_wm is not None:
|
||
st.last_fetched_at = new_wm
|
||
total_hits = 0
|
||
scanned_rows = 0
|
||
for w in words:
|
||
conds = [Message.text.is_not(None), Message.text != ""]
|
||
conds += _text_match_cond(w.word, dialect)
|
||
stmt = sa.select(
|
||
Message.msg_id, Message.channel, Message.owner_account_id,
|
||
Message.talker_id, Message.room_id, Message.timestamp, Message.text,
|
||
).where(*conds)
|
||
rows = (await session.execute(stmt)).all()
|
||
scanned_rows += len(rows) # MATCH 预筛返回行数,按词累计(非去重消息数)
|
||
for i in range(0, len(rows), 500):
|
||
total_hits += await _scan_batch(session, w, rows[i:i + 500])
|
||
await session.flush() # 行已 .all() 物化,分片 flush 控挂起对象数
|
||
st.scanned += scanned_rows
|
||
st.hits += total_hits
|
||
st.last_run_at = dt.datetime.now(dt.UTC)
|
||
await session.flush()
|
||
logger.info("[sensitive-scan] full backfill done: words=%s hits=%s",
|
||
len(words), total_hits)
|
||
return {"words": len(words), "hits": total_hits}
|