fix(analytics): run_full水位前置防回填窗口漏扫+README/DEPLOY/脚本号收尾
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -165,7 +165,7 @@ async def run_incremental(session: AsyncSession) -> dict:
|
||||
|
||||
|
||||
async def run_full(session: AsyncSession) -> dict:
|
||||
"""全量回填(CLI 一次性):每词单独扫全库;水位推进同增量。
|
||||
"""全量回填(CLI 一次性):每词单独扫全库;水位在扫描前取(见内注)。
|
||||
|
||||
每词结果整批取回(MATCH/LIKE 已在服务端预筛成命中行)再分片复用
|
||||
_scan_batch——不在未取尽的结果上发新语句/flush(MySQL 驱动会
|
||||
@@ -178,6 +178,15 @@ async def run_full(session: AsyncSession) -> dict:
|
||||
)
|
||||
).scalars().all()
|
||||
st = await _get_state(session)
|
||||
# 水位在扫描前取——回填期间新入库的行 fetched_at > 水位,下轮增量自然
|
||||
# 扫到;若在词循环后才取,回填窗口内 commit 的行 fetched_at ≤ 水位,
|
||||
# >= 增量也扫不到(首部署夜间回填与 02:00 全量批写重叠时漏扫)。
|
||||
# >= 边界重扫由 UNIQUE(word_id,msg_id) 去重。
|
||||
new_wm = (
|
||||
await session.execute(sa.select(sa.func.max(Message.fetched_at)))
|
||||
).scalar_one_or_none()
|
||||
if new_wm is not None:
|
||||
st.last_fetched_at = new_wm
|
||||
total_hits = 0
|
||||
scanned_rows = 0
|
||||
for w in words:
|
||||
@@ -192,11 +201,6 @@ async def run_full(session: AsyncSession) -> dict:
|
||||
for i in range(0, len(rows), 500):
|
||||
total_hits += await _scan_batch(session, w, rows[i:i + 500])
|
||||
await session.flush() # 行已 .all() 物化,分片 flush 控挂起对象数
|
||||
new_wm = (
|
||||
await session.execute(sa.select(sa.func.max(Message.fetched_at)))
|
||||
).scalar_one_or_none()
|
||||
if new_wm is not None:
|
||||
st.last_fetched_at = new_wm
|
||||
st.scanned += scanned_rows
|
||||
st.hits += total_hits
|
||||
st.last_run_at = dt.datetime.now(dt.UTC)
|
||||
|
||||
Reference in New Issue
Block a user