fix: 归档脚本微审修复(fsync/文件名防覆盖/VACUUM容错/导出删除对账)
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
+54
-17
@@ -14,11 +14,19 @@ messages → conversations, 单事务), 删了行才 VACUUM。导出/删除集
|
||||
'+00:00' 后缀——同族字符串字典序比较对两者皆单调(差异位在 '.'/'+',
|
||||
仅整秒边界有微差, 天级截止无实际影响);
|
||||
- 导出先落盘后删除: 文件写失败 → 数据一字节不动(多一份孤儿导出无害,
|
||||
反之是不可逆丢失); 目录懒建(无可归档/失败时不产生任何文件);
|
||||
反之是不可逆丢失); 落盘 = 临时文件 fsync 后原子 replace(崩溃级: 断电
|
||||
窗口内要么旧文件在、要么新文件完整, "导出先于删除"不击穿); 目录懒建
|
||||
(无可归档/失败时不产生任何文件);
|
||||
- 导出/删除行数对账: 两边计数差一行即非零退出(删除已提交, 不一致必须
|
||||
人工响, 不静默放过);
|
||||
- --dry-run 只列不动: 库零变化、不导出、不 VACUUM;
|
||||
- VACUUM 被占用锁(SQLITE_BUSY)不判归档失败: 导出/删除已安全完成, 提示
|
||||
稍后手动执行;
|
||||
- archives 目录 = tenants 目录的兄弟 data/archives(同 registry
|
||||
uploads_root 的派生纪律: 生产 <repo>/tenants → <repo>/data/archives,
|
||||
测试注入 tmp tenants → tmp 内隔离; data/ 在 .gitignore 内天然覆盖)。
|
||||
测试注入 tmp tenants → tmp 内隔离; data/ 在 .gitignore 内天然覆盖);
|
||||
导出文件名带时分秒(<日期>-<时分秒>-<N>d.json), 同日同参重跑不覆盖
|
||||
唯一副本。
|
||||
|
||||
用法(Windows 须 PYTHONUTF8=1, 输出含中文):
|
||||
PYTHONUTF8=1 python scripts/archive.py --tenant <slug> \
|
||||
@@ -29,6 +37,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -139,12 +148,15 @@ async def _export_document(conn, cutoff: str) -> dict:
|
||||
|
||||
|
||||
def _write_export(path: Path, doc: dict) -> None:
|
||||
"""先落盘后删库的'落盘'端: 目录懒建 + 原子替换(临时文件 + replace,
|
||||
中断不留半份导出)。"""
|
||||
"""先落盘后删库的'落盘'端: 目录懒建 + 临时文件写完 flush+fsync 再
|
||||
原子 replace——进程级升级到崩溃级(断电窗口内要么旧文件在、要么新文件
|
||||
完整落盘, '导出先于删除'的承诺不被断电击穿)。"""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".json.tmp")
|
||||
tmp.write_text(
|
||||
json.dumps(doc, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(doc, f, ensure_ascii=False, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
@@ -191,7 +203,8 @@ async def run(args) -> None:
|
||||
return
|
||||
|
||||
export_path = archives_dir / args.tenant / (
|
||||
f"{datetime.now(timezone.utc).strftime('%Y%m%d')}-{days}d.json")
|
||||
f"{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}"
|
||||
f"-{days}d.json")
|
||||
deleted: dict[str, int] = {}
|
||||
if not old_ids:
|
||||
print("归档会话: 0 条")
|
||||
@@ -206,7 +219,11 @@ async def run(args) -> None:
|
||||
"days": days, "cutoff": cutoff,
|
||||
"conversation_count": len(old_ids)}
|
||||
doc.update(await _export_document(conn, cutoff))
|
||||
_write_export(export_path, doc)
|
||||
try:
|
||||
_write_export(export_path, doc)
|
||||
except OSError as exc:
|
||||
raise SystemExit(
|
||||
f"导出写盘失败({exc}): 中止, 库零变化(未删除)") from exc
|
||||
|
||||
# ---- 清库(倒 FK, 单事务; 子查询同源截止串) ------------------------
|
||||
for table in DELETE_ORDER:
|
||||
@@ -218,20 +235,40 @@ async def run(args) -> None:
|
||||
deleted[table] = cur.rowcount
|
||||
await conn.commit()
|
||||
|
||||
# ---- VACUUM(删了行才跑; 已 commit, 事务外) ------------------------
|
||||
pages_before = await _scalar(conn, "PRAGMA page_count")
|
||||
await conn.execute("VACUUM")
|
||||
pages_after = await _scalar(conn, "PRAGMA page_count")
|
||||
|
||||
# ---- 报告 ----------------------------------------------------------
|
||||
# ---- 对账: 导出行数 vs 删除行数(删除已提交, 差一行必须人工响) ------
|
||||
mismatched = []
|
||||
for table in DELETE_ORDER:
|
||||
exported = (len(doc[table]) if table == "conversations"
|
||||
else sum(len(rows) for rows in doc[table].values()))
|
||||
if exported != deleted[table]:
|
||||
mismatched.append(f"{table}(导出={exported} "
|
||||
f"删除={deleted[table]})")
|
||||
if mismatched:
|
||||
raise SystemExit("导出/删除对账不一致: "
|
||||
+ ", ".join(mismatched)
|
||||
+ "; 导出文件已落盘, 请人工核查")
|
||||
print(f"导出: {export_path}")
|
||||
print(f"归档会话: {len(old_ids)} 条; 各表删除行数(倒 FK 序):")
|
||||
for table in DELETE_ORDER:
|
||||
print(f" {table}: {deleted[table]}")
|
||||
print(f"导出: {export_path}")
|
||||
print("导出/删除对账: 一致")
|
||||
|
||||
# ---- VACUUM(删了行才跑; 已 commit, 事务外; 占用锁不判归档失败) ----
|
||||
try:
|
||||
pages_before = await _scalar(conn, "PRAGMA page_count")
|
||||
await conn.execute("VACUUM")
|
||||
pages_after = await _scalar(conn, "PRAGMA page_count")
|
||||
print(f"VACUUM: 已执行(page_count {pages_before} "
|
||||
f"→ {pages_after})")
|
||||
print("归档结论: 导出已落盘, 删除已提交, VACUUM 完成")
|
||||
except sqlite3.OperationalError as exc:
|
||||
print(f"VACUUM: 被占用锁跳过({exc})——归档已完成, "
|
||||
f"请稍后手动执行 VACUUM")
|
||||
print("归档结论: 导出已落盘, 删除已提交")
|
||||
|
||||
# ---- 报告收尾: 库文件前后大小 --------------------------------------
|
||||
size_after = tenant_db.stat().st_size
|
||||
print(f"库文件: {size_before} → {size_after} 字节")
|
||||
print(f"VACUUM: 已执行(page_count {pages_before} → {pages_after})")
|
||||
print("归档结论: 导出已落盘, 删除已提交, VACUUM 完成")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
Reference in New Issue
Block a user