feat(sync): reconciliation scheduler with digest and bootstrap healing
P2 Task 4(inbox 对账):
- services/sync 新增 SyncReconciler(runOnce 可测;main 双循环,间隔
SYNC_RECONCILE_INTERVAL_MS env 可调,CN_API_URL 指向 cn 入口):每轮
先按 album/category/photo/asset 逐类分页拉取 cn reconcile「当前状态
事件」并直接签名推送到既有 content-batches 管道。sinceVersion 取 0
(全量当前态)而非 per-type 已应用最大版本——cn reconcile 以类型级
全局版本阈值过滤,类型内单实体滞后会被 max 掩蔽;ov 的 eventId 幂等
+ 版本单调把已追平实体全部判重(recon 确定性 eventId 重扫零成本),
偏差登记:计划原文对账返回「缺失清单」,契约已定为可重放 SyncEvent
数组(免建第二条拉取通道),openapi 描述同步更新
- cn 新增 POST /internal/sync/digest:每相册权威计数 {categoryCount,
publishedPhotoCount, photoCount, assetCount};ov 比对副本计数(照片
取 [publishedPhotoCount, photoCount] 区间口径——hidden/deleted 照片
若从未以 published 态同步过 ov 合法无行,缺席即撤下),不一致相册
自动触发 cn bootstrap 重建;ov-only 相册(cn 行已删,bootstrap 无法
表达删除)告警登记不自动处置
- 收敛语义两处落位:① sync-inbox 版本比较改「严格更旧才拒绝」——
同版本新 eventId(bootstrap/reconcile 快照重发)为当前态幂等重断言
应用,修复 ov 缺失副本行;A26 不受影响(更旧版本仍拒绝,cn 侧
entityVersion 与状态一一对应)② album.updated 快照载荷(bootstrap/
reconcile)携带分类全量清单,ov 存在即整体替换 albums_ov.categories
——修复错过的 category 墓碑(Task 3 登记边界:cn 行已删,无现存实体
事件可重放,唯全量替换可收敛)
- e2e(reconcile.e2e-spec):漏事件追赶(ov v1/cn v5 事件投后即失)→
清扫追平;digest 修复缺失 photos_ov 行(bootstrap 重发 → 同版本
重断言重建);digest 修复丢失的分类墓碑(全量清单替换收敛);稳态
空转(applied=0、无 bootstrap);Task 3 套件保持绿
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
+75
-20
@@ -1,18 +1,28 @@
|
||||
// services/sync/src/main.ts —— sync 发送器进程入口(P2 Task 3)。构建产物
|
||||
// dist/main.js 由根 dev:sync / 生产启动命令运行;env 一律来自进程环境(根
|
||||
// dotenv-cli 注入),此处不读文件。循环:每 SYNC_PUSH_INTERVAL_MS 轮询一次
|
||||
// outbox(SYNC_BATCH_SIZE 一批,high 优先——见 sender.ts 排序说明;满批立即
|
||||
// 追下一轮,积压排空不被轮询间隔拖慢)。失败退避:连续失败按间隔 ×2^n 指数
|
||||
// 递增、60s 封顶,成功即归零。优雅退出:SIGINT/SIGTERM 停轮询(等待在飞
|
||||
// 一轮,fetch 超时已封顶 30s)后断开 Prisma。
|
||||
// services/sync/src/main.ts —— sync 进程入口(P2 Task 3 发送器 + Task 4 对账
|
||||
// 调度)。构建产物 dist/main.js 由根 dev:sync / 生产启动命令运行;env 一律来自
|
||||
// 进程环境(根 dotenv-cli 注入),此处不读文件。
|
||||
//
|
||||
// 两个独立循环:
|
||||
// · 发送循环:每 SYNC_PUSH_INTERVAL_MS 轮询 outbox(SYNC_BATCH_SIZE 一批,
|
||||
// high 优先——见 sender.ts 排序说明;满批立即追下一轮,积压排空不被轮询
|
||||
// 间隔拖慢)。失败退避:连续失败按间隔 ×2^n 指数递增、60s 封顶,成功归零。
|
||||
// · 对账循环(Task 4):每 SYNC_RECONCILE_INTERVAL_MS 一轮 SyncReconciler.
|
||||
// runOnce(当前态清扫重放 + digest 比对触发 bootstrap,见 reconciler.ts)。
|
||||
// 失败仅记日志、下轮再试(分钟级节奏,无需退避——幂等安全)。
|
||||
//
|
||||
// 双库连接:cn 库(outbox 取数)+ ov 库(副本计数直读)。优雅退出:
|
||||
// SIGINT/SIGTERM 停两循环(等待在飞一轮,fetch 超时已封顶 30s)后断开两连接。
|
||||
import { pino } from 'pino';
|
||||
import { PrismaClient } from '@photo/db-cn';
|
||||
import { PrismaClient as OvPrismaClient } from '@photo/db-ov';
|
||||
import {
|
||||
SYNC_BATCH_SIZE,
|
||||
SYNC_INTERNAL_KEY_ENV,
|
||||
SYNC_PUSH_INTERVAL_MS,
|
||||
SYNC_RECONCILE_INTERVAL_MS,
|
||||
} from '@photo/contracts';
|
||||
import { OutboxSender } from './sender';
|
||||
import { SyncReconciler } from './reconciler';
|
||||
|
||||
/** 失败退避封顶(对齐 SYNC_RECONCILE_INTERVAL_MS 量级:分钟级封顶足够)。 */
|
||||
const MAX_BACKOFF_MS = 60_000;
|
||||
@@ -37,13 +47,35 @@ function readIntEnv(
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const sleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
/**
|
||||
* 可被停机唤醒的 sleep:对账循环节拍是分钟级,若让 SIGTERM 干等一个完整
|
||||
* interval 才退出不可接受——shutdown 置 stopping 后调用 wake 提前放行,
|
||||
* 循环条件 !stopping 随即退出(在飞一轮的各步均有界,见 shutdown 注释)。
|
||||
*/
|
||||
let wakeSleeper: (() => void) | null = null;
|
||||
const sleepStoppable = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
wakeSleeper = null;
|
||||
resolve();
|
||||
}, ms);
|
||||
wakeSleeper = () => {
|
||||
clearTimeout(timer);
|
||||
wakeSleeper = null;
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const logger = pino({ level: process.env.LOG_LEVEL ?? 'info' });
|
||||
const intervalMs = readIntEnv(logger, 'SYNC_PUSH_INTERVAL_MS', SYNC_PUSH_INTERVAL_MS);
|
||||
const batchSize = readIntEnv(logger, 'SYNC_BATCH_SIZE', SYNC_BATCH_SIZE);
|
||||
const reconcileIntervalMs = readIntEnv(
|
||||
logger,
|
||||
'SYNC_RECONCILE_INTERVAL_MS',
|
||||
SYNC_RECONCILE_INTERVAL_MS,
|
||||
);
|
||||
const cnApiUrl = process.env.CN_API_URL ?? 'http://127.0.0.1:8080';
|
||||
const ovApiUrl = process.env.OV_API_URL ?? 'http://127.0.0.1:8081';
|
||||
const internalKey = process.env[SYNC_INTERNAL_KEY_ENV] ?? '';
|
||||
if (internalKey === '') {
|
||||
@@ -52,23 +84,31 @@ async function main(): Promise<void> {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const prismaCn = new PrismaClient();
|
||||
const prismaOv = new OvPrismaClient();
|
||||
const sender = new OutboxSender({
|
||||
prisma,
|
||||
prisma: prismaCn,
|
||||
ovApiUrl,
|
||||
internalKey,
|
||||
batchSize,
|
||||
timeoutMs: REQUEST_TIMEOUT_MS,
|
||||
logger,
|
||||
});
|
||||
const reconciler = new SyncReconciler({
|
||||
prismaOv,
|
||||
cnApiUrl,
|
||||
ovApiUrl,
|
||||
internalKey,
|
||||
timeoutMs: REQUEST_TIMEOUT_MS,
|
||||
logger,
|
||||
});
|
||||
logger.info(
|
||||
{ ovApiUrl, intervalMs, batchSize },
|
||||
'sync sender started',
|
||||
{ cnApiUrl, ovApiUrl, intervalMs, batchSize, reconcileIntervalMs },
|
||||
'sync sender + reconciler started',
|
||||
);
|
||||
|
||||
let stopping = false;
|
||||
let inFlight: Promise<void> | null = null;
|
||||
const loop = async (): Promise<void> => {
|
||||
const senderLoop = async (): Promise<void> => {
|
||||
let failureStreak = 0;
|
||||
while (!stopping) {
|
||||
try {
|
||||
@@ -94,23 +134,38 @@ async function main(): Promise<void> {
|
||||
},
|
||||
'outbox push failed(整批未标记,下轮重推)',
|
||||
);
|
||||
await sleep(backoffMs);
|
||||
await sleepStoppable(backoffMs);
|
||||
continue;
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
await sleepStoppable(intervalMs);
|
||||
}
|
||||
};
|
||||
inFlight = loop();
|
||||
const reconcileLoop = async (): Promise<void> => {
|
||||
while (!stopping) {
|
||||
try {
|
||||
await reconciler.runOnce();
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err: err instanceof Error ? err.message : String(err) },
|
||||
'reconcile cycle failed(幂等安全,下轮再试)',
|
||||
);
|
||||
}
|
||||
await sleepStoppable(reconcileIntervalMs);
|
||||
}
|
||||
};
|
||||
const inFlight: Promise<void>[] = [senderLoop(), reconcileLoop()];
|
||||
|
||||
const shutdown = (signal: string): void => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
wakeSleeper?.();
|
||||
logger.info({ signal }, 'sync sender shutting down');
|
||||
void (async () => {
|
||||
try {
|
||||
// 等在飞一轮收尾(runOnce 各步均有界:HTTP 30s 超时 / DB 查询即回)。
|
||||
await inFlight;
|
||||
await prisma.$disconnect();
|
||||
await Promise.all(inFlight);
|
||||
await prismaCn.$disconnect();
|
||||
await prismaOv.$disconnect();
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err: err instanceof Error ? err.message : String(err) },
|
||||
|
||||
Reference in New Issue
Block a user