feat(sync): reconciliation scheduler with digest and bootstrap healing
P2 Task 4（inbox 对账）：
- services/sync 新增 SyncReconciler（runOnce 可测；main 双循环，间隔
SYNC_RECONCILE_INTERVAL_MS env 可调，CN_API_URL 指向 cn 入口）：每轮
先按 album/category/photo/asset 逐类分页拉取 cn reconcile「当前状态
事件」并直接签名推送到既有 content-batches 管道。sinceVersion 取 0
（全量当前态）而非 per-type 已应用最大版本——cn reconcile 以类型级
全局版本阈值过滤，类型内单实体滞后会被 max 掩蔽；ov 的 eventId 幂等
+ 版本单调把已追平实体全部判重（recon 确定性 eventId 重扫零成本），
偏差登记：计划原文对账返回「缺失清单」，契约已定为可重放 SyncEvent
数组（免建第二条拉取通道），openapi 描述同步更新
- cn 新增 POST /internal/sync/digest：每相册权威计数 {categoryCount,
publishedPhotoCount, photoCount, assetCount}；ov 比对副本计数（照片
取 [publishedPhotoCount, photoCount] 区间口径——hidden/deleted 照片
若从未以 published 态同步过 ov 合法无行，缺席即撤下），不一致相册
自动触发 cn bootstrap 重建；ov-only 相册（cn 行已删，bootstrap 无法
表达删除）告警登记不自动处置
- 收敛语义两处落位：① sync-inbox 版本比较改「严格更旧才拒绝」——
同版本新 eventId（bootstrap/reconcile 快照重发）为当前态幂等重断言
应用，修复 ov 缺失副本行；A26 不受影响（更旧版本仍拒绝，cn 侧
entityVersion 与状态一一对应）② album.updated 快照载荷（bootstrap/
reconcile）携带分类全量清单，ov 存在即整体替换 albums_ov.categories
——修复错过的 category 墓碑（Task 3 登记边界：cn 行已删，无现存实体
事件可重放，唯全量替换可收敛）
- e2e（reconcile.e2e-spec）：漏事件追赶（ov v1/cn v5 事件投后即失）→
清扫追平；digest 修复缺失 photos_ov 行（bootstrap 重发 → 同版本
重断言重建）；digest 修复丢失的分类墓碑（全量清单替换收敛）；稳态
空转（applied=0、无 bootstrap）；Task 3 套件保持绿
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -2,7 +2,7 @@
|
||||
"name": "@photo/sync",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "CN→OV outbox sender (轮询 outbox_events 推送 content-batches 到 ov 入口；P2 Task 3)",
|
||||
"description": "CN→OV sync process (outbox sender + ov reconciliation scheduler; P2 Task 3/4)",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"start": "tsc -p tsconfig.build.json && node dist/main.js",
|
||||
@@ -14,13 +14,13 @@
|
||||
"dependencies": {
|
||||
"@photo/contracts": "workspace:*",
|
||||
"@photo/db-cn": "workspace:*",
|
||||
"@photo/db-ov": "workspace:*",
|
||||
"pino": "^10.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nestjs/common": "^10.4.6",
|
||||
"@nestjs/core": "^10.4.6",
|
||||
"@nestjs/platform-express": "^10.4.6",
|
||||
"@photo/db-ov": "workspace:*",
|
||||
"@photo/storage": "workspace:^",
|
||||
"@types/jest": "^29.5.13",
|
||||
"@types/node": "^22.10.0",
|
||||
|
||||
+75
-20
@@ -1,18 +1,28 @@
|
||||
// services/sync/src/main.ts —— sync 发送器进程入口（P2 Task 3）。构建产物
|
||||
// dist/main.js 由根 dev:sync / 生产启动命令运行；env 一律来自进程环境（根
|
||||
// dotenv-cli 注入），此处不读文件。循环：每 SYNC_PUSH_INTERVAL_MS 轮询一次
|
||||
// outbox（SYNC_BATCH_SIZE 一批，high 优先——见 sender.ts 排序说明；满批立即
|
||||
// 追下一轮，积压排空不被轮询间隔拖慢）。失败退避：连续失败按间隔 ×2^n 指数
|
||||
// 递增、60s 封顶，成功即归零。优雅退出：SIGINT/SIGTERM 停轮询（等待在飞
|
||||
// 一轮，fetch 超时已封顶 30s）后断开 Prisma。
|
||||
// services/sync/src/main.ts —— sync 进程入口（P2 Task 3 发送器 + Task 4 对账
|
||||
// 调度）。构建产物 dist/main.js 由根 dev:sync / 生产启动命令运行；env 一律来自
|
||||
// 进程环境（根 dotenv-cli 注入），此处不读文件。
|
||||
//
|
||||
// 两个独立循环：
|
||||
// · 发送循环：每 SYNC_PUSH_INTERVAL_MS 轮询 outbox（SYNC_BATCH_SIZE 一批，
|
||||
// high 优先——见 sender.ts 排序说明；满批立即追下一轮，积压排空不被轮询
|
||||
// 间隔拖慢）。失败退避：连续失败按间隔 ×2^n 指数递增、60s 封顶，成功归零。
|
||||
// · 对账循环（Task 4）：每 SYNC_RECONCILE_INTERVAL_MS 一轮 SyncReconciler.
|
||||
// runOnce（当前态清扫重放 + digest 比对触发 bootstrap，见 reconciler.ts）。
|
||||
// 失败仅记日志、下轮再试（分钟级节奏，无需退避——幂等安全）。
|
||||
//
|
||||
// 双库连接：cn 库（outbox 取数）+ ov 库（副本计数直读）。优雅退出：
|
||||
// SIGINT/SIGTERM 停两循环（等待在飞一轮，fetch 超时已封顶 30s）后断开两连接。
|
||||
import { pino } from 'pino';
|
||||
import { PrismaClient } from '@photo/db-cn';
|
||||
import { PrismaClient as OvPrismaClient } from '@photo/db-ov';
|
||||
import {
|
||||
SYNC_BATCH_SIZE,
|
||||
SYNC_INTERNAL_KEY_ENV,
|
||||
SYNC_PUSH_INTERVAL_MS,
|
||||
SYNC_RECONCILE_INTERVAL_MS,
|
||||
} from '@photo/contracts';
|
||||
import { OutboxSender } from './sender';
|
||||
import { SyncReconciler } from './reconciler';
|
||||
|
||||
/** 失败退避封顶（对齐 SYNC_RECONCILE_INTERVAL_MS 量级：分钟级封顶足够）。 */
|
||||
const MAX_BACKOFF_MS = 60_000;
|
||||
@@ -37,13 +47,35 @@ function readIntEnv(
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const sleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
/**
|
||||
* 可被停机唤醒的 sleep：对账循环节拍是分钟级，若让 SIGTERM 干等一个完整
|
||||
* interval 才退出不可接受——shutdown 置 stopping 后调用 wake 提前放行，
|
||||
* 循环条件 !stopping 随即退出（在飞一轮的各步均有界，见 shutdown 注释）。
|
||||
*/
|
||||
let wakeSleeper: (() => void) | null = null;
|
||||
const sleepStoppable = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
wakeSleeper = null;
|
||||
resolve();
|
||||
}, ms);
|
||||
wakeSleeper = () => {
|
||||
clearTimeout(timer);
|
||||
wakeSleeper = null;
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const logger = pino({ level: process.env.LOG_LEVEL ?? 'info' });
|
||||
const intervalMs = readIntEnv(logger, 'SYNC_PUSH_INTERVAL_MS', SYNC_PUSH_INTERVAL_MS);
|
||||
const batchSize = readIntEnv(logger, 'SYNC_BATCH_SIZE', SYNC_BATCH_SIZE);
|
||||
const reconcileIntervalMs = readIntEnv(
|
||||
logger,
|
||||
'SYNC_RECONCILE_INTERVAL_MS',
|
||||
SYNC_RECONCILE_INTERVAL_MS,
|
||||
);
|
||||
const cnApiUrl = process.env.CN_API_URL ?? 'http://127.0.0.1:8080';
|
||||
const ovApiUrl = process.env.OV_API_URL ?? 'http://127.0.0.1:8081';
|
||||
const internalKey = process.env[SYNC_INTERNAL_KEY_ENV] ?? '';
|
||||
if (internalKey === '') {
|
||||
@@ -52,23 +84,31 @@ async function main(): Promise<void> {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const prismaCn = new PrismaClient();
|
||||
const prismaOv = new OvPrismaClient();
|
||||
const sender = new OutboxSender({
|
||||
prisma,
|
||||
prisma: prismaCn,
|
||||
ovApiUrl,
|
||||
internalKey,
|
||||
batchSize,
|
||||
timeoutMs: REQUEST_TIMEOUT_MS,
|
||||
logger,
|
||||
});
|
||||
const reconciler = new SyncReconciler({
|
||||
prismaOv,
|
||||
cnApiUrl,
|
||||
ovApiUrl,
|
||||
internalKey,
|
||||
timeoutMs: REQUEST_TIMEOUT_MS,
|
||||
logger,
|
||||
});
|
||||
logger.info(
|
||||
{ ovApiUrl, intervalMs, batchSize },
|
||||
'sync sender started',
|
||||
{ cnApiUrl, ovApiUrl, intervalMs, batchSize, reconcileIntervalMs },
|
||||
'sync sender + reconciler started',
|
||||
);
|
||||
|
||||
let stopping = false;
|
||||
let inFlight: Promise<void> | null = null;
|
||||
const loop = async (): Promise<void> => {
|
||||
const senderLoop = async (): Promise<void> => {
|
||||
let failureStreak = 0;
|
||||
while (!stopping) {
|
||||
try {
|
||||
@@ -94,23 +134,38 @@ async function main(): Promise<void> {
|
||||
},
|
||||
'outbox push failed（整批未标记，下轮重推）',
|
||||
);
|
||||
await sleep(backoffMs);
|
||||
await sleepStoppable(backoffMs);
|
||||
continue;
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
await sleepStoppable(intervalMs);
|
||||
}
|
||||
};
|
||||
inFlight = loop();
|
||||
const reconcileLoop = async (): Promise<void> => {
|
||||
while (!stopping) {
|
||||
try {
|
||||
await reconciler.runOnce();
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err: err instanceof Error ? err.message : String(err) },
|
||||
'reconcile cycle failed（幂等安全，下轮再试）',
|
||||
);
|
||||
}
|
||||
await sleepStoppable(reconcileIntervalMs);
|
||||
}
|
||||
};
|
||||
const inFlight: Promise<void>[] = [senderLoop(), reconcileLoop()];
|
||||
|
||||
const shutdown = (signal: string): void => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
wakeSleeper?.();
|
||||
logger.info({ signal }, 'sync sender shutting down');
|
||||
void (async () => {
|
||||
try {
|
||||
// 等在飞一轮收尾（runOnce 各步均有界：HTTP 30s 超时 / DB 查询即回）。
|
||||
await inFlight;
|
||||
await prisma.$disconnect();
|
||||
await Promise.all(inFlight);
|
||||
await prismaCn.$disconnect();
|
||||
await prismaOv.$disconnect();
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err: err instanceof Error ? err.message : String(err) },
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
// services/sync/src/reconciler.ts —— ov 补漏对账调度器（P2 Task 4）。
|
||||
// runOnce() 两阶段，每 SYNC_RECONCILE_INTERVAL_MS 一轮（main 循环驱动）：
|
||||
//
|
||||
// 阶段一「当前态清扫」：对 album/category/photo/asset 四类逐类分页拉取 cn
|
||||
// reconcile 清单（**当前状态事件**，非版本清单——Task 4 设计决策：重放走
|
||||
// 既有 content-batches 管道即幂等，无需第二条拉取通道），直接签名推送到 ov
|
||||
// content-batches。sinceVersion 取 0（全量当前态）而非「per-type 已应用最大
|
||||
// 版本」——cn reconcile 以**类型级全局版本阈值**过滤，类型内单实体滞后
|
||||
// （如 A 实体已 v9、B 实体 v3→v4）会被 max 掩蔽；0 则由 ov 的 eventId 幂等
|
||||
// + 版本单调把「已追平的实体」全部判重/判旧（确定性 recon:<type>:<id>:<ver>
|
||||
// 合成 eventId，重扫即纯重复检查），任何滞后/新实体必然被补上。仿真规模
|
||||
// （单册数百照片）下每轮开销可忽略；真实地域部署若需缩量，改点在 cn 侧
|
||||
// 提供 per-entity 水位比对（登记为 P5 演进项，不在 P2）。
|
||||
//
|
||||
// 阶段二「digest 比对 → bootstrap 重建」：拉取 cn digest（每相册权威计数：
|
||||
// categoryCount / publishedPhotoCount..photoCount 区间 / assetCount）与 ov
|
||||
// 副本计数比对。不一致的相册 → 触发 cn bootstrap {albumId}（当前状态快照
|
||||
// 写回 outbox，由常驻发送器投递）。收敛机制依赖两条 Task 4 语义：
|
||||
// · sync-inbox 对**同版本新 eventId** 事件做当前态重断言应用（cn 的
|
||||
// entityVersion 与状态一一对应）→ 修复 ov 缺失的 photos_ov 行；
|
||||
// · bootstrap/reconcile 的 album.updated 载荷携带分类全量清单，ov 整体
|
||||
// 替换 albums_ov.categories → 修复**错过的 category 墓碑**（cn 行已删，
|
||||
// 任何「重放现存实体」通道都无法表达的删除语义，Task 3 已登记边界）。
|
||||
// 照片计数取区间口径：hidden/deleted 照片若从未以 published 态同步过，ov
|
||||
// 合法地无行（缺席即撤下），故 ov 行数 ∈ [publishedPhotoCount, photoCount]
|
||||
// 即一致。ov 有而 cn 无的相册（cn 行已删，不可经 bootstrap 重建）仅告警
|
||||
// （P4a 管理页消费 sync_state/sync_inbox 的延迟与积压口径）。
|
||||
import type { PrismaClient } from '@photo/db-ov';
|
||||
import type { components } from '@photo/contracts';
|
||||
import {
|
||||
SYNC_SIGNATURE_HEADER,
|
||||
SYNC_TIMESTAMP_HEADER,
|
||||
signSyncBody,
|
||||
} from '@photo/contracts';
|
||||
import type { Logger } from 'pino';
|
||||
|
||||
type SyncEventWire = components['schemas']['SyncEvent'];
|
||||
type DigestResponse = components['schemas']['DigestResponse'];
|
||||
|
||||
/** 线格式端点路径（全局前缀 v1 + InternalSyncGuard 域；与 sender 同约定）。 */
|
||||
const RECONCILE_PATH = '/v1/internal/sync/reconcile';
|
||||
const DIGEST_PATH = '/v1/internal/sync/digest';
|
||||
const BOOTSTRAP_PATH = '/v1/internal/sync/bootstrap';
|
||||
const CONTENT_BATCHES_PATH = '/v1/internal/sync/content-batches';
|
||||
|
||||
/** 清扫覆盖的实体类型（slot-state 预留，无当前态可拉）。 */
|
||||
const SWEEP_ENTITY_TYPES = ['album', 'category', 'photo', 'asset'] as const;
|
||||
|
||||
const DEFAULT_RECONCILE_LIMIT = 500; // 任务口径单页上限（契约上限 1000）
|
||||
const DEFAULT_PUSH_BATCH_SIZE = 100; // content-batches 契约单批上限
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
/** 分页保险丝：单类型最多拉 100 页（5 万实体）——防响应形状异常导致的死循环。 */
|
||||
const MAX_PAGES_PER_TYPE = 100;
|
||||
|
||||
export interface SyncReconcilerOptions {
|
||||
/** ov 库客户端（副本计数比对直读；cn 侧数据一律经内部 HTTP，不直连 cn 库）。 */
|
||||
prismaOv: PrismaClient;
|
||||
/** cn 入口基址（如 http://127.0.0.1:8080，不含路径）。 */
|
||||
cnApiUrl: string;
|
||||
/** ov 入口基址（重放事件推送目标）。 */
|
||||
ovApiUrl: string;
|
||||
/** HMAC 密钥（与两端 InternalSyncGuard 同一 SYNC_INTERNAL_KEY 值）。 */
|
||||
internalKey: string;
|
||||
/** reconcile 单页上限（默认 500）。 */
|
||||
reconcileLimit?: number;
|
||||
/** content-batches 单批上限（默认契约 100）。 */
|
||||
pushBatchSize?: number;
|
||||
/** 单次 HTTP 超时毫秒（默认 30s）。 */
|
||||
timeoutMs?: number;
|
||||
logger?: Logger;
|
||||
}
|
||||
|
||||
/** 单轮 runOnce 的观测口径（main 循环打点；测试用于断言追赶/补漏/稳态）。 */
|
||||
export interface ReconcileStats {
|
||||
/** 阶段一从 cn 拉到的当前态事件数（四类型合计）。 */
|
||||
sweptEvents: number;
|
||||
/** 阶段一重放中 ov 实际应用（新版本/同版本重断言）的条数。 */
|
||||
replayApplied: number;
|
||||
/** 阶段二 cn digest 相册数。 */
|
||||
digestAlbums: number;
|
||||
/** 计数不一致 → 已触发 bootstrap 的相册（ healed by sender 投递）。 */
|
||||
mismatchedAlbums: string[];
|
||||
/** ov 存在而 cn 无的相册（cn 行已删——不可 bootstrap 重建，仅告警登记）。 */
|
||||
ovOrphanAlbums: string[];
|
||||
}
|
||||
|
||||
/** ov 侧每相册副本计数（albums_ov.categories 为 JSON，行数小，直接取回计算）。 */
|
||||
interface OvAlbumCounts {
|
||||
categoryCount: number;
|
||||
photoCount: number;
|
||||
}
|
||||
|
||||
export class SyncReconciler {
|
||||
private readonly reconcileLimit: number;
|
||||
private readonly pushBatchSize: number;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly cnBase: string;
|
||||
private readonly ovBase: string;
|
||||
|
||||
constructor(private readonly options: SyncReconcilerOptions) {
|
||||
this.reconcileLimit = options.reconcileLimit ?? DEFAULT_RECONCILE_LIMIT;
|
||||
this.pushBatchSize = options.pushBatchSize ?? DEFAULT_PUSH_BATCH_SIZE;
|
||||
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
this.cnBase = options.cnApiUrl.replace(/\/+$/, '');
|
||||
this.ovBase = options.ovApiUrl.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
/** 单轮对账：阶段一清扫重放 + 阶段二 digest 比对触发 bootstrap。 */
|
||||
async runOnce(): Promise<ReconcileStats> {
|
||||
const { sweptEvents, replayApplied } = await this.sweep();
|
||||
const digest = await this.digestCompare();
|
||||
const stats: ReconcileStats = {
|
||||
sweptEvents,
|
||||
replayApplied,
|
||||
digestAlbums: digest.digestAlbums,
|
||||
mismatchedAlbums: digest.mismatchedAlbums,
|
||||
ovOrphanAlbums: digest.ovOrphanAlbums,
|
||||
};
|
||||
this.options.logger?.info(
|
||||
{
|
||||
sweptEvents,
|
||||
replayApplied,
|
||||
digestAlbums: stats.digestAlbums,
|
||||
mismatched: stats.mismatchedAlbums.length,
|
||||
ovOrphans: stats.ovOrphanAlbums.length,
|
||||
},
|
||||
'reconcile cycle done',
|
||||
);
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段一：逐类型分页拉取 cn 当前态事件并签名推送到 ov content-batches。
|
||||
* 任一类型失败抛错（整轮中止，下轮重来——幂等安全）。
|
||||
*/
|
||||
private async sweep(): Promise<{
|
||||
sweptEvents: number;
|
||||
replayApplied: number;
|
||||
}> {
|
||||
let sweptEvents = 0;
|
||||
let replayApplied = 0;
|
||||
for (const entityType of SWEEP_ENTITY_TYPES) {
|
||||
for (let page = 0; page < MAX_PAGES_PER_TYPE; page += 1) {
|
||||
const response = await this.signedPost<{
|
||||
events: SyncEventWire[];
|
||||
}>(this.cnBase, RECONCILE_PATH, {
|
||||
entityType,
|
||||
sinceVersion: 0,
|
||||
limit: this.reconcileLimit,
|
||||
});
|
||||
const events = response.events;
|
||||
sweptEvents += events.length;
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < events.length;
|
||||
offset += this.pushBatchSize
|
||||
) {
|
||||
const chunk = events.slice(offset, offset + this.pushBatchSize);
|
||||
const batch = await this.signedPost<{
|
||||
acceptedEventIds: string[];
|
||||
applied: number;
|
||||
}>(this.ovBase, CONTENT_BATCHES_PATH, { batch: chunk });
|
||||
replayApplied += batch.applied;
|
||||
}
|
||||
if (events.length < this.reconcileLimit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { sweptEvents, replayApplied };
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段二：cn digest 与 ov 副本计数比对；不一致相册触发 cn bootstrap
|
||||
* （快照写回 outbox，由常驻发送器投递——本调度器不直接投递 bootstrap 事件，
|
||||
* 与发送器职责分离）。返回比对口径供打点/测试断言。
|
||||
*/
|
||||
private async digestCompare(): Promise<{
|
||||
digestAlbums: number;
|
||||
mismatchedAlbums: string[];
|
||||
ovOrphanAlbums: string[];
|
||||
}> {
|
||||
const digest = await this.signedPost<DigestResponse>(
|
||||
this.cnBase,
|
||||
DIGEST_PATH,
|
||||
{},
|
||||
);
|
||||
const cnAlbums = new Map(digest.albums.map((row) => [row.albumId, row]));
|
||||
const ovCounts = await this.ovAlbumCounts();
|
||||
const mismatchedAlbums: string[] = [];
|
||||
for (const cnRow of digest.albums) {
|
||||
const ov = ovCounts.get(cnRow.albumId);
|
||||
if (
|
||||
ov === undefined ||
|
||||
ov.categoryCount !== cnRow.categoryCount ||
|
||||
ov.photoCount < cnRow.publishedPhotoCount ||
|
||||
ov.photoCount > cnRow.photoCount
|
||||
) {
|
||||
mismatchedAlbums.push(cnRow.albumId);
|
||||
}
|
||||
}
|
||||
const ovOrphanAlbums = [...ovCounts.keys()]
|
||||
.filter((albumId) => !cnAlbums.has(albumId))
|
||||
.sort();
|
||||
if (ovOrphanAlbums.length > 0) {
|
||||
// cn 行已删（或 ov 指向了错误源）：bootstrap 无法表达「删除相册」，
|
||||
// 不自动处置——告警登记，人工/P4a 管理通道跟进。
|
||||
this.options.logger?.error(
|
||||
{ ovOrphanAlbums },
|
||||
'digest: ov 存在 cn 已不存在的相册副本（不可自动收敛，登记跟进）',
|
||||
);
|
||||
}
|
||||
for (const albumId of mismatchedAlbums) {
|
||||
await this.signedPost(this.cnBase, BOOTSTRAP_PATH, { albumId });
|
||||
}
|
||||
if (mismatchedAlbums.length > 0) {
|
||||
this.options.logger?.warn(
|
||||
{ mismatchedAlbums },
|
||||
'digest: 副本计数不一致，已触发受影响相册 bootstrap 重建',
|
||||
);
|
||||
}
|
||||
return {
|
||||
digestAlbums: digest.albums.length,
|
||||
mismatchedAlbums,
|
||||
ovOrphanAlbums,
|
||||
};
|
||||
}
|
||||
|
||||
/** ov 每相册副本计数：albums_ov 行（categories JSON 长度）+ photos_ov 分组数。 */
|
||||
private async ovAlbumCounts(): Promise<Map<string, OvAlbumCounts>> {
|
||||
const [albumRows, photoRows] = await Promise.all([
|
||||
this.options.prismaOv.albums_ov.findMany({
|
||||
select: { albumId: true, categories: true },
|
||||
orderBy: { albumId: 'asc' },
|
||||
}),
|
||||
this.options.prismaOv.photos_ov.groupBy({
|
||||
by: ['albumId'],
|
||||
_count: { _all: true },
|
||||
}),
|
||||
]);
|
||||
const photosByAlbum = new Map(
|
||||
photoRows.map((row) => [row.albumId, row._count._all]),
|
||||
);
|
||||
const counts = new Map<string, OvAlbumCounts>();
|
||||
for (const row of albumRows) {
|
||||
counts.set(row.albumId, {
|
||||
categoryCount: Array.isArray(row.categories)
|
||||
? row.categories.length
|
||||
: 0,
|
||||
photoCount: photosByAlbum.get(row.albumId) ?? 0,
|
||||
});
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/**
|
||||
* 签名 POST：同一 JSON.stringify 产物既作签名输入也作请求体（rawBody 验签
|
||||
* 字节一致，与 sender.push 同约定）；非 2xx 抛错（调用方整轮重试语义）。
|
||||
*/
|
||||
private async signedPost<T>(
|
||||
base: string,
|
||||
path: string,
|
||||
body: unknown,
|
||||
): Promise<T> {
|
||||
const rawBody = JSON.stringify(body);
|
||||
const timestamp = Math.floor(Date.now() / 1000).toString();
|
||||
let response: Response;
|
||||
const endpoint = `${base}${path}`;
|
||||
try {
|
||||
response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
[SYNC_TIMESTAMP_HEADER]: timestamp,
|
||||
[SYNC_SIGNATURE_HEADER]: signSyncBody(this.options.internalKey, rawBody),
|
||||
},
|
||||
body: rawBody,
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
});
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`${path} 请求失败（${endpoint}）：${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '');
|
||||
throw new Error(
|
||||
`${path} 非 2xx（${endpoint}）：status=${response.status} body=${text.slice(0, 500)}`,
|
||||
);
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
// services/sync/test/reconcile.e2e-spec.ts —— P2 Task 4 对账调度 E2E。
|
||||
// 复用 sync.e2e-spec.ts 的进程内双入口 harness（cn/ov 真实 HTTP + 真实双
|
||||
// MySQL + HMAC），被测对象换成生产 SyncReconciler 实例（runOnce 直驱）。
|
||||
//
|
||||
// 覆盖（任务验收场景）：
|
||||
// ① 漏事件追赶：ov 停在 v1、cn 已到 v5 且 outbox 事件「投后即失」→
|
||||
// runOnce 当前态清扫 → ov 副本追到 v5；
|
||||
// ② digest 修复缺失副本行：直接删 photos_ov 行（不 bump 任何版本）→
|
||||
// digest 计数不一致 → 自动触发该相册 bootstrap → 发送器投递 → 行恢复；
|
||||
// ③ digest 修复错过的分类墓碑（Task 3 登记边界）：cn 删分类且墓碑 outbox
|
||||
// 行被移除（对账无法为已删行重放）→ digest categoryCount 不一致 →
|
||||
// bootstrap 的 album.updated 携带分类全量清单 → ov categories 整体替换
|
||||
// 收敛为空；
|
||||
// ④ 稳态空转：一切一致时 runOnce 无 bootstrap 触发、重放全部判重/判旧
|
||||
// （applied=0）、outbox 无新增未投递。
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { type NestExpressApplication } from '@nestjs/platform-express';
|
||||
import request from 'supertest';
|
||||
import sharp from 'sharp';
|
||||
import { PrismaClient } from '@photo/db-cn';
|
||||
import { PrismaClient as OvPrismaClient } from '@photo/db-ov';
|
||||
import {
|
||||
createStorageFromEnv,
|
||||
type MinioStorageAdapter,
|
||||
} from '@photo/storage';
|
||||
import { AppCnModule } from '../../api/src/app-cn.module';
|
||||
import { AppOvModule } from '../../api/src/app-ov.module';
|
||||
import { seedAdmin } from '../../api/src/scripts/seed-admin';
|
||||
import { configureApp } from '../../api/src/shared/configure-app';
|
||||
import { PublishService } from '../../worker/src/publish.service';
|
||||
import { resetDb } from '../../api/test/reset';
|
||||
import { resetOvDb } from '../../api/test/reset-ov';
|
||||
import { OutboxSender } from '../src/sender';
|
||||
import { SyncReconciler } from '../src/reconciler';
|
||||
|
||||
jest.setTimeout(180_000);
|
||||
|
||||
const ADMIN_ACCOUNT = process.env.ADMIN_ACCOUNT ?? 'admin';
|
||||
const ADMIN_PASSWORD = process.env.ADMIN_PASSWORD ?? 'admin-secret-change-me';
|
||||
const SYNC_KEY = process.env.SYNC_INTERNAL_KEY ?? 'dev-sync-secret-change-me';
|
||||
|
||||
/** 测试内生成真实 JPEG（同 sync.e2e-spec；无二进制入库）。 */
|
||||
async function makeJpeg(
|
||||
width: number,
|
||||
height: number,
|
||||
rgb: { r: number; g: number; b: number },
|
||||
): Promise<Buffer> {
|
||||
return sharp({ create: { width, height, channels: 3, background: rgb } })
|
||||
.jpeg({ quality: 85 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
describe('sync reconcile + digest healing (e2e)', () => {
|
||||
let cnApp: NestExpressApplication;
|
||||
let ovApp: NestExpressApplication;
|
||||
let cnBase: string;
|
||||
let ovBase: string;
|
||||
let prismaCn: PrismaClient;
|
||||
let prismaOv: OvPrismaClient;
|
||||
let storage: MinioStorageAdapter;
|
||||
let sender: OutboxSender;
|
||||
let reconciler: SyncReconciler;
|
||||
let adminToken: string;
|
||||
|
||||
let albumId: string;
|
||||
let categoryId: string;
|
||||
let photo1Id: string;
|
||||
let photo2Id: string;
|
||||
|
||||
/** 排空发送器（对账触发的 bootstrap 事件由它投递，职责分离）。 */
|
||||
async function drainSender(): Promise<void> {
|
||||
for (let round = 0; round < 20; round++) {
|
||||
const stats = await sender.runOnce();
|
||||
if (stats.picked === 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
throw new Error('drainSender 超时：20 轮后 outbox 仍有未投递事件');
|
||||
}
|
||||
|
||||
/** 上传全链路：init → presign PUT 实写 MinIO → complete → 直驱发布事务。 */
|
||||
async function uploadAndPublish(
|
||||
clientTaskId: string,
|
||||
jpeg: Buffer,
|
||||
): Promise<string> {
|
||||
const init = await request(cnBase)
|
||||
.post('/v1/uploads')
|
||||
.set('Authorization', `Bearer ${adminToken}`)
|
||||
.send({
|
||||
albumId,
|
||||
clientTaskId,
|
||||
filename: 'reconcile.jpg',
|
||||
bytes: jpeg.length,
|
||||
contentType: 'image/jpeg',
|
||||
categoryId,
|
||||
});
|
||||
expect(init.status).toBe(201);
|
||||
const { uploadId, photoId, credential } = init.body as {
|
||||
uploadId: string;
|
||||
photoId: string;
|
||||
credential: { url: string; method: string; headers: Record<string, string> };
|
||||
};
|
||||
const put = await fetch(credential.url, {
|
||||
method: credential.method,
|
||||
headers: credential.headers,
|
||||
body: new Uint8Array(jpeg),
|
||||
});
|
||||
expect(put.ok).toBe(true);
|
||||
const done = await request(cnBase)
|
||||
.post(`/v1/uploads/${uploadId}/complete`)
|
||||
.set('Authorization', `Bearer ${adminToken}`);
|
||||
expect(done.status).toBe(200);
|
||||
expect(await new PublishService(prismaCn, storage).processPhoto(photoId)).toBe(
|
||||
'published',
|
||||
);
|
||||
return photoId;
|
||||
}
|
||||
|
||||
/** 相册开关翻转一次（publicEnabled 当前值取反；版本 +1、产生 album.updated）。 */
|
||||
async function togglePublicEnabled(): Promise<void> {
|
||||
const current = await prismaCn.albums.findUniqueOrThrow({
|
||||
where: { id: albumId },
|
||||
select: { publicEnabled: true },
|
||||
});
|
||||
const res = await request(cnBase)
|
||||
.patch(`/v1/albums/${albumId}`)
|
||||
.set('Authorization', `Bearer ${adminToken}`)
|
||||
.send({ publicEnabled: !current.publicEnabled });
|
||||
expect(res.status).toBe(200);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
expect(SYNC_KEY).not.toBe('');
|
||||
await resetDb();
|
||||
await resetOvDb();
|
||||
|
||||
cnApp = await NestFactory.create<NestExpressApplication>(AppCnModule, {
|
||||
rawBody: true,
|
||||
});
|
||||
configureApp(cnApp);
|
||||
await cnApp.listen(0);
|
||||
cnBase = await cnApp.getUrl();
|
||||
|
||||
ovApp = await NestFactory.create<NestExpressApplication>(AppOvModule, {
|
||||
rawBody: true,
|
||||
});
|
||||
configureApp(ovApp);
|
||||
await ovApp.listen(0);
|
||||
ovBase = await ovApp.getUrl();
|
||||
|
||||
prismaCn = new PrismaClient();
|
||||
prismaOv = new OvPrismaClient();
|
||||
storage = createStorageFromEnv();
|
||||
sender = new OutboxSender({
|
||||
prisma: prismaCn,
|
||||
ovApiUrl: ovBase,
|
||||
internalKey: SYNC_KEY,
|
||||
});
|
||||
reconciler = new SyncReconciler({
|
||||
prismaOv,
|
||||
cnApiUrl: cnBase,
|
||||
ovApiUrl: ovBase,
|
||||
internalKey: SYNC_KEY,
|
||||
});
|
||||
|
||||
await seedAdmin(prismaCn, ADMIN_ACCOUNT, ADMIN_PASSWORD);
|
||||
const login = await request(cnBase).post('/v1/auth/password').send({
|
||||
account: ADMIN_ACCOUNT,
|
||||
password: ADMIN_PASSWORD,
|
||||
});
|
||||
expect(login.status).toBe(200);
|
||||
adminToken = login.body.sessionToken as string;
|
||||
|
||||
// 基线：相册（v1）+ 分类（v1）+ 两张发布照片（各 v2）→ ov 全量同步。
|
||||
const album = await request(cnBase)
|
||||
.post('/v1/albums')
|
||||
.set('Authorization', `Bearer ${adminToken}`)
|
||||
.send({
|
||||
translations: [
|
||||
{ locale: 'zh-CN', title: '对账相册' },
|
||||
{ locale: 'en', title: 'Reconcile Gala' },
|
||||
],
|
||||
defaultLocale: 'zh-CN',
|
||||
startAt: '2026-10-01T10:00:00.000Z',
|
||||
});
|
||||
expect(album.status).toBe(201);
|
||||
albumId = album.body.id as string;
|
||||
|
||||
const category = await request(cnBase)
|
||||
.post(`/v1/albums/${albumId}/categories`)
|
||||
.set('Authorization', `Bearer ${adminToken}`)
|
||||
.send({
|
||||
sortOrder: 0,
|
||||
translations: [
|
||||
{ locale: 'zh-CN', name: '主舞台' },
|
||||
{ locale: 'en', name: 'Main Stage' },
|
||||
],
|
||||
});
|
||||
expect(category.status).toBe(201);
|
||||
categoryId = category.body.id as string;
|
||||
|
||||
photo1Id = await uploadAndPublish(
|
||||
'recon-a',
|
||||
await makeJpeg(900, 600, { r: 170, g: 120, b: 70 }),
|
||||
);
|
||||
photo2Id = await uploadAndPublish(
|
||||
'recon-b',
|
||||
await makeJpeg(800, 600, { r: 70, g: 120, b: 170 }),
|
||||
);
|
||||
await drainSender();
|
||||
|
||||
expect(
|
||||
await prismaOv.albums_ov.findUniqueOrThrow({ where: { albumId } }),
|
||||
).toMatchObject({ entityVersion: 1, publicEnabled: true });
|
||||
expect(await prismaOv.photos_ov.count({ where: { albumId } })).toBe(2);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await cnApp.close();
|
||||
await ovApp.close();
|
||||
await prismaCn.$disconnect();
|
||||
await prismaOv.$disconnect();
|
||||
});
|
||||
|
||||
it('① 漏事件追赶：cn 到 v5、事件「投后即失」→ runOnce 清扫 → ov 追到 v5', async () => {
|
||||
// cn 连续 4 次开关翻转：v1 → v5。
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
await togglePublicEnabled();
|
||||
}
|
||||
expect(
|
||||
(
|
||||
await prismaCn.albums.findUniqueOrThrow({
|
||||
where: { id: albumId },
|
||||
select: { entityVersion: true, publicEnabled: true },
|
||||
})
|
||||
),
|
||||
).toMatchObject({ entityVersion: 5, publicEnabled: true });
|
||||
|
||||
// 模拟「已确认投递但 ov 从未收到」（发送器停机 + outbox 清扫后的漏账）：
|
||||
// 未投递事件全部伪造 deliveredAt —— 发送器永不再推，只能靠对账补。
|
||||
await prismaCn.outbox_events.updateMany({
|
||||
where: { deliveredAt: null },
|
||||
data: { deliveredAt: new Date() },
|
||||
});
|
||||
|
||||
const stats = await reconciler.runOnce();
|
||||
// 四类型当前态：album v5 + category v1 + photo×2 v2。
|
||||
expect(stats.sweptEvents).toBe(4);
|
||||
// album v5 > ov 已应用 v1 → 应用；其余同版本重断言（Task 4 语义）。
|
||||
expect(stats.replayApplied).toBe(4);
|
||||
expect(stats.mismatchedAlbums).toEqual([]);
|
||||
|
||||
expect(
|
||||
await prismaOv.albums_ov.findUniqueOrThrow({ where: { albumId } }),
|
||||
).toMatchObject({ entityVersion: 5, publicEnabled: true });
|
||||
});
|
||||
|
||||
it('② digest 修复缺失副本行：删 photos_ov 行 → bootstrap 自动触发 → 行恢复', async () => {
|
||||
// ov 侧漂移：照片行直接丢失（不 bump 任何版本——清扫的确定性 eventId
|
||||
// 已判重，唯一能发现的是 digest 计数比对）。
|
||||
await prismaOv.photos_ov.delete({ where: { photoId: photo1Id } });
|
||||
expect(await prismaOv.photos_ov.count({ where: { albumId } })).toBe(1);
|
||||
|
||||
const stats = await reconciler.runOnce();
|
||||
expect(stats.mismatchedAlbums).toEqual([albumId]);
|
||||
// bootstrap 已把该相册当前状态快照写回 outbox（发送器负责投递）。
|
||||
expect(
|
||||
await prismaCn.outbox_events.count({ where: { deliveredAt: null } }),
|
||||
).toBeGreaterThan(0);
|
||||
|
||||
await drainSender();
|
||||
|
||||
// 同版本重断言语义（sync-inbox ②，Task 4）：bootstrap 新 eventId + 同
|
||||
// entityVersion → 重应用 → 缺失行按当前态重建（photo2 未受牵连）。
|
||||
const restored = await prismaOv.photos_ov.findUniqueOrThrow({
|
||||
where: { photoId: photo1Id },
|
||||
});
|
||||
expect(restored).toMatchObject({
|
||||
albumId,
|
||||
visibility: 'published',
|
||||
entityVersion: 2,
|
||||
});
|
||||
expect(
|
||||
await prismaOv.photos_ov.findUniqueOrThrow({
|
||||
where: { photoId: photo2Id },
|
||||
}),
|
||||
).toMatchObject({ visibility: 'published', entityVersion: 2 });
|
||||
|
||||
// 收敛：再跑一轮 digest 无不一致、无新 bootstrap。
|
||||
const steady = await reconciler.runOnce();
|
||||
expect(steady.mismatchedAlbums).toEqual([]);
|
||||
expect(
|
||||
await prismaCn.outbox_events.count({ where: { deliveredAt: null } }),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it('③ digest 修复错过的分类墓碑：cn 行已删且墓碑事件丢失 → 分类全量清单整体替换收敛', async () => {
|
||||
// cn 删除分类（tombstone outbox 事件产生），随后模拟墓碑「投后即失」：
|
||||
// 直接删除 outbox 行——reconcile 清扫无法为已不存在的行重放任何事件
|
||||
// （Task 3 登记边界），ov categories 残留陈旧元素。
|
||||
const remove = await request(cnBase)
|
||||
.delete(`/v1/categories/${categoryId}`)
|
||||
.set('Authorization', `Bearer ${adminToken}`);
|
||||
expect(remove.status).toBe(204);
|
||||
await prismaCn.outbox_events.deleteMany({
|
||||
where: { type: 'category.deleted', albumId },
|
||||
});
|
||||
let replica = await prismaOv.albums_ov.findUniqueOrThrow({
|
||||
where: { albumId },
|
||||
});
|
||||
expect((replica.categories as unknown[]).length).toBe(1);
|
||||
|
||||
const stats = await reconciler.runOnce();
|
||||
expect(stats.mismatchedAlbums).toEqual([albumId]);
|
||||
await drainSender();
|
||||
|
||||
// bootstrap 的 album.updated 携带分类全量清单（此刻为空）→ ov 整体替换。
|
||||
replica = await prismaOv.albums_ov.findUniqueOrThrow({
|
||||
where: { albumId },
|
||||
});
|
||||
expect(replica.categories).toEqual([]);
|
||||
// cn 侧分类确已删净（对账收敛到权威态，非仅表面计数一致）。
|
||||
expect(
|
||||
await prismaCn.categories.count({ where: { albumId } }),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it('④ 稳态空转：一致时无 bootstrap、重放全判重（applied=0）', async () => {
|
||||
const stats = await reconciler.runOnce();
|
||||
expect(stats).toMatchObject({
|
||||
digestAlbums: 1,
|
||||
mismatchedAlbums: [],
|
||||
ovOrphanAlbums: [],
|
||||
replayApplied: 0,
|
||||
});
|
||||
// 清扫仍会拉当前态（sweptEvents>0）但全部 eventId 判重——无副作用。
|
||||
expect(stats.sweptEvents).toBeGreaterThan(0);
|
||||
expect(
|
||||
await prismaCn.outbox_events.count({ where: { deliveredAt: null } }),
|
||||
).toBe(0);
|
||||
expect(await prismaOv.photos_ov.count({ where: { albumId } })).toBe(2);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user