From 452ec12bbdb96d35708818a696fe1a18ea32a4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Fri, 18 Sep 2026 20:05:25 +0800 Subject: [PATCH] test(sync): cross-region simulation integration suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2 Task 11: 六幕串行场景套件（cn+ov 双入口进程内起真服务，真实双 MySQL/MinIO/HMAC/WS）——①端到端复制+互动+统计回传（A25 观众路径只打 ov、A27 快照替换）②隐藏传播（high 优先+WS photo.hidden+恢复发布复原） ③发送器停机积压→排空追平→对账稳态④墓碑不复活（旧 published 重放拒绝 +bootstrap 重跑仍隐藏，A26）⑤紧急停用（ov 404/WS error、cn 无感、enable 复原含新照片）⑥50 并发点赞/30 并发浏览幂等小样（终值==Σdelta、恰+15）。 services/sync devDeps 增 ws/@types/ws/ioredis（WS 场景客户端 + 限速 滑窗键清场，同 api spec 约定；版本与 api 对齐）。 Co-Authored-By: Claude Code --- pnpm-lock.yaml | 207 +++++- services/sync/package.json | 7 +- services/sync/test/cross-region.e2e-spec.ts | 735 ++++++++++++++++++++ 3 files changed, 929 insertions(+), 20 deletions(-) create mode 100644 services/sync/test/cross-region.e2e-spec.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ddd1f37..3da7a92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -447,12 +447,18 @@ importers: '@types/supertest': specifier: ^6.0.2 version: 6.0.3 + '@types/ws': + specifier: ^8.5.12 + version: 8.18.1 dotenv: specifier: ^16.4.5 version: 16.6.1 + ioredis: + specifier: ^6.0.0 + version: 6.0.0 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)) + version: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) sharp: specifier: ^0.35.0 version: 0.35.4(@types/node@22.20.3) @@ -461,10 +467,13 @@ importers: version: 7.2.2 ts-jest: specifier: ^29.2.5 - version: 29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.20.3))(typescript@5.6.3) + version: 29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)))(typescript@5.6.3) typescript: specifier: ^5.6.0 version: 5.6.3 + ws: + specifier: ^8.18.0 + version: 8.21.3 services/worker: dependencies: @@ -6783,6 +6792,41 @@ snapshots: jest-util: 29.7.0 slash: 3.0.0 + '@jest/core@29.7.0(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3))': + dependencies: + '@jest/console': 29.7.0 + '@jest/reporters': 29.7.0 + '@jest/test-result': 29.7.0 + '@jest/transform': 29.7.0 + '@jest/types': 29.6.3 + '@types/node': 24.13.5 + ansi-escapes: 4.3.2 + chalk: 4.1.2 + ci-info: 3.9.0 + exit: 0.1.2 + graceful-fs: 4.2.11 + jest-changed-files: 29.7.0 + jest-config: 29.7.0(@types/node@24.13.5)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + jest-haste-map: 29.7.0 + jest-message-util: 29.7.0 + jest-regex-util: 29.6.3 + jest-resolve: 29.7.0 + jest-resolve-dependencies: 29.7.0 + jest-runner: 29.7.0 + jest-runtime: 29.7.0 + jest-snapshot: 29.7.0 + jest-util: 29.7.0 + jest-validate: 29.7.0 + jest-watcher: 29.7.0 + micromatch: 4.0.8 + pretty-format: 29.7.0 + slash: 3.0.0 + strip-ansi: 6.0.1 + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + - ts-node + '@jest/core@29.7.0(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3))': dependencies: '@jest/console': 29.7.0 @@ -8731,6 +8775,21 @@ snapshots: object-assign: 4.1.1 vary: 1.1.2 + create-jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)): + dependencies: + '@jest/types': 29.6.3 + chalk: 4.1.2 + exit: 0.1.2 + graceful-fs: 4.2.11 + jest-config: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + jest-util: 29.7.0 + prompts: 2.4.2 + transitivePeerDependencies: + - '@types/node' + - babel-plugin-macros + - supports-color + - ts-node + create-jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)): dependencies: '@jest/types': 29.6.3 @@ -9624,6 +9683,25 @@ snapshots: - babel-plugin-macros - supports-color + jest-cli@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)): + dependencies: + '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + '@jest/test-result': 29.7.0 + '@jest/types': 29.6.3 + chalk: 4.1.2 + create-jest: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + exit: 0.1.2 + import-local: 3.2.0 + jest-config: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + jest-util: 29.7.0 + jest-validate: 29.7.0 + yargs: 17.7.3 + transitivePeerDependencies: + - '@types/node' + - babel-plugin-macros + - supports-color + - ts-node + jest-cli@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)): dependencies: '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)) @@ -9643,6 +9721,37 @@ snapshots: - supports-color - ts-node + jest-config@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)): + dependencies: + '@babel/core': 7.29.7 + '@jest/test-sequencer': 29.7.0 + '@jest/types': 29.6.3 + babel-jest: 29.7.0(@babel/core@7.29.7) + chalk: 4.1.2 + ci-info: 3.9.0 + deepmerge: 4.3.1 + glob: 7.2.3 + graceful-fs: 4.2.11 + jest-circus: 29.7.0 + jest-environment-node: 29.7.0 + jest-get-type: 29.6.3 + jest-regex-util: 29.6.3 + jest-resolve: 29.7.0 + jest-runner: 29.7.0 + jest-util: 29.7.0 + jest-validate: 29.7.0 + micromatch: 4.0.8 + parse-json: 5.2.0 + pretty-format: 29.7.0 + slash: 3.0.0 + strip-json-comments: 3.1.1 + optionalDependencies: + '@types/node': 22.20.3 + ts-node: 10.9.2(@types/node@22.20.3)(typescript@5.6.3) + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + jest-config@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)): dependencies: '@babel/core': 7.29.7 @@ -9674,6 +9783,37 @@ snapshots: - babel-plugin-macros - supports-color + jest-config@29.7.0(@types/node@24.13.5)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)): + dependencies: + '@babel/core': 7.29.7 + '@jest/test-sequencer': 29.7.0 + '@jest/types': 29.6.3 + babel-jest: 29.7.0(@babel/core@7.29.7) + chalk: 4.1.2 + ci-info: 3.9.0 + deepmerge: 4.3.1 + glob: 7.2.3 + graceful-fs: 4.2.11 + jest-circus: 29.7.0 + jest-environment-node: 29.7.0 + jest-get-type: 29.6.3 + jest-regex-util: 29.6.3 + jest-resolve: 29.7.0 + jest-runner: 29.7.0 + jest-util: 29.7.0 + jest-validate: 29.7.0 + micromatch: 4.0.8 + parse-json: 5.2.0 + pretty-format: 29.7.0 + slash: 3.0.0 + strip-json-comments: 3.1.1 + optionalDependencies: + '@types/node': 24.13.5 + ts-node: 10.9.2(@types/node@22.20.3)(typescript@5.6.3) + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + jest-config@29.7.0(@types/node@24.13.5)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)): dependencies: '@babel/core': 7.29.7 @@ -9920,6 +10060,18 @@ snapshots: merge-stream: 2.0.0 supports-color: 8.1.1 + jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)): + dependencies: + '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + '@jest/types': 29.6.3 + import-local: 3.2.0 + jest-cli: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + transitivePeerDependencies: + - '@types/node' + - babel-plugin-macros + - supports-color + - ts-node + jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)): dependencies: '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)) @@ -11224,6 +11376,26 @@ snapshots: dependencies: typescript: 5.9.3 + ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)))(typescript@5.6.3): + dependencies: + bs-logger: 0.2.6 + fast-json-stable-stringify: 2.1.0 + handlebars: 4.7.9 + jest: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3)) + json5: 2.2.3 + lodash.memoize: 4.1.2 + make-error: 1.3.6 + semver: 7.8.5 + type-fest: 4.41.0 + typescript: 5.6.3 + yargs-parser: 21.1.1 + optionalDependencies: + '@babel/core': 7.29.7 + '@jest/transform': 29.7.0 + '@jest/types': 29.6.3 + babel-jest: 29.7.0(@babel/core@7.29.7) + jest-util: 29.7.0 + ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)))(typescript@5.9.3): dependencies: bs-logger: 0.2.6 @@ -11244,25 +11416,24 @@ snapshots: babel-jest: 29.7.0(@babel/core@7.29.7) jest-util: 29.7.0 - ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.20.3))(typescript@5.6.3): + ts-node@10.9.2(@types/node@22.20.3)(typescript@5.6.3): dependencies: - bs-logger: 0.2.6 - fast-json-stable-stringify: 2.1.0 - handlebars: 4.7.9 - jest: 29.7.0(@types/node@22.20.3)(ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3)) - json5: 2.2.3 - lodash.memoize: 4.1.2 + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.13 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 22.20.3 + acorn: 8.18.0 + acorn-walk: 8.3.5 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.4 make-error: 1.3.6 - semver: 7.8.5 - type-fest: 4.41.0 typescript: 5.6.3 - yargs-parser: 21.1.1 - optionalDependencies: - '@babel/core': 7.29.7 - '@jest/transform': 29.7.0 - '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.29.7) - jest-util: 29.7.0 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + optional: true ts-node@10.9.2(@types/node@22.20.3)(typescript@5.9.3): dependencies: diff --git a/services/sync/package.json b/services/sync/package.json index 8019fe1..e5ca5cc 100644 --- a/services/sync/package.json +++ b/services/sync/package.json @@ -2,7 +2,7 @@ "name": "@photo/sync", "version": "0.1.0", "private": true, - "description": "CN→OV sync process (outbox sender + ov reconciliation scheduler; P2 Task 3/4)", + "description": "CN→OV sync process (outbox sender + ov reconciliation scheduler + stat return; P2 Task 3/4/9)", "scripts": { "build": "tsc -p tsconfig.build.json", "start": "tsc -p tsconfig.build.json && node dist/main.js", @@ -25,11 +25,14 @@ "@types/jest": "^29.5.13", "@types/node": "^22.10.0", "@types/supertest": "^6.0.2", + "@types/ws": "^8.5.12", "dotenv": "^16.4.5", + "ioredis": "^6.0.0", "jest": "^29.7.0", "sharp": "^0.35.0", "supertest": "^7.0.0", "ts-jest": "^29.2.5", - "typescript": "^5.6.0" + "typescript": "^5.6.0", + "ws": "^8.18.0" } } diff --git a/services/sync/test/cross-region.e2e-spec.ts b/services/sync/test/cross-region.e2e-spec.ts new file mode 100644 index 0000000..c50a2b7 --- /dev/null +++ b/services/sync/test/cross-region.e2e-spec.ts @@ -0,0 +1,735 @@ +// services/sync/test/cross-region.e2e-spec.ts —— P2 Task 11 跨地域全链路集成 +// 套件。复用 sync.e2e-spec.ts 的进程内双入口 harness（cn=AppCnModule、 +// ov=AppOvModule 真实 HTTP+WS 入口 + 真实双 MySQL + MinIO + HMAC 全链路字节 +// 一致），被测对象覆盖 P2 全部新增组件：OutboxSender、SyncReconciler、 +// StatReturnSender、ov 公开读/点赞/浏览端点、realtime 网关、紧急停用通道。 +// +// 与 sync/reconcile/stat-return 三套 spec 的分工：那些是**组件级**逐属性验证 +// （幂等锚、版本单调、聚合折叠、digest 语义……）；本套件是**场景级**业务闭环 +// ——六幕串行剧本推进同一相册，每幕都是一个运维/产品故事的全链路走查： +// ① 端到端：cn 建册（zh+en）+分类 → staff API 上传 3 张真实 JPEG → worker +// 发布 → 发送器 → **ov 公开面**可见（A25 本地直读：观众路径全程只打 ov +// 入口——本 spec 所有 /public/** 调用一律指向 ovBase，cn 入口仅承载 +// staff/内部动作）→ 两会话点赞 + 两访客浏览 → 统计回传 runReturnOnce → +// cn 副本 likeCount=Σdelta、viewCountReplica=2、statsSyncedAt 落位（A27）； +// ② 隐藏传播：cn 隐藏 → high 优先投递 → ov 列表即时缩面 + 已订阅 WS 客户端 +// 收到 photo.hidden 帧 → cn 恢复发布（hidden→published）→ ov 列表复原； +// ③ 中断积压恢复：发送器「停机」（本套件不装常驻循环，投递一律手工 +// drainSender——停机即不调用）→ 连发 5 张 → ov 迟滞不见（积压在 +// outbox）→ 排空 → 5 张齐现 → 对账 runOnce 稳态（无 mismatch/无孤儿）； +// ④ 墓碑不复活（A26）：photo3 隐藏后，伪造其**旧 published 事件**（更低 +// entityVersion、全新 eventId）直推 ov content-batches → 版本比较拒绝； +// 再跑全量 bootstrap（当前态快照重放）→ 仍隐藏； +// ⑤ 紧急停用：ov 本地 disable → 公开读 404 + WS subscribe error → cn 侧 +// 上传/发布/同步**无感**照常（副本照收）→ enable → 公开面复原（含停用 +// 期间新发布的照片）； +// ⑥ 并发幂等小样：50 并发点赞（25 会话×2 批）终值==Σdelta 分毫不差； +// 30 并发浏览（15 访客×2 requestId）恰 +15——窗口去重 + 唯一约束在真实 +// 并发事务下的语义验证（P5 才做容量结论）。 +// 限速滑窗键（Redis ov，TTL 24h/30min）跨 spec 残留：beforeAll SCAN 清场 +//（同 api likes-realtime/visits spec 约定），保证 ⑥ 的并发量不被前序 spec +// 的同 IP 预算挤占。 +import { randomUUID } from 'node:crypto'; +import { NestFactory } from '@nestjs/core'; +import { type NestExpressApplication } from '@nestjs/platform-express'; +import Redis from 'ioredis'; +import request from 'supertest'; +import sharp from 'sharp'; +import WebSocket from 'ws'; +import { PrismaClient } from '@photo/db-cn'; +import { PrismaClient as OvPrismaClient } from '@photo/db-ov'; +import { + type ClientMessage, + ErrorCodes, + type ServerMessage, + REALTIME_WS_PATH, + signSyncBody, +} from '@photo/contracts'; +import { + createStorageFromEnv, + type MinioStorageAdapter, +} from '@photo/storage'; +import { AppCnModule } from '../../api/src/app-cn.module'; +import { AppOvModule } from '../../api/src/app-ov.module'; +import { seedAdmin } from '../../api/src/scripts/seed-admin'; +import { configureApp } from '../../api/src/shared/configure-app'; +import { ContractWsAdapter } from '../../api/src/realtime/contract-ws.adapter'; +import { PublishService } from '../../worker/src/publish.service'; +import { resetDb } from '../../api/test/reset'; +import { resetOvDb } from '../../api/test/reset-ov'; +import { OutboxSender } from '../src/sender'; +import { SyncReconciler } from '../src/reconciler'; +import { StatReturnSender } from '../src/stat-return'; + +jest.setTimeout(300_000); + +const ADMIN_ACCOUNT = process.env.ADMIN_ACCOUNT ?? 'admin'; +const ADMIN_PASSWORD = process.env.ADMIN_PASSWORD ?? 'admin-secret-change-me'; +const SYNC_KEY = process.env.SYNC_INTERNAL_KEY ?? 'dev-sync-secret-change-me'; + +const CONTENT_BATCHES_PATH = '/v1/internal/sync/content-batches'; +const BOOTSTRAP_PATH = '/v1/internal/sync/bootstrap'; +const EMERGENCY_DISABLE_PATH = '/v1/internal/emergency/disable'; +const EMERGENCY_ENABLE_PATH = '/v1/internal/emergency/enable'; + +/** subscribe 为异步校验（DB 查询）：发送后的就绪等待（同 likes-realtime spec）。 */ +const SUBSCRIBE_SETTLE_MS = 300; + +/** 测试内生成真实 JPEG（同 sync.e2e-spec；无二进制入库）。 */ +async function makeJpeg( + width: number, + height: number, + rgb: { r: number; g: number; b: number }, +): Promise { + return sharp({ create: { width, height, channels: 3, background: rgb } }) + .jpeg({ quality: 85 }) + .toBuffer(); +} + +/** HMAC 签名 POST（内部端点：emergency/bootstrap/content-batches）。 */ +async function signedPost( + base: string, + path: string, + body: unknown, +): Promise { + const raw = JSON.stringify(body); + return fetch(`${base}${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-sync-timestamp': String(Math.floor(Date.now() / 1000)), + 'x-sync-signature': signSyncBody(SYNC_KEY, raw), + }, + body: raw, + }); +} + +/** 契约帧 WS 测试客户端（同 api likes-realtime spec：收帧队列 + 条件等待）。 */ +class WsClient { + private readonly ws: WebSocket; + private readonly frames: ServerMessage[] = []; + private readonly waiters: Array<{ + predicate: (frame: ServerMessage) => boolean; + resolve: (frame: ServerMessage) => void; + reject: (err: Error) => void; + timer: NodeJS.Timeout; + }> = []; + + private constructor(ws: WebSocket) { + this.ws = ws; + this.ws.on('message', (data) => { + let frame: ServerMessage; + try { + frame = JSON.parse(String(data)) as ServerMessage; + } catch { + return; + } + this.frames.push(frame); + const index = this.waiters.findIndex((w) => w.predicate(frame)); + if (index >= 0) { + const [waiter] = this.waiters.splice(index, 1); + clearTimeout(waiter.timer); + waiter.resolve(frame); + } + }); + } + + static async open(url: string): Promise { + const ws = new WebSocket(url); + await new Promise((resolve, reject) => { + ws.once('open', () => resolve()); + ws.once('error', (err: Error) => reject(err)); + }); + return new WsClient(ws); + } + + send(frame: ClientMessage): void { + this.ws.send(JSON.stringify(frame)); + } + + /** 等待首条满足谓词的帧（含历史帧；超时拒绝）。 */ + waitFor( + predicate: (frame: ServerMessage) => boolean, + timeoutMs: number, + label: string, + ): Promise { + const existing = this.frames.find(predicate); + if (existing !== undefined) { + return Promise.resolve(existing); + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const index = this.waiters.findIndex((w) => w.reject === reject); + if (index >= 0) { + this.waiters.splice(index, 1); + } + reject( + new Error( + `waitFor(${label}) 超时 ${timeoutMs}ms；已收帧: ${JSON.stringify(this.frames)}`, + ), + ); + }, timeoutMs); + this.waiters.push({ predicate, resolve, reject, timer }); + }); + } + + close(): void { + this.ws.close(); + } +} + +/** ov 公开列表条目（断言用子集）。 */ +interface PublicPhotoItem { + photoId: string; + likeCount: number; +} + +describe('cross-region cn↔ov full chain (e2e)', () => { + let cnApp: NestExpressApplication; + let ovApp: NestExpressApplication; + let cnBase: string; + let ovBase: string; + let prismaCn: PrismaClient; + let prismaOv: OvPrismaClient; + let storage: MinioStorageAdapter; + let sender: OutboxSender; + let reconciler: SyncReconciler; + let statReturn: StatReturnSender; + let adminToken: string; + + let albumId: string; + let publicToken: string; + let categoryId: string; + /** 前三张（场景①的 staff 上传面）。 */ + let photo1: string; + let photo2: string; + let photo3: string; + /** 场景③的 5 张积压照片（场景⑥并发点赞选其中第一张作靶）。 */ + let backlogPhotoIds: string[] = []; + + /** 排空发送器：循环 runOnce 直至无未投递事件（「常驻发送器」的测试替身）。 */ + async function drainSender(): Promise { + for (let round = 0; round < 20; round++) { + const stats = await sender.runOnce(); + if (stats.picked === 0) { + return; + } + } + throw new Error('drainSender 超时：20 轮后 outbox 仍有未投递事件'); + } + + /** 上传全链路：init → presign PUT 实写 MinIO → complete → 直驱发布事务。 */ + async function uploadAndPublish( + clientTaskId: string, + jpeg: Buffer, + ): Promise { + const init = await request(cnBase) + .post('/v1/uploads') + .set('Authorization', `Bearer ${adminToken}`) + .send({ + albumId, + clientTaskId, + filename: 'cross-region.jpg', + bytes: jpeg.length, + contentType: 'image/jpeg', + categoryId, + }); + expect(init.status).toBe(201); + const { uploadId, photoId, credential } = init.body as { + uploadId: string; + photoId: string; + credential: { url: string; method: string; headers: Record }; + }; + const put = await fetch(credential.url, { + method: credential.method, + headers: credential.headers, + body: new Uint8Array(jpeg), + }); + expect(put.ok).toBe(true); + const done = await request(cnBase) + .post(`/v1/uploads/${uploadId}/complete`) + .set('Authorization', `Bearer ${adminToken}`); + expect(done.status).toBe(200); + expect(await new PublishService(prismaCn, storage).processPhoto(photoId)).toBe( + 'published', + ); + return photoId; + } + + // —— 观众面 helper：全部只打 ov 入口（A25 本地直读语义的落地口径）—— + + async function ovAlbumStatus(): Promise { + const res = await request(ovBase).get( + `/v1/public/albums/${publicToken}`, + ); + return res.status; + } + + async function ovPhotoIds(): Promise { + const res = await request(ovBase).get( + `/v1/public/albums/${publicToken}/photos`, + ); + expect(res.status).toBe(200); + return ((res.body as { items: PublicPhotoItem[] }).items).map( + (item) => item.photoId, + ); + } + + function wsUrl(): string { + return ovBase.replace(/^http/, 'ws') + REALTIME_WS_PATH; + } + + beforeAll(async () => { + expect(SYNC_KEY).not.toBe(''); + await resetDb(); + await resetOvDb(); + // 限速滑窗键清场（同 api likes-realtime/visits spec 约定）：⑥ 的 50 点赞 + // +30 浏览并发量需要完整的 IP/会话预算。 + const redis = new Redis(process.env.REDIS_URL_OV ?? 'redis://127.0.0.1:6381'); + for (const pattern of ['like:window:*', 'visit:window:*']) { + let cursor = '0'; + do { + const [next, keys] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 1000); + cursor = next; + if (keys.length > 0) { + await redis.del(...keys); + } + } while (cursor !== '0'); + } + await redis.quit(); + + cnApp = await NestFactory.create(AppCnModule, { + rawBody: true, + }); + configureApp(cnApp); + await cnApp.listen(0); + cnBase = await cnApp.getUrl(); + + ovApp = await NestFactory.create(AppOvModule, { + rawBody: true, + }); + ovApp.useWebSocketAdapter(new ContractWsAdapter(ovApp)); + configureApp(ovApp); + await ovApp.listen(0); + ovBase = await ovApp.getUrl(); + + prismaCn = new PrismaClient(); + prismaOv = new OvPrismaClient(); + storage = createStorageFromEnv(); + sender = new OutboxSender({ + prisma: prismaCn, + ovApiUrl: ovBase, + internalKey: SYNC_KEY, + }); + reconciler = new SyncReconciler({ + prismaOv, + cnApiUrl: cnBase, + ovApiUrl: ovBase, + internalKey: SYNC_KEY, + }); + statReturn = new StatReturnSender({ + prismaOv, + cnApiUrl: cnBase, + internalKey: SYNC_KEY, + }); + + await seedAdmin(prismaCn, ADMIN_ACCOUNT, ADMIN_PASSWORD); + const login = await request(cnBase).post('/v1/auth/password').send({ + account: ADMIN_ACCOUNT, + password: ADMIN_PASSWORD, + }); + expect(login.status).toBe(200); + adminToken = login.body.sessionToken as string; + }); + + afterAll(async () => { + await cnApp.close(); + await ovApp.close(); + await prismaCn.$disconnect(); + await prismaOv.$disconnect(); + }); + + it('① 端到端：上传→发布→同步→ov 可见→点赞/浏览→回传→cn 副本一致（A25+A27）', async () => { + const album = await request(cnBase) + .post('/v1/albums') + .set('Authorization', `Bearer ${adminToken}`) + .send({ + translations: [ + { locale: 'zh-CN', title: '跨境晚会', description: '全链路验收' }, + { locale: 'en', title: 'Cross-Region Gala' }, + ], + defaultLocale: 'zh-CN', + startAt: '2026-10-01T10:00:00.000Z', + }); + expect(album.status).toBe(201); + albumId = album.body.id as string; + publicToken = album.body.publicToken as string; + + const category = await request(cnBase) + .post(`/v1/albums/${albumId}/categories`) + .set('Authorization', `Bearer ${adminToken}`) + .send({ + sortOrder: 0, + translations: [ + { locale: 'zh-CN', name: '主舞台' }, + { locale: 'en', name: 'Main Stage' }, + ], + }); + expect(category.status).toBe(201); + categoryId = category.body.id as string; + + photo1 = await uploadAndPublish( + 'cross-1', + await makeJpeg(900, 600, { r: 190, g: 90, b: 90 }), + ); + photo2 = await uploadAndPublish( + 'cross-2', + await makeJpeg(880, 600, { r: 90, g: 190, b: 90 }), + ); + photo3 = await uploadAndPublish( + 'cross-3', + await makeJpeg(860, 600, { r: 90, g: 90, b: 190 }), + ); + await drainSender(); + + // 观众面只打 ov（A25）：相册公开信息 + 列表 3 张。 + expect(await ovAlbumStatus()).toBe(200); + expect(await ovPhotoIds()).toHaveLength(3); + + // 两会话点赞 photo1（N=7）：delta 3 + 4。 + const likeA = await request(ovBase) + .post(`/v1/public/albums/${publicToken}/photos/${photo1}/likes`) + .send({ sessionId: 'cross-session-a', batchId: randomUUID(), delta: 3 }); + expect(likeA.status).toBe(200); + expect(likeA.body).toMatchObject({ total: 3, batchAccepted: true }); + const likeB = await request(ovBase) + .post(`/v1/public/albums/${publicToken}/photos/${photo1}/likes`) + .send({ sessionId: 'cross-session-b', batchId: randomUUID(), delta: 4 }); + expect(likeB.status).toBe(200); + expect(likeB.body).toMatchObject({ total: 7, batchAccepted: true }); + + // 两访客浏览 → viewCount=2（各自 counted）。 + for (const visitorId of ['cross-visitor-1', 'cross-visitor-2']) { + const res = await request(ovBase) + .post(`/v1/public/albums/${publicToken}/visits`) + .send({ visitorId, requestId: `cross-req-${visitorId}` }); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ counted: true }); + } + + // 统计回传一轮：2 likes 行 + 2 views 行（每个 counted 访客各一行）→ + // 折叠 2 快照（likes 按 photo 折叠、views 按 album 折叠）全受理（A27）。 + const ret = await statReturn.runReturnOnce(); + expect(ret).toEqual({ picked: 4, snapshots: 2, accepted: 2 }); + + // cn 副本一致：likeCount=Σdelta、viewCountReplica=2、statsSyncedAt 落位。 + const cnPhoto = await prismaCn.photos.findUniqueOrThrow({ + where: { id: photo1 }, + select: { likeCount: true }, + }); + expect(cnPhoto.likeCount).toBe(7); + const cnAlbum = await prismaCn.albums.findUniqueOrThrow({ + where: { id: albumId }, + select: { viewCountReplica: true, statsSyncedAt: true }, + }); + expect(cnAlbum.viewCountReplica).toBe(2); + expect(cnAlbum.statsSyncedAt).not.toBeNull(); + + // ov 列表 likeCount 即本地副本值（观众路径无跨境读，A25）。 + const items = await request(ovBase) + .get(`/v1/public/albums/${publicToken}/photos`); + const mine = ( + (items.body as { items: PublicPhotoItem[] }).items + ).find((item) => item.photoId === photo1); + expect(mine?.likeCount).toBe(7); + }); + + it('② 隐藏传播：cn 隐藏 → ov 列表缩面 + WS photo.hidden → 恢复发布 → 复原', async () => { + const client = await WsClient.open(wsUrl()); + client.send({ type: 'subscribe', albumToken: publicToken }); + await new Promise((resolve) => setTimeout(resolve, SUBSCRIBE_SETTLE_MS)); + + const hide = await request(cnBase) + .patch(`/v1/photos/${photo2}`) + .set('Authorization', `Bearer ${adminToken}`) + .send({ visibility: 'hidden' }); + expect(hide.status).toBe(200); + await drainSender(); + + // ov 公开面即时缩面（high 优先事件已投递）。 + const visible = await ovPhotoIds(); + expect(visible).toHaveLength(2); + expect(visible).not.toContain(photo2); + + // 已订阅客户端收到 photo.hidden 帧（realtime 网关经 OV_PHOTO_APPLIED 转发）。 + const frame = await client.waitFor( + (f) => f.type === 'photo.hidden' && f.photoId === photo2, + 10_000, + 'photo.hidden', + ); + expect(frame).toMatchObject({ + type: 'photo.hidden', + photoId: photo2, + albumId, + entityVersion: 3, + }); + + // cn 恢复发布（hidden→published，owner/admin 可操作）→ ov 复原。 + const restore = await request(cnBase) + .patch(`/v1/photos/${photo2}`) + .set('Authorization', `Bearer ${adminToken}`) + .send({ visibility: 'published' }); + expect(restore.status).toBe(200); + expect(restore.body).toMatchObject({ visibility: 'published' }); + await drainSender(); + expect(await ovPhotoIds()).toHaveLength(3); + client.close(); + }); + + it('③ 中断积压恢复：发送器停机 → 5 张积压 → 排空追平 → 对账稳态', async () => { + // 「停机」：本套件发送器无常驻循环（不调 drainSender 即停机）——连发 5 张。 + const backlog: string[] = []; + for (let i = 0; i < 5; i += 1) { + backlog.push( + await uploadAndPublish( + `cross-backlog-${i}`, + await makeJpeg(800 + i * 10, 600, { + r: 40 + i * 20, + g: 120, + b: 160, + }), + ), + ); + } + backlogPhotoIds = backlog; + + // 积压可见性：ov 迟滞不见（无任何投递），outbox 5 条未投递。 + expect(await ovPhotoIds()).toHaveLength(3); + expect( + await prismaCn.outbox_events.count({ where: { deliveredAt: null } }), + ).toBeGreaterThanOrEqual(5); + + // 恢复：排空发送器 → 5 张齐现。 + await drainSender(); + const visible = await ovPhotoIds(); + expect(visible).toHaveLength(8); + for (const photoId of backlog) { + expect(visible).toContain(photoId); + } + + // 对账稳态：digest 一致（无 bootstrap 需求、无 ov 孤儿相册）。 + const steady = await reconciler.runOnce(); + expect(steady.mismatchedAlbums).toEqual([]); + expect(steady.ovOrphanAlbums).toEqual([]); + }); + + it('④ 墓碑不复活（A26）：旧 published 重放被版本比较拒绝；bootstrap 仍隐藏', async () => { + // photo3 隐藏（v2 → v3，high）→ 投递 → ov 副本隐藏。 + const hide = await request(cnBase) + .patch(`/v1/photos/${photo3}`) + .set('Authorization', `Bearer ${adminToken}`) + .send({ visibility: 'hidden' }); + expect(hide.status).toBe(200); + await drainSender(); + expect( + await prismaOv.photos_ov.findUniqueOrThrow({ + where: { photoId: photo3 }, + select: { visibility: true, entityVersion: true }, + }), + ).toMatchObject({ visibility: 'hidden', entityVersion: 3 }); + + // 伪造重放：photo3 的旧 published 事件（v2 载荷、全新 eventId）直推 ov。 + const publishedRow = await prismaCn.outbox_events.findFirstOrThrow({ + where: { type: 'photo.published', photoId: photo3 }, + }); + expect(publishedRow.entityVersion).toBe(2); + const replayId = `cross-replay-${randomUUID()}`; + const replay = await signedPost(ovBase, CONTENT_BATCHES_PATH, { + batch: [ + { + eventId: replayId, + schemaVersion: 1, + entityType: 'photo', + entityId: photo3, + albumId, + entityVersion: publishedRow.entityVersion, + tombstone: false, + priority: 'normal', + payload: publishedRow.payload as Record, + }, + ], + }); + expect(replay.status).toBe(200); + expect(await replay.json()).toEqual({ + acceptedEventIds: [replayId], + applied: 0, + }); + expect( + await prismaOv.photos_ov.findUniqueOrThrow({ + where: { photoId: photo3 }, + select: { visibility: true }, + }), + ).toMatchObject({ visibility: 'hidden' }); + + // 全量 bootstrap 重跑（当前态快照）→ 排空 → photo3 仍隐藏。 + const boot = await signedPost(cnBase, BOOTSTRAP_PATH, { albumId }); + expect(boot.status).toBe(200); + // 当前态：album v1 + category v1 + photo1 v2 + photo2 v4（隐藏后恢复）+ + // photo3 v3（hidden）+ 5 张积压各 v2 = 10 事件；watermark = 最大版本 4。 + expect(await boot.json()).toEqual({ eventCount: 10, watermark: 4 }); + await drainSender(); + expect( + await prismaOv.photos_ov.findUniqueOrThrow({ + where: { photoId: photo3 }, + select: { visibility: true, entityVersion: true }, + }), + ).toMatchObject({ visibility: 'hidden', entityVersion: 3 }); + }); + + it('⑤ 紧急停用：ov 屏蔽（404 + WS error）→ cn 无感 → enable 复原含新照片', async () => { + const client = await WsClient.open(wsUrl()); + + const disable = await signedPost(ovBase, EMERGENCY_DISABLE_PATH, { + albumId, + reason: 'cross-region e2e: 紧急停用演练', + }); + expect(disable.status).toBe(200); + expect(await disable.json()).toEqual({ disabled: true, albumId }); + + // 观众面全屏蔽：公开读 404（统一 ALBUM_NOT_PUBLIC 防枚举语义）。 + expect(await ovAlbumStatus()).toBe(404); + const list = await request(ovBase).get( + `/v1/public/albums/${publicToken}/photos`, + ); + expect(list.status).toBe(404); + expect((list.body as { code: string }).code).toBe(ErrorCodes.ALBUM_NOT_PUBLIC); + + // WS subscribe 同一可见性口径 → error 帧（连接保持）。 + client.send({ type: 'subscribe', albumToken: publicToken }); + const error = await client.waitFor( + (f) => f.type === 'error', + 10_000, + 'subscribe error', + ); + expect(error).toMatchObject({ + type: 'error', + code: ErrorCodes.ALBUM_NOT_PUBLIC, + }); + + // cn 无感：上传/发布/同步照常（副本照收，仅公开面被 ov 本地位拦下）。 + const photo9 = await uploadAndPublish( + 'cross-emergency', + await makeJpeg(840, 600, { r: 200, g: 150, b: 50 }), + ); + await drainSender(); + expect( + await prismaOv.photos_ov.findUniqueOrThrow({ + where: { photoId: photo9 }, + select: { visibility: true }, + }), + ).toMatchObject({ visibility: 'published' }); + // 停用期间公开面仍屏蔽。 + expect(await ovAlbumStatus()).toBe(404); + + // enable：唯一恢复通道 → 公开面复原，含停用期间新发布的照片（photo3 仍 + // 处于场景④的隐藏态缺席——8 = 2 原始 + 5 积压 + photo9）。 + const enable = await signedPost(ovBase, EMERGENCY_ENABLE_PATH, { albumId }); + expect(enable.status).toBe(200); + expect(await enable.json()).toEqual({ disabled: false, albumId }); + expect(await ovAlbumStatus()).toBe(200); + const visible = await ovPhotoIds(); + expect(visible).toHaveLength(8); + expect(visible).toContain(photo9); + client.close(); + }); + + it('⑥ 并发幂等小样：50 并发点赞终值==Σdelta；30 并发浏览恰 +15', async () => { + // 50 并发点赞（25 会话 × 2 批，delta 1..3 确定性轮转）打在积压面第一张 + //（③ 落档的 backlog[0]：初始 likeTotal=0，Σdelta 即终值）。 + const targetPhotoId = backlogPhotoIds[0]; + const likePlan: Array<{ sessionId: string; batchId: string; delta: number }> = + []; + let expectedTotal = 0; + for (let i = 0; i < 50; i += 1) { + const delta = 1 + (i % 3); + expectedTotal += delta; + likePlan.push({ + sessionId: `cross-conc-sess-${Math.floor(i / 2)}`, + batchId: randomUUID(), + delta, + }); + } + + const likeResponses = await Promise.all( + likePlan.map((batch) => + fetch( + `${ovBase}/v1/public/albums/${publicToken}/photos/${targetPhotoId}/likes`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(batch), + }, + ), + ), + ); + for (const res of likeResponses) { + expect(res.status).toBe(200); + expect(((await res.json()) as { batchAccepted: boolean }).batchAccepted) + .toBe(true); + } + + // ov 权威终值 == Σdelta 分毫不差（行锁串行化 + 幂等批次，无丢失/双计）。 + const ovRow = await prismaOv.photos_ov.findUniqueOrThrow({ + where: { photoId: targetPhotoId }, + select: { likeTotal: true }, + }); + expect(ovRow.likeTotal).toBe(expectedTotal); + + // 回传后 cn 副本 == 同一权威值（快照替换，非增量累加）。 + await statReturn.runReturnOnce(); + const cnRow = await prismaCn.photos.findUniqueOrThrow({ + where: { id: targetPhotoId }, + select: { likeCount: true }, + }); + expect(cnRow.likeCount).toBe(expectedTotal); + + // 30 并发浏览（15 访客 × 2 requestId）：每访客恰计 1 → viewCount 2+15=17。 + const visitPlan: Array<{ visitorId: string; requestId: string }> = []; + for (let i = 0; i < 15; i += 1) { + visitPlan.push( + { visitorId: `cross-conc-visitor-${i}`, requestId: randomUUID() }, + { visitorId: `cross-conc-visitor-${i}`, requestId: randomUUID() }, + ); + } + const visitResponses = await Promise.all( + visitPlan.map((report) => + fetch(`${ovBase}/v1/public/albums/${publicToken}/visits`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(report), + }), + ), + ); + const counted = await Promise.all( + visitResponses.map(async (res) => { + expect(res.status).toBe(200); + return ((await res.json()) as { counted: boolean }).counted; + }), + ); + // 恰 15 个 counted=true（每访客一个；并发同访客竞态由 session 唯一约束仲裁）。 + expect(counted.filter(Boolean)).toHaveLength(15); + + const stats = await prismaOv.album_stats.findUniqueOrThrow({ + where: { albumId }, + select: { viewCount: true }, + }); + expect(stats.viewCount).toBe(17); + + await statReturn.runReturnOnce(); + const cnAlbum = await prismaCn.albums.findUniqueOrThrow({ + where: { id: albumId }, + select: { viewCountReplica: true }, + }); + expect(cnAlbum.viewCountReplica).toBe(17); + }); +});