feat(worker): sharp pipeline with orientation, variants, exif strip

Task 20: services/worker package (@photo/worker, commonjs) with jest
unit/e2e configs mirroring api conventions (root .env via test/load-env.ts,
moduleNameMapper to @photo/contracts/db-cn/storage TS sources, maxWorkers 1,
test/reset.ts = MySQL TRUNCATE + MinIO photo-live/dev prefix wipe) and the
sharp processing pipeline runPipeline(): metadata whitelist gate
(jpeg/png/webp/avif/heif/tiff, else PipelineError UNSUPPORTED_TYPE),
decode failures mapped to PipelineError DECODE_FAILED, no-arg .rotate()
auto-orientation (EXIF 5-8 swaps stored width/height), thumb <=480w q75
and browse <=1920 q82 JPEG variants with withoutEnlargement, and no
withMetadata so outputs carry no EXIF/GPS at all. sharp 0.35 prebuilt
loads script-free under pnpm 10; its win32 libvips lacks a HEVC encoder,
so the HEVC-HEIF unit case self-skips with a console note (G2 verifies
with real iPhone samples) while the AV1-HEIF case covers the same
libheif decode path / 'heif' format bucket.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-09-17 23:16:20 +08:00
parent f5f1855b58
commit 64a2cec788
10 changed files with 820 additions and 6 deletions
+26
View File
@@ -0,0 +1,26 @@
// services/worker/jest.config.cjs —— 单测(src/*.spec.ts;Task 20 pipeline 单测)。
// 镜像 services/api/jest.config.ts 的约定(CJS 包直接用 .cjs,无需 ts-node):
// · maxWorkers 1:单进程串行(e2e 配置继承后共享同一 MySQL/Redis/MinIO,
// spec 间以 test/reset.ts TRUNCATE + 前缀清理隔离,并行会互相清掉对方数据);
// · setupFiles 经 test/load-env.ts 载入仓库根 .env(全仓库唯一 env 文件);
// · @photo/contracts(ESM)/@photo/db-cn/@photo/storage(CJS)经 moduleNameMapper
// 直指 TS 源码,ts-jest 以 CJS 编译,无需先 build 依赖包 dist;
// · contracts 源码 NodeNext 风格 .js 后缀导入 → 映射回 .ts。
/** @type {import('jest').Config} */
module.exports = {
maxWorkers: 1,
testEnvironment: 'node',
rootDir: '.',
setupFiles: ['./test/load-env.ts'],
transform: {
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
},
moduleNameMapper: {
'^@photo/contracts$': '<rootDir>/../../packages/contracts/src/index.ts',
'^@photo/db-cn$': '<rootDir>/../../packages/db-cn/src/index.ts',
'^@photo/storage$': '<rootDir>/../../packages/storage/src/index.ts',
'^(\\.{1,2}/.*)\\.js$': '$1',
},
testMatch: ['<rootDir>/src/**/*.spec.ts', '<rootDir>/test/**/*.spec.ts'],
testPathIgnorePatterns: ['/node_modules/', '/dist/'],
};
+9
View File
@@ -0,0 +1,9 @@
// services/worker/jest.e2e.config.cjs —— e2e(test/*.e2e-spec.ts,Task 21 发布事务/
// 补偿扫描),复用单测配置仅切换 testMatch;隔离工具见 test/reset.ts。
/** @type {import('jest').Config} */
module.exports = {
// CJS 配置文件复用基础配置只能经 require(node10 互操作),局部豁免 lint 规则。
// eslint-disable-next-line @typescript-eslint/no-require-imports
...require('./jest.config.cjs'),
testMatch: ['<rootDir>/test/**/*.e2e-spec.ts'],
};
+32
View File
@@ -0,0 +1,32 @@
{
"name": "@photo/worker",
"version": "0.1.0",
"private": true,
"description": "CN photo worker (sharp 变体流水线、发布事务 + outbox、补偿扫描;Task 20/21)",
"scripts": {
"build": "tsc -p tsconfig.build.json",
"start": "tsc -p tsconfig.build.json && node dist/main.js",
"typecheck": "tsc --noEmit",
"lint": "eslint .",
"test": "jest",
"test:e2e": "jest --config jest.e2e.config.cjs"
},
"dependencies": {
"@photo/contracts": "workspace:*",
"@photo/db-cn": "workspace:*",
"@photo/storage": "workspace:^",
"bullmq": "^6.3.6",
"ioredis": "^6.0.0",
"pino": "^10.0.0",
"sharp": "^0.35.0"
},
"devDependencies": {
"@aws-sdk/client-s3": "^3.700.0",
"@types/jest": "^29.5.13",
"@types/node": "^22.10.0",
"dotenv": "^16.4.5",
"jest": "^29.7.0",
"ts-jest": "^29.2.5",
"typescript": "^5.6.0"
}
}
+154
View File
@@ -0,0 +1,154 @@
// services/worker/src/pipeline.spec.ts —— runPipeline 单测(Task 20,真实 sharp 缓冲)。
// 图像 fixture 全部在测试内经 sharp 生成(无二进制入库):
// · EXIF orientation 用 withMetadata({ orientation }) 构造(sharp 写入 EXIF 标签,
// 像素不旋转——正好模拟相机旁路存储);
// · HEIC 用例分两层:HEVC 压缩 HEIF 的编码能力视 sharp 预编译二进制而定
// (当前 win32 预构建 libvips 缺 hevc 编码器 → 跳过并留 console 备注,G2 真机
// iPhone 样本验收);AV1 压缩 HEIF 走同一 libheif 解码路径与 'heif' 格式桶
// (sharp metadata.format 同为 heif),可稳定覆盖流水线的 HEIF 解码分支。
import { randomBytes } from 'node:crypto';
import sharp from 'sharp';
import { PipelineError, runPipeline } from './pipeline';
/** 生成纯色 JPEG(可选 EXIF orientation)。 */
async function makeJpeg(
width: number,
height: number,
orientation?: number,
): Promise<Buffer> {
let image = sharp({
create: { width, height, channels: 3, background: { r: 200, g: 100, b: 50 } },
});
if (orientation !== undefined) {
image = image.withMetadata({ orientation });
}
return image.jpeg({ quality: 85 }).toBuffer();
}
describe('runPipeline', () => {
test('合法 JPEG:thumb ≤480w / browse ≤1920w,双 jpeg,EXIF 全剥离', async () => {
const input = await makeJpeg(2400, 1600, 1); // 携带 EXIF(orientation 1)
const out = await runPipeline(input);
expect(out.width).toBe(2400);
expect(out.height).toBe(1600);
expect(Buffer.isBuffer(out.thumb)).toBe(true);
expect(Buffer.isBuffer(out.browse)).toBe(true);
const thumbMeta = await sharp(out.thumb).metadata();
expect(thumbMeta.format).toBe('jpeg');
expect(thumbMeta.width).toBe(480);
expect(thumbMeta.height).toBe(320);
const browseMeta = await sharp(out.browse).metadata();
expect(browseMeta.format).toBe('jpeg');
expect(browseMeta.width).toBe(1920);
expect(browseMeta.height).toBe(1280);
// EXIF 剥离(GPS 等全部元数据随 EXIF 段一并去除)。
expect(thumbMeta.orientation).toBeUndefined();
expect(thumbMeta.exif).toBeUndefined();
expect(browseMeta.orientation).toBeUndefined();
expect(browseMeta.exif).toBeUndefined();
});
test('orientation 6 JPEG:自动转正,宽高交换', async () => {
// 旁路存储 400x300 + orientation 6(顺时针 90°)→ 显示 300x400。
const input = await makeJpeg(400, 300, 6);
const out = await runPipeline(input);
expect(out.width).toBe(300);
expect(out.height).toBe(400);
// 像素真的转了:thumb(≤480 不放大)即 300x400。
const thumbMeta = await sharp(out.thumb).metadata();
expect(thumbMeta.width).toBe(300);
expect(thumbMeta.height).toBe(400);
// 输出不再携带 orientation(已物理转正)。
expect(thumbMeta.orientation).toBeUndefined();
});
test('损坏缓冲(jpeg 魔数 + 随机字节)→ PipelineError(DECODE_FAILED)', async () => {
const corrupt = Buffer.concat([
Buffer.from([0xff, 0xd8, 0xff, 0xe0]),
randomBytes(8192),
]);
const err = await runPipeline(corrupt).then(
() => null,
(e: unknown) => e,
);
expect(err).toBeInstanceOf(PipelineError);
expect((err as PipelineError).code).toBe('DECODE_FAILED');
});
test('PNG 输入可处理,输出仍为 jpeg', async () => {
const input = await sharp({
create: { width: 640, height: 480, channels: 3, background: { r: 10, g: 20, b: 30 } },
})
.png()
.toBuffer();
const out = await runPipeline(input);
expect(out.width).toBe(640);
expect(out.height).toBe(480);
expect((await sharp(out.thumb).metadata()).format).toBe('jpeg');
expect((await sharp(out.browse).metadata()).format).toBe('jpeg');
});
test('小图不放大(withoutEnlargement):300px 原图 thumb/browse 均原尺寸', async () => {
const input = await makeJpeg(300, 200);
const out = await runPipeline(input);
expect(out.thumbWidth).toBe(300);
expect(out.thumbHeight).toBe(200);
expect(out.browseWidth).toBe(300);
expect(out.browseHeight).toBe(200);
});
test('白名单外格式(gif)→ PipelineError(UNSUPPORTED_TYPE)', async () => {
const gif = await sharp({
create: { width: 24, height: 24, channels: 3, background: { r: 1, g: 2, b: 3 } },
})
.gif()
.toBuffer();
const err = await runPipeline(gif).then(
() => null,
(e: unknown) => e,
);
expect(err).toBeInstanceOf(PipelineError);
expect((err as PipelineError).code).toBe('UNSUPPORTED_TYPE');
});
test('HEIF(HVEC) 输入可解码出 jpeg 变体(缺 hevc 编码器则跳过,G2 真机验证)', async () => {
let heic: Buffer;
try {
heic = await sharp({
create: { width: 300, height: 200, channels: 3, background: { r: 90, g: 120, b: 150 } },
})
.heif({ compression: 'hevc', quality: 50 })
.toBuffer();
} catch (err) {
// 预编译 libvips 不含 hevc 编码器:HEVC 压缩 HEIF(iPhone 默认)的端到端
// 验证留给 G2 真机样本;解码能力由下方 AV1-HEIF 用例覆盖同一 libheif 路径。
console.warn(
`[pipeline.spec] HEVC 编码不可用,跳过 HEVC 分支(G2 真机样本验证): ${
err instanceof Error ? err.message : String(err)
}`,
);
return;
}
const out = await runPipeline(heic);
expect(out.width).toBe(300);
expect(out.height).toBe(200);
expect((await sharp(out.browse).metadata()).format).toBe('jpeg');
});
test('HEIF(AV1) 输入(同一 heif 解码路径/格式桶)产出 jpeg 变体', async () => {
const avif = await sharp({
create: { width: 300, height: 200, channels: 3, background: { r: 90, g: 120, b: 150 } },
})
.avif({ quality: 50 })
.toBuffer();
const out = await runPipeline(avif);
expect(out.width).toBe(300);
expect(out.height).toBe(200);
expect((await sharp(out.browse).metadata()).format).toBe('jpeg');
});
});
+113
View File
@@ -0,0 +1,113 @@
// services/worker/src/pipeline.ts —— sharp 处理流水线(Task 20)。
// 输入原图 Buffer(jpeg/png/heic 等白名单格式),输出 thumb(≤480w, q75)与
// browse(≤1920w, q82)两个 JPEG 变体 + 原图尺寸。核心约定:
// · .rotate() 无参 = 按 EXIF orientation 自动转正(orientation 5-8 交换宽高);
// · 不调用 withMetadata → 输出不含任何 EXIF(含 GPS)——公开变体不携带拍摄
// 元数据(隐私),输出 metadata 的 orientation/exif 字段均为 undefined;
// · resize withoutEnlargement → 小图不放大(300px 原图的 thumb/browse 均为原尺寸);
// · 解码失败/尺寸缺失 → PipelineError('DECODE_FAILED')(processor 落
// photos.errorCode 并驱动 BullMQ 重试/终败);格式不在白名单 →
// PipelineError('UNSUPPORTED_TYPE')(与 api UPLOAD_CONTENT_TYPES 对齐,
// 以 sharp format 名表示;G2 真机 HEIC 样本验证后收口)。
import sharp from 'sharp';
import type { Metadata } from 'sharp';
export type PipelineErrorCode = 'DECODE_FAILED' | 'UNSUPPORTED_TYPE';
export class PipelineError extends Error {
constructor(
readonly code: PipelineErrorCode,
message: string,
) {
super(message);
this.name = 'PipelineError';
}
}
export interface PipelineResult {
thumb: Buffer;
browse: Buffer;
/** 原图(自动转向后)尺寸 → photos.width/height。 */
width: number;
height: number;
/** 变体实际尺寸(withoutEnlargement 下小于等于上限)→ photo_variants.width/height。 */
thumbWidth: number;
thumbHeight: number;
browseWidth: number;
browseHeight: number;
}
export const THUMB_MAX_WIDTH = 480;
export const BROWSE_MAX_WIDTH = 1920;
/** 可解码格式白名单(sharp metadata.format 取值)。 */
const SUPPORTED_FORMATS: ReadonlySet<string> = new Set([
'jpeg',
'png',
'webp',
'avif',
'heif',
'tiff',
]);
export async function runPipeline(input: Buffer): Promise<PipelineResult> {
let meta: Metadata;
try {
meta = await sharp(input).metadata();
} catch (err) {
// 头部/解码抛错一律 DECODE_FAILED(含魔数残缺、截断、随机字节拼接)。
throw new PipelineError(
'DECODE_FAILED',
`原图解码失败: ${err instanceof Error ? err.message : String(err)}`,
);
}
if (meta.format === undefined || !SUPPORTED_FORMATS.has(meta.format)) {
throw new PipelineError(
'UNSUPPORTED_TYPE',
`不支持的图片格式: ${meta.format ?? 'unknown'}`,
);
}
if (meta.width === undefined || meta.height === undefined) {
throw new PipelineError('DECODE_FAILED', '无法解析原图尺寸');
}
// EXIF orientation 5-8(90°/270° 旋转):显示尺寸交换宽高;像素级转正由
// 下方 .rotate() 在变体输出时完成(输出无 EXIF,尺寸即转向后尺寸)。
const swap = (meta.orientation ?? 1) >= 5;
const width = swap ? meta.height : meta.width;
const height = swap ? meta.width : meta.height;
let thumb: Buffer;
let browse: Buffer;
try {
thumb = await sharp(input)
.rotate()
.resize({ width: THUMB_MAX_WIDTH, withoutEnlargement: true })
.jpeg({ quality: 75 })
.toBuffer();
browse = await sharp(input)
.rotate()
.resize({ width: BROWSE_MAX_WIDTH, withoutEnlargement: true })
.jpeg({ quality: 82 })
.toBuffer();
} catch (err) {
throw new PipelineError(
'DECODE_FAILED',
`变体生成失败: ${err instanceof Error ? err.message : String(err)}`,
);
}
// 变体尺寸以输出头部为准(精确反映 withoutEnlargement/转向结果)。
const [thumbMeta, browseMeta] = await Promise.all([
sharp(thumb).metadata(),
sharp(browse).metadata(),
]);
return {
thumb,
browse,
width,
height,
thumbWidth: thumbMeta.width ?? 0,
thumbHeight: thumbMeta.height ?? 0,
browseWidth: browseMeta.width ?? 0,
browseHeight: browseMeta.height ?? 0,
};
}
+7
View File
@@ -0,0 +1,7 @@
// services/worker/test/load-env.ts
// 全仓库唯一 env 加载机制(Jest 侧):仓库根 .env(与 services/api 同约定)。
// 开发/生产经根 dotenv-cli 注入;运行时代码只读 process.env。
import { join } from 'node:path';
import { config } from 'dotenv';
config({ path: join(__dirname, '../../../.env') });
+104
View File
@@ -0,0 +1,104 @@
// services/worker/test/reset.ts
// worker e2e 共用隔离工具(镜像 services/api/test/reset.ts):每个 e2e spec 的
// beforeAll 调用 resetDb()。MySQL 逐表 TRUNCATE(SET FOREIGN_KEY_CHECKS=0 包裹,
// users 被子表引用);MinIO 侧清理 photo-live/dev 前缀对象(worker 的 variants
// 写入与 api 的 staging/originals 同前缀)。S3Client 为 devDependencies
// (与 @photo/storage 内部同款)——StoragePort 不含列举/删除(YAGNI),隔离工具
// 直连 S3 协议。
import {
DeleteObjectsCommand,
ListObjectsV2Command,
S3Client,
} from '@aws-sdk/client-s3';
import { PrismaClient } from '@photo/db-cn';
export const RESET_TABLES: readonly string[] = [
// 与 api/test/reset.ts 同表同序(子表在前)。
'outbox_events',
'photo_jobs',
'photo_variants',
'upload_sessions',
'photos',
'category_translations',
'categories',
'album_assets',
'album_translations',
'album_members',
'albums',
'audit_logs',
'sessions',
'users',
];
/** e2e 对象隔离边界:与 api src/uploads/support.ts 的 OBJECT_KEY_ROOT 一致。 */
const RESET_OBJECT_PREFIX = 'photo-live/dev/';
/** 删除 MinIO 中 RESET_OBJECT_PREFIX 前缀的全部对象(分页列举 + 批量删除)。 */
async function resetMinioObjects(): Promise<void> {
const required = [
'S3_ENDPOINT',
'S3_REGION',
'S3_ACCESS_KEY',
'S3_SECRET_KEY',
'S3_BUCKET',
] as const;
const missing = required.filter((key) => !process.env[key]);
if (missing.length > 0) {
throw new Error(`resetMinioObjects: 缺少环境变量 ${missing.join(', ')}`);
}
const client = new S3Client({
endpoint: process.env.S3_ENDPOINT,
region: process.env.S3_REGION,
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY as string,
secretAccessKey: process.env.S3_SECRET_KEY as string,
},
forcePathStyle: true,
});
const bucket = process.env.S3_BUCKET as string;
try {
for (;;) {
const listed = await client.send(
new ListObjectsV2Command({
Bucket: bucket,
Prefix: RESET_OBJECT_PREFIX,
MaxKeys: 1000,
}),
);
const objects = (listed.Contents ?? []).filter(
(obj) => typeof obj.Key === 'string',
);
if (objects.length === 0) {
break;
}
await client.send(
new DeleteObjectsCommand({
Bucket: bucket,
Delete: {
Objects: objects.map((obj) => ({ Key: obj.Key as string })),
Quiet: true,
},
}),
);
if (!listed.IsTruncated) {
break;
}
}
} finally {
await client.destroy();
}
}
export async function resetDb(): Promise<void> {
await resetMinioObjects();
const prisma = new PrismaClient();
try {
await prisma.$executeRawUnsafe('SET FOREIGN_KEY_CHECKS = 0');
for (const table of RESET_TABLES) {
await prisma.$executeRawUnsafe(`TRUNCATE TABLE ${table}`);
}
} finally {
await prisma.$executeRawUnsafe('SET FOREIGN_KEY_CHECKS = 1');
await prisma.$disconnect();
}
}
+5
View File
@@ -0,0 +1,5 @@
{
"extends": "./tsconfig.json",
"include": ["src"],
"exclude": ["node_modules", "dist", "test", "**/*.spec.ts", "**/*.e2e-spec.ts"]
}
+17
View File
@@ -0,0 +1,17 @@
{
"compilerOptions": {
"module": "commonjs",
"moduleResolution": "node10",
"target": "ES2022",
"lib": ["ES2022"],
"strict": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"skipLibCheck": true,
"declaration": false,
"sourceMap": true,
"outDir": "./dist",
"types": ["node", "jest"]
},
"include": ["src", "test"]
}