Files
photo-live/services/worker/src/process.processor.ts
T
浅念 94357e9f03 feat(worker): publish transaction with outbox event and compensating sweep
Task 21: PublishService.processPhoto streams the original from storage,
runs the sharp pipeline, puts thumb/browse variants under
photo-live/dev/albums/{albumId}/variants/{photoId}/1/, then commits ONE
Prisma transaction: photo_variants upserts (thumb+browse), a conditional
photos update WHERE processingState IN (uploaded, processing_failed) that
flips to ready+published, stamps publishedAt/width/height, clears
errorCode and bumps entityVersion, and the outbox_events row
(photo.published, payload = contracts PhotoPublishedPayload with the
post-increment entityVersion). A zero-row conditional update means a
concurrent winner already published -> idempotent 'already' with no
second event and no extra version bump. The BullMQ worker wrapper
(QUEUE_PHOTO_PROCESS, concurrency 2, jobId 'process-{photoId}' without
the reserved ':') claims photo_jobs pending->active (attempt++), marks
done for published/already/skipped alike, and rethrows pipeline/storage
errors for the producer-configured 3-attempt exponential backoff; the
final attempt (BullMQ increments attemptsMade only after
moveToFinished, so in-processor it is 0-based: current = attemptsMade+1)
marks photo_jobs failed + photo processing_failed with the
PipelineError code while visibility stays unpublished. sweepOnce()
claims due pending rows (nextRetryAt IS NULL OR < now) and re-enqueues
stale (>5min) active rows with jobId dedupe; startSweepInterval(30s)
boot-scans then loops, and main.ts wires pino logging, dedicated redis
connections and graceful SIGINT/SIGTERM shutdown.

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-09-17 23:21:57 +08:00

123 lines
4.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// services/worker/src/process.processor.ts —— QUEUE_PHOTO_PROCESS 消费者(Task 21)。
// BullMQ Worker 包装 PublishService:
// · 每次投递将 photo_jobs pending→active(attempt 自增),成功(含幂等 already/
// 跳过 skipped)置 done;失败原样抛给 BullMQ 按生产者配置重试(attempts:3,
// 指数退避 1s 起——与 api queue.provider.ts 一致);
// · 终败判定:BullMQ 在 moveToFailed/moveToCompleted 之后才自增 attemptsMade
// (v6 classes/job.js),processor 内它恒为「已完成尝试数」(0 基),故当前
// 尝试 = attemptsMade + 1;等于 attempts 上限即终败 → 单事务置
// photo_jobs.failed(errorCode) + photos.processing_failed(visibility 不动,
// 保持 unpublished——失败照片不公开),PipelineError 携带精确错误码;
// · Worker 使用专用 ioredis 连接(阻塞命令),连接生命周期由调用方管理
// (Task 19 gotcha:BullMQ 不接管外部连接;未监听的 error 事件会崩进程)。
import { Worker, type Job } from 'bullmq';
import type IORedis from 'ioredis';
import type { PrismaClient } from '@photo/db-cn';
import type { StoragePort } from '@photo/storage';
import type { Logger } from 'pino';
import { QUEUE_PHOTO_PROCESS } from '@photo/contracts';
import { PipelineError } from './pipeline';
import { PublishService, type ProcessOutcome } from './publish.service';
export type { ProcessOutcome } from './publish.service';
export interface PhotoProcessJobData {
photoId: string;
}
/** job 名称(与 api 生产者/photo_jobs.type 同值)。 */
export const PHOTO_PROCESS_JOB_NAME = 'process';
/**
* jobId 以 photoId 派生:并发投递/补偿扫描重复入队天然去重(BullMQ 同 jobId
* 不重复执行)。不含 ':'(BullMQ 保留分隔符,Task 19 gotcha)。
*/
export function photoProcessJobId(photoId: string): string {
return `${PHOTO_PROCESS_JOB_NAME}-${photoId}`;
}
/** 与 api 生产者一致的重试策略(sweep 补偿入队复用同一组值)。 */
export const PROCESS_JOB_ATTEMPTS = 3;
export const PROCESS_JOB_BACKOFF = { type: 'exponential', delay: 1000 } as const;
/** 终败errorCode 兜底(非流水线错误:对象缺失、网络等)。 */
export const FALLBACK_ERROR_CODE = 'PROCESS_FAILED';
export interface PhotoProcessWorkerOptions {
prisma: PrismaClient;
storage: StoragePort;
/** Worker 专用连接(阻塞命令),调用方负责 quit。 */
connection: IORedis;
concurrency?: number;
logger?: Logger;
}
export function createPhotoProcessWorker(
options: PhotoProcessWorkerOptions,
): Worker<PhotoProcessJobData, ProcessOutcome, string> {
const { prisma } = options;
const publish = new PublishService(prisma, options.storage, options.logger);
const processor = async (
job: Job<PhotoProcessJobData>,
): Promise<ProcessOutcome> => {
const photoId = job.data.photoId;
// 投递即认领:pending→active(sweep 已置 active 的重复投递幂等),attempt 计数。
await prisma.photo_jobs.updateMany({
where: {
photoId,
type: PHOTO_PROCESS_JOB_NAME,
state: { in: ['pending', 'active'] },
},
data: { state: 'active', attempt: { increment: 1 } },
});
try {
const outcome = await publish.processPhoto(photoId);
// published/already/skipped 对 job 而言都是「无需再投递」→ done。
await prisma.photo_jobs.updateMany({
where: {
photoId,
type: PHOTO_PROCESS_JOB_NAME,
state: { in: ['pending', 'active'] },
},
data: { state: 'done', errorCode: null, nextRetryAt: null },
});
options.logger?.info({ photoId, jobId: job.id, outcome }, 'process job 完成');
return outcome;
} catch (err) {
const isFinalAttempt = job.attemptsMade + 1 >= (job.opts.attempts ?? 1);
if (isFinalAttempt) {
const errorCode =
err instanceof PipelineError ? err.code : FALLBACK_ERROR_CODE;
options.logger?.error(
{ photoId, jobId: job.id, errorCode },
'process job 终败(重试耗尽)',
);
await prisma.$transaction([
prisma.photo_jobs.updateMany({
where: { photoId, type: PHOTO_PROCESS_JOB_NAME },
data: { state: 'failed', errorCode, nextRetryAt: null },
}),
prisma.photos.updateMany({
where: {
id: photoId,
processingState: { in: ['uploaded', 'processing_failed'] },
},
data: { processingState: 'processing_failed', errorCode },
}),
]);
}
throw err; // 交还 BullMQ:未耗尽则按退避重试,耗尽则入 failed set
}
};
return new Worker<PhotoProcessJobData, ProcessOutcome, string>(
QUEUE_PHOTO_PROCESS,
processor,
{
connection: options.connection,
concurrency: options.concurrency ?? 2,
},
);
}