// import_jobs 生命周期服务(Plan 3 Task 1,spec §7.2/§7.5、前置事实 6/13): // 解析站点与快照 → 幂等建 job → 状态推进(queued → running → succeeded/failed)→ // 报告落 //reports/import.json。 // // 本文件与 register.service.ts 同纪律保持 **零装饰器**:CLI(scripts/sf.ts 经 tsx)与 // 集成测试直接 import 本文件,不拉起 Nest 装饰器执行;Nest 侧由 import.module.ts 薄包装。 // // 三快照绑定(前置事实 13):job.source_snapshot_id 指 **wxr 快照**(主输入); // media_bundle / manual_url_list 不入 job 行(不扩列),按 (wxr 快照的 connection, kind) // 解析最新记入 job 报告 snapshots 段——导入管道(Task 4/6)从报告/上下文取用。 // // 幂等:import_jobs.idempotency_key UNIQUE——重复提交同 key **复用既有 job 不重跑** // (含失败 job:重试须换 key 或等 Task 6 的重试语义,本任务钉死“不静默重跑”);并发 // 同 key 由唯一约束收敛(插入撞 23505 → 重查复用)。 // // 版本块:四常量单源来自 @sf/adapter-wp 的 versions(ADAPTER/MAPPING/SANITIZER/PARSER); // job 行只落 mapping_version(MAPPING_VERSION),全四项进报告。默认幂等键 = // site + wxr 快照 + 三语义版本(adapter/mapping/sanitizer——任一 bump 即新 job,对齐 // spec §7.5 “版本变化可显式触发重新标准化”;parser 是诊断信息不进键)。 // // 管道注入:Task 6 才接真实管道(terms/content/media/routes 写入器);本任务默认**空 // 管道**(直接成功),pipeline 钩子供集成测试观察 running 中间态与 failed 路径。 import { mkdir, writeFile } from 'node:fs/promises' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { and, desc, eq } from 'drizzle-orm' import type { NodePgDatabase } from 'drizzle-orm/node-postgres' import { ADAPTER_VERSION, MAPPING_VERSION, PARSER_VERSION, SANITIZER_VERSION, } from '@sf/adapter-wp' import { importJobs, sites, sourceConnections, sourceSnapshots } from '../../db/schema.js' import type * as schema from '../../db/schema.js' import { isUniqueViolation } from '../source/register.service.js' /** 本文件位于 /apps/api/src/modules/import/ → 上五级是仓库根 */ const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..') export type ImportDb = NodePgDatabase export type ImportJobStatus = (typeof schema.importJobStatus.enumValues)[number] export type ImportErrorCode = | 'SITE_NOT_FOUND' | 'SNAPSHOT_NOT_FOUND' | 'NO_WXR_SNAPSHOT' | 'PIPELINE_FAILED' /** 导入失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */ export class ImportError extends Error { readonly code: ImportErrorCode constructor(code: ImportErrorCode, message: string) { super(message) this.name = 'ImportError' this.code = code } } /** 版本块(报告/上下文携带;job 行只落 mapping_version) */ export interface ImportVersionBlock { adapter: string mapping: string sanitizer: string parser: string } /** 已解析快照引用(报告与管道上下文的载荷;metadata 含 storageRoot 等登记期事实) */ export interface SnapshotRef { id: string kind: 'wxr' | 'media_bundle' | 'manual_url_list' sha256: string byteSize: number storagePath: string capturedAt: Date metadata: Record } /** 管道执行上下文:Task 6 真实管道的全部输入(站点 locale 接缝见前置事实 8) */ export interface ImportJobContext { db: ImportDb jobId: string siteId: string siteKey: string defaultLocale: string connectionId: string /** 主输入:wxr 快照(job.source_snapshot_id 指向它) */ wxrSnapshot: SnapshotRef /** 侧输入(前置事实 13):按 (connection, kind) 解析的最新快照,可能缺席(null) */ mediaBundleSnapshot: SnapshotRef | null manualUrlListSnapshot: SnapshotRef | null versions: ImportVersionBlock } /** 管道钩子(Task 6 接真实写入器;缺省空管道=直接成功)。抛错 → job failed */ export type ImportPipeline = (ctx: ImportJobContext) => Promise export interface CreateImportJobInput { siteKey: string /** 显式 wxr 快照 id;缺省按 (connection, kind='wxr') 选最新 */ snapshotId?: string /** 幂等键;缺省 = site + wxr 快照 + 三语义版本 自动派生 */ idempotencyKey?: string /** 连接名(最新快照解析的 connection 侧);缺省 'default'(与 register 缺省一致) */ connectionName?: string /** 报告根(落 //reports/import.json);缺省 repoRoot/artifacts/import */ reportRoot?: string /** 管道钩子(测试/Task 6 注入);缺省空管道 */ pipeline?: ImportPipeline } export interface CreateImportJobResult { jobId: string /** job 终态(或复用时既有状态——并发窗口下可能读到中间态) */ status: ImportJobStatus /** true = 命中既有 job(幂等键复用,未重跑管道、未改行) */ reused: boolean siteId: string siteKey: string connectionId: string wxrSnapshot: SnapshotRef mediaBundleSnapshot: SnapshotRef | null manualUrlListSnapshot: SnapshotRef | null versions: ImportVersionBlock reportPath: string | null durationMs: number } const VERSIONS: ImportVersionBlock = { adapter: ADAPTER_VERSION, mapping: MAPPING_VERSION, sanitizer: SANITIZER_VERSION, parser: PARSER_VERSION, } // ---- 快照解析 ---- const SNAPSHOT_COLUMNS = { id: sourceSnapshots.id, siteId: sourceSnapshots.siteId, kind: sourceSnapshots.kind, sha256: sourceSnapshots.sha256, byteSize: sourceSnapshots.byteSize, storagePath: sourceSnapshots.storagePath, sourceConnectionId: sourceSnapshots.sourceConnectionId, capturedAt: sourceSnapshots.capturedAt, metadata: sourceSnapshots.metadata, } type SnapshotRow = { id: string siteId: string kind: 'wxr' | 'sitemap' | 'media_bundle' | 'manual_url_list' sha256: string byteSize: number storagePath: string sourceConnectionId: string capturedAt: Date metadata: Record } function toRef(row: SnapshotRow): SnapshotRef { return { id: row.id, // 声音窄化:调用方按 kind 过滤查询(wxr/media_bundle/manual_url_list 三值)或已过 // kind==='wxr' 校验——drizzle 列类型不携带过滤谓词,四值联合在此收窄是安全的 kind: row.kind as SnapshotRef['kind'], sha256: row.sha256, byteSize: row.byteSize, storagePath: row.storagePath, capturedAt: row.capturedAt, metadata: row.metadata, } } /** 按 (connection, kind) 解析最新快照(capturedAt 降序、createdAt 降序兜底平局);无则 null */ async function latestSnapshot( db: ImportDb, connectionId: string, kind: 'wxr' | 'media_bundle' | 'manual_url_list', ): Promise { const rows = await db .select(SNAPSHOT_COLUMNS) .from(sourceSnapshots) .where(and(eq(sourceSnapshots.sourceConnectionId, connectionId), eq(sourceSnapshots.kind, kind))) .orderBy(desc(sourceSnapshots.capturedAt), desc(sourceSnapshots.createdAt)) .limit(1) return rows.length > 0 ? (rows[0] as SnapshotRow) : null } // ---- 幂等键 ---- /** 默认幂等键:site + wxr 快照 + 三语义版本(定义钉死——报告可复算、CLI 可显式覆盖) */ export function defaultIdempotencyKey(siteKey: string, wxrSnapshotId: string): string { return `import:${siteKey}:${wxrSnapshotId}:${ADAPTER_VERSION}/${MAPPING_VERSION}/${SANITIZER_VERSION}` } // ---- 主入口 ---- /** * 创建(并就地推进到底)一个导入 job: * 解析(site → wxr 快照 → 侧快照)→ 幂等建行(queued)→ running(attempt=1、started_at) * → 空管道/注入管道 → succeeded|failed(error_code/error_summary/finished_at)→ 报告落盘。 * 失败口径:解析阶段失败零 job 行;管道阶段失败 job 行保留(failed 终态 + error 列)。 */ export async function createImportJob( db: ImportDb, input: CreateImportJobInput, ): Promise { const startedAt = performance.now() // 阶段 0:站点解析(不自动建站——建站是 register 的职责) const siteRows = await db .select({ id: sites.id, key: sites.key, defaultLocale: sites.defaultLocale }) .from(sites) .where(eq(sites.key, input.siteKey)) .limit(1) if (siteRows.length === 0) { throw new ImportError('SITE_NOT_FOUND', `站点不存在:${input.siteKey}(先运行 sf source:register)`) } const site = siteRows[0]! // 阶段 1:wxr 主输入快照解析(显式 id 优先;缺省按 (connection, kind='wxr') 最新) let wxrRow: SnapshotRow | null = null if (input.snapshotId !== undefined) { const rows = await db .select(SNAPSHOT_COLUMNS) .from(sourceSnapshots) .where(eq(sourceSnapshots.id, input.snapshotId)) .limit(1) if (rows.length === 0) { throw new ImportError('SNAPSHOT_NOT_FOUND', `快照不存在:${input.snapshotId}`) } const row = rows[0] as SnapshotRow if (row.siteId !== site.id) { throw new ImportError( 'SNAPSHOT_NOT_FOUND', `快照 ${input.snapshotId} 不属于站点 ${input.siteKey}`, ) } if (row.kind !== 'wxr') { throw new ImportError( 'NO_WXR_SNAPSHOT', `快照 ${input.snapshotId} 的 kind 是 ${row.kind},不是 wxr——主输入必须是 wxr 快照`, ) } wxrRow = row } else { const connectionName = input.connectionName ?? 'default' // connection 判重键 (site_id, name) 无库级唯一约束(Plan 1 决定,register 是唯一写入方) const connRows = await db .select({ id: sourceConnections.id }) .from(sourceConnections) .where(and(eq(sourceConnections.siteId, site.id), eq(sourceConnections.name, connectionName))) .limit(1) if (connRows.length > 0) { wxrRow = await latestSnapshot(db, connRows[0]!.id, 'wxr') } if (wxrRow === null) { throw new ImportError( 'NO_WXR_SNAPSHOT', `站点 ${input.siteKey}(connection=${connectionName})没有任何 wxr 快照——先运行 sf source:register --wxr`, ) } } const connectionId = wxrRow.sourceConnectionId // 阶段 2:侧快照(前置事实 13)——挂在 wxr 快照的 connection 下按 kind 解析最新 const mediaBundleRow = await latestSnapshot(db, connectionId, 'media_bundle') const manualUrlListRow = await latestSnapshot(db, connectionId, 'manual_url_list') const wxrSnapshot = toRef(wxrRow) const mediaBundleSnapshot = mediaBundleRow === null ? null : toRef(mediaBundleRow) const manualUrlListSnapshot = manualUrlListRow === null ? null : toRef(manualUrlListRow) // 阶段 3:幂等建 job(queued)——命中既有 key 直接复用返回(不重跑) const idempotencyKey = input.idempotencyKey ?? defaultIdempotencyKey(input.siteKey, wxrSnapshot.id) const resolved = { siteKey: input.siteKey, connectionId, wxrSnapshot, mediaBundleSnapshot, manualUrlListSnapshot, } const existing = await reuseExisting(db, idempotencyKey) if (existing !== null) { return { ...existing, ...resolved, durationMs: performance.now() - startedAt } } let jobId: string try { const inserted = await db .insert(importJobs) .values({ siteId: site.id, sourceSnapshotId: wxrSnapshot.id, status: 'queued', mappingVersion: MAPPING_VERSION, idempotencyKey, }) .returning({ id: importJobs.id }) jobId = inserted[0]!.id } catch (err) { // 并发同 key:唯一约束收敛——重查复用(插入撞 23505 说明行已存在,重查必命中) if (isUniqueViolation(err)) { const raced = await reuseExisting(db, idempotencyKey) if (raced !== null) { return { ...raced, ...resolved, durationMs: performance.now() - startedAt } } } throw err } // 阶段 4:状态推进 queued → running(attempt 记首次执行) await db .update(importJobs) .set({ status: 'running', startedAt: new Date(), attempt: 1 }) .where(eq(importJobs.id, jobId)) // 阶段 5:管道(本任务缺省空管道;Task 6 接真实写入器)——抛错 → failed 终态 let failed: { code: string; summary: string } | null = null const ctx: ImportJobContext = { db, jobId, siteId: site.id, siteKey: site.key, defaultLocale: site.defaultLocale, connectionId, wxrSnapshot, mediaBundleSnapshot, manualUrlListSnapshot, versions: VERSIONS, } try { await (input.pipeline ?? emptyPipeline)(ctx) } catch (err) { failed = { code: err instanceof ImportError ? err.code : 'PIPELINE_FAILED', summary: err instanceof Error ? `${err.name}: ${err.message}` : String(err), } } // 阶段 6:终态落行(report_path 同步入行)→ 报告写盘 const reportRoot = resolve(input.reportRoot ?? join(REPO_ROOT, 'artifacts', 'import')) const reportDir = join(reportRoot, jobId, 'reports') const reportPath = join(reportDir, 'import.json') const finishedAt = new Date() const finalStatus: ImportJobStatus = failed === null ? 'succeeded' : 'failed' await db .update(importJobs) .set( failed === null ? { status: finalStatus, finishedAt, reportPath } : { status: finalStatus, finishedAt, reportPath, errorCode: failed.code, errorSummary: failed.summary, }, ) .where(eq(importJobs.id, jobId)) const report = { jobId, generatedAt: finishedAt.toISOString(), status: finalStatus, siteKey: site.key, siteId: site.id, connectionId, snapshots: { wxr: slimRef(wxrSnapshot), media_bundle: mediaBundleSnapshot === null ? null : slimRef(mediaBundleSnapshot), manual_url_list: manualUrlListSnapshot === null ? null : slimRef(manualUrlListSnapshot), }, versions: VERSIONS, error: failed, durationMs: performance.now() - startedAt, } await mkdir(reportDir, { recursive: true }) await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8') return { jobId, status: finalStatus, reused: false, siteId: site.id, siteKey: site.key, connectionId, wxrSnapshot, mediaBundleSnapshot, manualUrlListSnapshot, versions: VERSIONS, reportPath, durationMs: performance.now() - startedAt, } } /** 既有 job 的复用返回(不动行、不跑管道);无则 null */ async function reuseExisting( db: ImportDb, idempotencyKey: string, ): Promise | null> { const found = await db .select({ id: importJobs.id, status: importJobs.status, siteId: importJobs.siteId, reportPath: importJobs.reportPath, }) .from(importJobs) .where(eq(importJobs.idempotencyKey, idempotencyKey)) .limit(1) if (found.length === 0) return null const row = found[0]! return { jobId: row.id, status: row.status, reused: true, siteId: row.siteId, reportPath: row.reportPath, versions: VERSIONS, } } /** 报告里的快照引用瘦身版:不含 metadata(media_bundle 清单可达数千条,报告只留身份锚点) */ function slimRef(ref: SnapshotRef): { id: string kind: string sha256: string byteSize: number storagePath: string } { return { id: ref.id, kind: ref.kind, sha256: ref.sha256, byteSize: ref.byteSize, storagePath: ref.storagePath } } /** Task 1 空管道:直接成功(Task 6 以真实写入器替换缺省值) */ async function emptyPipeline(_ctx: ImportJobContext): Promise {}