Files
cms-website/apps/api/src/modules/import/import.service.ts
T
浅念 0e8b2b5356 feat(import): job 骨架与生命周期
Plan 3 Task 1:sf import 命令骨架 + import_jobs 生命周期(空管道,Task 6 接真实管道)。

- import.service.ts(零装饰器):createImportJob——站点/快照解析(缺省按 (connection,
  kind='wxr') 选最新;三快照绑定按前置事实 13 记入报告,job 行不扩列)、幂等建行
  (默认 key=site+wxr快照+三语义版本;重复提交复用既有 job 不重跑,并发撞唯一约束
  重查收敛)、状态推进 queued→running→succeeded/failed(attempt/started_at/finished_at/
  error_code/error_summary)、报告落 artifacts/import/<jobId>/reports/import.json
  (全四版本块:adapter/mapping/sanitizer/parser;job 行落 mapping_version)
- 可预期错误带 code:SITE_NOT_FOUND / SNAPSHOT_NOT_FOUND / NO_WXR_SNAPSHOT /
  PIPELINE_FAILED——解析期失败零 job 行
- adapter 事件加法式补 postId(前置事实 12):unsupported/error 事件可选携带
  String(postId),缺失时键缺席不硬造 source_id;row 事件与报告聚合零影响、不触发版本 bump
- canonicalJson 提取至 apps/api/src/common/canonical.ts 共享导出(前置事实 6),
  register 侧改 import 零行为变更(既有清单哈希测试守护)
- sf CLI + import 命令(--site/--snapshot/--key/--connection),输出 job 摘要与版本块
- 测试:tests/integration/import-job.test.ts(5 例:生命周期观察/最新快照解析/幂等复用
  与并发收敛/可预期失败零落行/failed 路径)+ adapter postId 3 例 + canonical 单测 6 例
  (265→279 全绿);typecheck 四段链/lint 绿;临时库 CLI 冒烟(register→import→复用)通过

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-21 15:30:42 +08:00

437 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// import_jobs 生命周期服务(Plan 3 Task 1,spec §7.2/§7.5、前置事实 6/13):
// 解析站点与快照 → 幂等建 job → 状态推进(queued → running → succeeded/failed)→
// 报告落 <reportRoot>/<jobId>/reports/import.json。
//
// 本文件与 register.service.ts 同纪律保持 **零装饰器**:CLI(scripts/sf.ts 经 tsx)与
// 集成测试直接 import 本文件,不拉起 Nest 装饰器执行;Nest 侧由 import.module.ts 薄包装。
//
// 三快照绑定(前置事实 13):job.source_snapshot_id 指 **wxr 快照**(主输入);
// media_bundle / manual_url_list 不入 job 行(不扩列),按 (wxr 快照的 connection, kind)
// 解析最新记入 job 报告 snapshots 段——导入管道(Task 4/6)从报告/上下文取用。
//
// 幂等:import_jobs.idempotency_key UNIQUE——重复提交同 key **复用既有 job 不重跑**
// (含失败 job:重试须换 key 或等 Task 6 的重试语义,本任务钉死“不静默重跑”);并发
// 同 key 由唯一约束收敛(插入撞 23505 → 重查复用)。
//
// 版本块:四常量单源来自 @sf/adapter-wp 的 versions(ADAPTER/MAPPING/SANITIZER/PARSER);
// job 行只落 mapping_version(MAPPING_VERSION),全四项进报告。默认幂等键 =
// site + wxr 快照 + 三语义版本(adapter/mapping/sanitizer——任一 bump 即新 job,对齐
// spec §7.5 “版本变化可显式触发重新标准化”;parser 是诊断信息不进键)。
//
// 管道注入:Task 6 才接真实管道(terms/content/media/routes 写入器);本任务默认**空
// 管道**(直接成功),pipeline 钩子供集成测试观察 running 中间态与 failed 路径。
import { mkdir, writeFile } from 'node:fs/promises'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { and, desc, eq } from 'drizzle-orm'
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
import {
ADAPTER_VERSION,
MAPPING_VERSION,
PARSER_VERSION,
SANITIZER_VERSION,
} from '@sf/adapter-wp'
import { importJobs, sites, sourceConnections, sourceSnapshots } from '../../db/schema.js'
import type * as schema from '../../db/schema.js'
import { isUniqueViolation } from '../source/register.service.js'
/** 本文件位于 <repo>/apps/api/src/modules/import/ → 上五级是仓库根 */
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..')
export type ImportDb = NodePgDatabase<typeof schema>
export type ImportJobStatus = (typeof schema.importJobStatus.enumValues)[number]
export type ImportErrorCode =
| 'SITE_NOT_FOUND'
| 'SNAPSHOT_NOT_FOUND'
| 'NO_WXR_SNAPSHOT'
| 'PIPELINE_FAILED'
/** 导入失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */
export class ImportError extends Error {
readonly code: ImportErrorCode
constructor(code: ImportErrorCode, message: string) {
super(message)
this.name = 'ImportError'
this.code = code
}
}
/** 版本块(报告/上下文携带;job 行只落 mapping_version) */
export interface ImportVersionBlock {
adapter: string
mapping: string
sanitizer: string
parser: string
}
/** 已解析快照引用(报告与管道上下文的载荷;metadata 含 storageRoot 等登记期事实) */
export interface SnapshotRef {
id: string
kind: 'wxr' | 'media_bundle' | 'manual_url_list'
sha256: string
byteSize: number
storagePath: string
capturedAt: Date
metadata: Record<string, unknown>
}
/** 管道执行上下文:Task 6 真实管道的全部输入(站点 locale 接缝见前置事实 8) */
export interface ImportJobContext {
db: ImportDb
jobId: string
siteId: string
siteKey: string
defaultLocale: string
connectionId: string
/** 主输入:wxr 快照(job.source_snapshot_id 指向它) */
wxrSnapshot: SnapshotRef
/** 侧输入(前置事实 13):按 (connection, kind) 解析的最新快照,可能缺席(null) */
mediaBundleSnapshot: SnapshotRef | null
manualUrlListSnapshot: SnapshotRef | null
versions: ImportVersionBlock
}
/** 管道钩子(Task 6 接真实写入器;缺省空管道=直接成功)。抛错 → job failed */
export type ImportPipeline = (ctx: ImportJobContext) => Promise<void>
export interface CreateImportJobInput {
siteKey: string
/** 显式 wxr 快照 id;缺省按 (connection, kind='wxr') 选最新 */
snapshotId?: string
/** 幂等键;缺省 = site + wxr 快照 + 三语义版本 自动派生 */
idempotencyKey?: string
/** 连接名(最新快照解析的 connection 侧);缺省 'default'(与 register 缺省一致) */
connectionName?: string
/** 报告根(落 <reportRoot>/<jobId>/reports/import.json);缺省 repoRoot/artifacts/import */
reportRoot?: string
/** 管道钩子(测试/Task 6 注入);缺省空管道 */
pipeline?: ImportPipeline
}
export interface CreateImportJobResult {
jobId: string
/** job 终态(或复用时既有状态——并发窗口下可能读到中间态) */
status: ImportJobStatus
/** true = 命中既有 job(幂等键复用,未重跑管道、未改行) */
reused: boolean
siteId: string
siteKey: string
connectionId: string
wxrSnapshot: SnapshotRef
mediaBundleSnapshot: SnapshotRef | null
manualUrlListSnapshot: SnapshotRef | null
versions: ImportVersionBlock
reportPath: string | null
durationMs: number
}
const VERSIONS: ImportVersionBlock = {
adapter: ADAPTER_VERSION,
mapping: MAPPING_VERSION,
sanitizer: SANITIZER_VERSION,
parser: PARSER_VERSION,
}
// ---- 快照解析 ----
const SNAPSHOT_COLUMNS = {
id: sourceSnapshots.id,
siteId: sourceSnapshots.siteId,
kind: sourceSnapshots.kind,
sha256: sourceSnapshots.sha256,
byteSize: sourceSnapshots.byteSize,
storagePath: sourceSnapshots.storagePath,
sourceConnectionId: sourceSnapshots.sourceConnectionId,
capturedAt: sourceSnapshots.capturedAt,
metadata: sourceSnapshots.metadata,
}
type SnapshotRow = {
id: string
siteId: string
kind: 'wxr' | 'sitemap' | 'media_bundle' | 'manual_url_list'
sha256: string
byteSize: number
storagePath: string
sourceConnectionId: string
capturedAt: Date
metadata: Record<string, unknown>
}
function toRef(row: SnapshotRow): SnapshotRef {
return {
id: row.id,
// 声音窄化:调用方按 kind 过滤查询(wxr/media_bundle/manual_url_list 三值)或已过
// kind==='wxr' 校验——drizzle 列类型不携带过滤谓词,四值联合在此收窄是安全的
kind: row.kind as SnapshotRef['kind'],
sha256: row.sha256,
byteSize: row.byteSize,
storagePath: row.storagePath,
capturedAt: row.capturedAt,
metadata: row.metadata,
}
}
/** 按 (connection, kind) 解析最新快照(capturedAt 降序、createdAt 降序兜底平局);无则 null */
async function latestSnapshot(
db: ImportDb,
connectionId: string,
kind: 'wxr' | 'media_bundle' | 'manual_url_list',
): Promise<SnapshotRow | null> {
const rows = await db
.select(SNAPSHOT_COLUMNS)
.from(sourceSnapshots)
.where(and(eq(sourceSnapshots.sourceConnectionId, connectionId), eq(sourceSnapshots.kind, kind)))
.orderBy(desc(sourceSnapshots.capturedAt), desc(sourceSnapshots.createdAt))
.limit(1)
return rows.length > 0 ? (rows[0] as SnapshotRow) : null
}
// ---- 幂等键 ----
/** 默认幂等键:site + wxr 快照 + 三语义版本(定义钉死——报告可复算、CLI 可显式覆盖) */
export function defaultIdempotencyKey(siteKey: string, wxrSnapshotId: string): string {
return `import:${siteKey}:${wxrSnapshotId}:${ADAPTER_VERSION}/${MAPPING_VERSION}/${SANITIZER_VERSION}`
}
// ---- 主入口 ----
/**
* 创建(并就地推进到底)一个导入 job:
* 解析(site → wxr 快照 → 侧快照)→ 幂等建行(queued)→ running(attempt=1、started_at)
* → 空管道/注入管道 → succeeded|failed(error_code/error_summary/finished_at)→ 报告落盘。
* 失败口径:解析阶段失败零 job 行;管道阶段失败 job 行保留(failed 终态 + error 列)。
*/
export async function createImportJob(
db: ImportDb,
input: CreateImportJobInput,
): Promise<CreateImportJobResult> {
const startedAt = performance.now()
// 阶段 0:站点解析(不自动建站——建站是 register 的职责)
const siteRows = await db
.select({ id: sites.id, key: sites.key, defaultLocale: sites.defaultLocale })
.from(sites)
.where(eq(sites.key, input.siteKey))
.limit(1)
if (siteRows.length === 0) {
throw new ImportError('SITE_NOT_FOUND', `站点不存在:${input.siteKey}(先运行 sf source:register)`)
}
const site = siteRows[0]!
// 阶段 1:wxr 主输入快照解析(显式 id 优先;缺省按 (connection, kind='wxr') 最新)
let wxrRow: SnapshotRow | null = null
if (input.snapshotId !== undefined) {
const rows = await db
.select(SNAPSHOT_COLUMNS)
.from(sourceSnapshots)
.where(eq(sourceSnapshots.id, input.snapshotId))
.limit(1)
if (rows.length === 0) {
throw new ImportError('SNAPSHOT_NOT_FOUND', `快照不存在:${input.snapshotId}`)
}
const row = rows[0] as SnapshotRow
if (row.siteId !== site.id) {
throw new ImportError(
'SNAPSHOT_NOT_FOUND',
`快照 ${input.snapshotId} 不属于站点 ${input.siteKey}`,
)
}
if (row.kind !== 'wxr') {
throw new ImportError(
'NO_WXR_SNAPSHOT',
`快照 ${input.snapshotId} 的 kind 是 ${row.kind},不是 wxr——主输入必须是 wxr 快照`,
)
}
wxrRow = row
} else {
const connectionName = input.connectionName ?? 'default'
// connection 判重键 (site_id, name) 无库级唯一约束(Plan 1 决定,register 是唯一写入方)
const connRows = await db
.select({ id: sourceConnections.id })
.from(sourceConnections)
.where(and(eq(sourceConnections.siteId, site.id), eq(sourceConnections.name, connectionName)))
.limit(1)
if (connRows.length > 0) {
wxrRow = await latestSnapshot(db, connRows[0]!.id, 'wxr')
}
if (wxrRow === null) {
throw new ImportError(
'NO_WXR_SNAPSHOT',
`站点 ${input.siteKey}(connection=${connectionName})没有任何 wxr 快照——先运行 sf source:register --wxr`,
)
}
}
const connectionId = wxrRow.sourceConnectionId
// 阶段 2:侧快照(前置事实 13)——挂在 wxr 快照的 connection 下按 kind 解析最新
const mediaBundleRow = await latestSnapshot(db, connectionId, 'media_bundle')
const manualUrlListRow = await latestSnapshot(db, connectionId, 'manual_url_list')
const wxrSnapshot = toRef(wxrRow)
const mediaBundleSnapshot = mediaBundleRow === null ? null : toRef(mediaBundleRow)
const manualUrlListSnapshot = manualUrlListRow === null ? null : toRef(manualUrlListRow)
// 阶段 3:幂等建 job(queued)——命中既有 key 直接复用返回(不重跑)
const idempotencyKey = input.idempotencyKey ?? defaultIdempotencyKey(input.siteKey, wxrSnapshot.id)
const resolved = {
siteKey: input.siteKey,
connectionId,
wxrSnapshot,
mediaBundleSnapshot,
manualUrlListSnapshot,
}
const existing = await reuseExisting(db, idempotencyKey)
if (existing !== null) {
return { ...existing, ...resolved, durationMs: performance.now() - startedAt }
}
let jobId: string
try {
const inserted = await db
.insert(importJobs)
.values({
siteId: site.id,
sourceSnapshotId: wxrSnapshot.id,
status: 'queued',
mappingVersion: MAPPING_VERSION,
idempotencyKey,
})
.returning({ id: importJobs.id })
jobId = inserted[0]!.id
} catch (err) {
// 并发同 key:唯一约束收敛——重查复用(插入撞 23505 说明行已存在,重查必命中)
if (isUniqueViolation(err)) {
const raced = await reuseExisting(db, idempotencyKey)
if (raced !== null) {
return { ...raced, ...resolved, durationMs: performance.now() - startedAt }
}
}
throw err
}
// 阶段 4:状态推进 queued → running(attempt 记首次执行)
await db
.update(importJobs)
.set({ status: 'running', startedAt: new Date(), attempt: 1 })
.where(eq(importJobs.id, jobId))
// 阶段 5:管道(本任务缺省空管道;Task 6 接真实写入器)——抛错 → failed 终态
let failed: { code: string; summary: string } | null = null
const ctx: ImportJobContext = {
db,
jobId,
siteId: site.id,
siteKey: site.key,
defaultLocale: site.defaultLocale,
connectionId,
wxrSnapshot,
mediaBundleSnapshot,
manualUrlListSnapshot,
versions: VERSIONS,
}
try {
await (input.pipeline ?? emptyPipeline)(ctx)
} catch (err) {
failed = {
code: err instanceof ImportError ? err.code : 'PIPELINE_FAILED',
summary: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
}
}
// 阶段 6:终态落行(report_path 同步入行)→ 报告写盘
const reportRoot = resolve(input.reportRoot ?? join(REPO_ROOT, 'artifacts', 'import'))
const reportDir = join(reportRoot, jobId, 'reports')
const reportPath = join(reportDir, 'import.json')
const finishedAt = new Date()
const finalStatus: ImportJobStatus = failed === null ? 'succeeded' : 'failed'
await db
.update(importJobs)
.set(
failed === null
? { status: finalStatus, finishedAt, reportPath }
: {
status: finalStatus,
finishedAt,
reportPath,
errorCode: failed.code,
errorSummary: failed.summary,
},
)
.where(eq(importJobs.id, jobId))
const report = {
jobId,
generatedAt: finishedAt.toISOString(),
status: finalStatus,
siteKey: site.key,
siteId: site.id,
connectionId,
snapshots: {
wxr: slimRef(wxrSnapshot),
media_bundle: mediaBundleSnapshot === null ? null : slimRef(mediaBundleSnapshot),
manual_url_list: manualUrlListSnapshot === null ? null : slimRef(manualUrlListSnapshot),
},
versions: VERSIONS,
error: failed,
durationMs: performance.now() - startedAt,
}
await mkdir(reportDir, { recursive: true })
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8')
return {
jobId,
status: finalStatus,
reused: false,
siteId: site.id,
siteKey: site.key,
connectionId,
wxrSnapshot,
mediaBundleSnapshot,
manualUrlListSnapshot,
versions: VERSIONS,
reportPath,
durationMs: performance.now() - startedAt,
}
}
/** 既有 job 的复用返回(不动行、不跑管道);无则 null */
async function reuseExisting(
db: ImportDb,
idempotencyKey: string,
): Promise<Pick<CreateImportJobResult, 'jobId' | 'status' | 'reused' | 'siteId' | 'reportPath' | 'versions'> | null> {
const found = await db
.select({
id: importJobs.id,
status: importJobs.status,
siteId: importJobs.siteId,
reportPath: importJobs.reportPath,
})
.from(importJobs)
.where(eq(importJobs.idempotencyKey, idempotencyKey))
.limit(1)
if (found.length === 0) return null
const row = found[0]!
return {
jobId: row.id,
status: row.status,
reused: true,
siteId: row.siteId,
reportPath: row.reportPath,
versions: VERSIONS,
}
}
/** 报告里的快照引用瘦身版:不含 metadata(media_bundle 清单可达数千条,报告只留身份锚点) */
function slimRef(ref: SnapshotRef): {
id: string
kind: string
sha256: string
byteSize: number
storagePath: string
} {
return { id: ref.id, kind: ref.kind, sha256: ref.sha256, byteSize: ref.byteSize, storagePath: ref.storagePath }
}
/** Task 1 空管道:直接成功(Task 6 以真实写入器替换缺省值) */
async function emptyPipeline(_ctx: ImportJobContext): Promise<void> {}