0e8b2b5356
Plan 3 Task 1:sf import 命令骨架 + import_jobs 生命周期(空管道,Task 6 接真实管道)。 - import.service.ts(零装饰器):createImportJob——站点/快照解析(缺省按 (connection, kind='wxr') 选最新;三快照绑定按前置事实 13 记入报告,job 行不扩列)、幂等建行 (默认 key=site+wxr快照+三语义版本;重复提交复用既有 job 不重跑,并发撞唯一约束 重查收敛)、状态推进 queued→running→succeeded/failed(attempt/started_at/finished_at/ error_code/error_summary)、报告落 artifacts/import/<jobId>/reports/import.json (全四版本块:adapter/mapping/sanitizer/parser;job 行落 mapping_version) - 可预期错误带 code:SITE_NOT_FOUND / SNAPSHOT_NOT_FOUND / NO_WXR_SNAPSHOT / PIPELINE_FAILED——解析期失败零 job 行 - adapter 事件加法式补 postId(前置事实 12):unsupported/error 事件可选携带 String(postId),缺失时键缺席不硬造 source_id;row 事件与报告聚合零影响、不触发版本 bump - canonicalJson 提取至 apps/api/src/common/canonical.ts 共享导出(前置事实 6), register 侧改 import 零行为变更(既有清单哈希测试守护) - sf CLI + import 命令(--site/--snapshot/--key/--connection),输出 job 摘要与版本块 - 测试:tests/integration/import-job.test.ts(5 例:生命周期观察/最新快照解析/幂等复用 与并发收敛/可预期失败零落行/failed 路径)+ adapter postId 3 例 + canonical 单测 6 例 (265→279 全绿);typecheck 四段链/lint 绿;临时库 CLI 冒烟(register→import→复用)通过 Co-Authored-By: Claude Code <noreply@anthropic.com>
437 lines
16 KiB
TypeScript
437 lines
16 KiB
TypeScript
// import_jobs 生命周期服务(Plan 3 Task 1,spec §7.2/§7.5、前置事实 6/13):
|
||
// 解析站点与快照 → 幂等建 job → 状态推进(queued → running → succeeded/failed)→
|
||
// 报告落 <reportRoot>/<jobId>/reports/import.json。
|
||
//
|
||
// 本文件与 register.service.ts 同纪律保持 **零装饰器**:CLI(scripts/sf.ts 经 tsx)与
|
||
// 集成测试直接 import 本文件,不拉起 Nest 装饰器执行;Nest 侧由 import.module.ts 薄包装。
|
||
//
|
||
// 三快照绑定(前置事实 13):job.source_snapshot_id 指 **wxr 快照**(主输入);
|
||
// media_bundle / manual_url_list 不入 job 行(不扩列),按 (wxr 快照的 connection, kind)
|
||
// 解析最新记入 job 报告 snapshots 段——导入管道(Task 4/6)从报告/上下文取用。
|
||
//
|
||
// 幂等:import_jobs.idempotency_key UNIQUE——重复提交同 key **复用既有 job 不重跑**
|
||
// (含失败 job:重试须换 key 或等 Task 6 的重试语义,本任务钉死“不静默重跑”);并发
|
||
// 同 key 由唯一约束收敛(插入撞 23505 → 重查复用)。
|
||
//
|
||
// 版本块:四常量单源来自 @sf/adapter-wp 的 versions(ADAPTER/MAPPING/SANITIZER/PARSER);
|
||
// job 行只落 mapping_version(MAPPING_VERSION),全四项进报告。默认幂等键 =
|
||
// site + wxr 快照 + 三语义版本(adapter/mapping/sanitizer——任一 bump 即新 job,对齐
|
||
// spec §7.5 “版本变化可显式触发重新标准化”;parser 是诊断信息不进键)。
|
||
//
|
||
// 管道注入:Task 6 才接真实管道(terms/content/media/routes 写入器);本任务默认**空
|
||
// 管道**(直接成功),pipeline 钩子供集成测试观察 running 中间态与 failed 路径。
|
||
import { mkdir, writeFile } from 'node:fs/promises'
|
||
import { dirname, join, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { and, desc, eq } from 'drizzle-orm'
|
||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
|
||
import {
|
||
ADAPTER_VERSION,
|
||
MAPPING_VERSION,
|
||
PARSER_VERSION,
|
||
SANITIZER_VERSION,
|
||
} from '@sf/adapter-wp'
|
||
import { importJobs, sites, sourceConnections, sourceSnapshots } from '../../db/schema.js'
|
||
import type * as schema from '../../db/schema.js'
|
||
import { isUniqueViolation } from '../source/register.service.js'
|
||
|
||
/** 本文件位于 <repo>/apps/api/src/modules/import/ → 上五级是仓库根 */
|
||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..')
|
||
|
||
export type ImportDb = NodePgDatabase<typeof schema>
|
||
|
||
export type ImportJobStatus = (typeof schema.importJobStatus.enumValues)[number]
|
||
|
||
export type ImportErrorCode =
|
||
| 'SITE_NOT_FOUND'
|
||
| 'SNAPSHOT_NOT_FOUND'
|
||
| 'NO_WXR_SNAPSHOT'
|
||
| 'PIPELINE_FAILED'
|
||
|
||
/** 导入失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */
|
||
export class ImportError extends Error {
|
||
readonly code: ImportErrorCode
|
||
constructor(code: ImportErrorCode, message: string) {
|
||
super(message)
|
||
this.name = 'ImportError'
|
||
this.code = code
|
||
}
|
||
}
|
||
|
||
/** 版本块(报告/上下文携带;job 行只落 mapping_version) */
|
||
export interface ImportVersionBlock {
|
||
adapter: string
|
||
mapping: string
|
||
sanitizer: string
|
||
parser: string
|
||
}
|
||
|
||
/** 已解析快照引用(报告与管道上下文的载荷;metadata 含 storageRoot 等登记期事实) */
|
||
export interface SnapshotRef {
|
||
id: string
|
||
kind: 'wxr' | 'media_bundle' | 'manual_url_list'
|
||
sha256: string
|
||
byteSize: number
|
||
storagePath: string
|
||
capturedAt: Date
|
||
metadata: Record<string, unknown>
|
||
}
|
||
|
||
/** 管道执行上下文:Task 6 真实管道的全部输入(站点 locale 接缝见前置事实 8) */
|
||
export interface ImportJobContext {
|
||
db: ImportDb
|
||
jobId: string
|
||
siteId: string
|
||
siteKey: string
|
||
defaultLocale: string
|
||
connectionId: string
|
||
/** 主输入:wxr 快照(job.source_snapshot_id 指向它) */
|
||
wxrSnapshot: SnapshotRef
|
||
/** 侧输入(前置事实 13):按 (connection, kind) 解析的最新快照,可能缺席(null) */
|
||
mediaBundleSnapshot: SnapshotRef | null
|
||
manualUrlListSnapshot: SnapshotRef | null
|
||
versions: ImportVersionBlock
|
||
}
|
||
|
||
/** 管道钩子(Task 6 接真实写入器;缺省空管道=直接成功)。抛错 → job failed */
|
||
export type ImportPipeline = (ctx: ImportJobContext) => Promise<void>
|
||
|
||
export interface CreateImportJobInput {
|
||
siteKey: string
|
||
/** 显式 wxr 快照 id;缺省按 (connection, kind='wxr') 选最新 */
|
||
snapshotId?: string
|
||
/** 幂等键;缺省 = site + wxr 快照 + 三语义版本 自动派生 */
|
||
idempotencyKey?: string
|
||
/** 连接名(最新快照解析的 connection 侧);缺省 'default'(与 register 缺省一致) */
|
||
connectionName?: string
|
||
/** 报告根(落 <reportRoot>/<jobId>/reports/import.json);缺省 repoRoot/artifacts/import */
|
||
reportRoot?: string
|
||
/** 管道钩子(测试/Task 6 注入);缺省空管道 */
|
||
pipeline?: ImportPipeline
|
||
}
|
||
|
||
export interface CreateImportJobResult {
|
||
jobId: string
|
||
/** job 终态(或复用时既有状态——并发窗口下可能读到中间态) */
|
||
status: ImportJobStatus
|
||
/** true = 命中既有 job(幂等键复用,未重跑管道、未改行) */
|
||
reused: boolean
|
||
siteId: string
|
||
siteKey: string
|
||
connectionId: string
|
||
wxrSnapshot: SnapshotRef
|
||
mediaBundleSnapshot: SnapshotRef | null
|
||
manualUrlListSnapshot: SnapshotRef | null
|
||
versions: ImportVersionBlock
|
||
reportPath: string | null
|
||
durationMs: number
|
||
}
|
||
|
||
const VERSIONS: ImportVersionBlock = {
|
||
adapter: ADAPTER_VERSION,
|
||
mapping: MAPPING_VERSION,
|
||
sanitizer: SANITIZER_VERSION,
|
||
parser: PARSER_VERSION,
|
||
}
|
||
|
||
// ---- 快照解析 ----
|
||
|
||
const SNAPSHOT_COLUMNS = {
|
||
id: sourceSnapshots.id,
|
||
siteId: sourceSnapshots.siteId,
|
||
kind: sourceSnapshots.kind,
|
||
sha256: sourceSnapshots.sha256,
|
||
byteSize: sourceSnapshots.byteSize,
|
||
storagePath: sourceSnapshots.storagePath,
|
||
sourceConnectionId: sourceSnapshots.sourceConnectionId,
|
||
capturedAt: sourceSnapshots.capturedAt,
|
||
metadata: sourceSnapshots.metadata,
|
||
}
|
||
|
||
type SnapshotRow = {
|
||
id: string
|
||
siteId: string
|
||
kind: 'wxr' | 'sitemap' | 'media_bundle' | 'manual_url_list'
|
||
sha256: string
|
||
byteSize: number
|
||
storagePath: string
|
||
sourceConnectionId: string
|
||
capturedAt: Date
|
||
metadata: Record<string, unknown>
|
||
}
|
||
|
||
function toRef(row: SnapshotRow): SnapshotRef {
|
||
return {
|
||
id: row.id,
|
||
// 声音窄化:调用方按 kind 过滤查询(wxr/media_bundle/manual_url_list 三值)或已过
|
||
// kind==='wxr' 校验——drizzle 列类型不携带过滤谓词,四值联合在此收窄是安全的
|
||
kind: row.kind as SnapshotRef['kind'],
|
||
sha256: row.sha256,
|
||
byteSize: row.byteSize,
|
||
storagePath: row.storagePath,
|
||
capturedAt: row.capturedAt,
|
||
metadata: row.metadata,
|
||
}
|
||
}
|
||
|
||
/** 按 (connection, kind) 解析最新快照(capturedAt 降序、createdAt 降序兜底平局);无则 null */
|
||
async function latestSnapshot(
|
||
db: ImportDb,
|
||
connectionId: string,
|
||
kind: 'wxr' | 'media_bundle' | 'manual_url_list',
|
||
): Promise<SnapshotRow | null> {
|
||
const rows = await db
|
||
.select(SNAPSHOT_COLUMNS)
|
||
.from(sourceSnapshots)
|
||
.where(and(eq(sourceSnapshots.sourceConnectionId, connectionId), eq(sourceSnapshots.kind, kind)))
|
||
.orderBy(desc(sourceSnapshots.capturedAt), desc(sourceSnapshots.createdAt))
|
||
.limit(1)
|
||
return rows.length > 0 ? (rows[0] as SnapshotRow) : null
|
||
}
|
||
|
||
// ---- 幂等键 ----
|
||
|
||
/** 默认幂等键:site + wxr 快照 + 三语义版本(定义钉死——报告可复算、CLI 可显式覆盖) */
|
||
export function defaultIdempotencyKey(siteKey: string, wxrSnapshotId: string): string {
|
||
return `import:${siteKey}:${wxrSnapshotId}:${ADAPTER_VERSION}/${MAPPING_VERSION}/${SANITIZER_VERSION}`
|
||
}
|
||
|
||
// ---- 主入口 ----
|
||
|
||
/**
|
||
* 创建(并就地推进到底)一个导入 job:
|
||
* 解析(site → wxr 快照 → 侧快照)→ 幂等建行(queued)→ running(attempt=1、started_at)
|
||
* → 空管道/注入管道 → succeeded|failed(error_code/error_summary/finished_at)→ 报告落盘。
|
||
* 失败口径:解析阶段失败零 job 行;管道阶段失败 job 行保留(failed 终态 + error 列)。
|
||
*/
|
||
export async function createImportJob(
|
||
db: ImportDb,
|
||
input: CreateImportJobInput,
|
||
): Promise<CreateImportJobResult> {
|
||
const startedAt = performance.now()
|
||
|
||
// 阶段 0:站点解析(不自动建站——建站是 register 的职责)
|
||
const siteRows = await db
|
||
.select({ id: sites.id, key: sites.key, defaultLocale: sites.defaultLocale })
|
||
.from(sites)
|
||
.where(eq(sites.key, input.siteKey))
|
||
.limit(1)
|
||
if (siteRows.length === 0) {
|
||
throw new ImportError('SITE_NOT_FOUND', `站点不存在:${input.siteKey}(先运行 sf source:register)`)
|
||
}
|
||
const site = siteRows[0]!
|
||
|
||
// 阶段 1:wxr 主输入快照解析(显式 id 优先;缺省按 (connection, kind='wxr') 最新)
|
||
let wxrRow: SnapshotRow | null = null
|
||
if (input.snapshotId !== undefined) {
|
||
const rows = await db
|
||
.select(SNAPSHOT_COLUMNS)
|
||
.from(sourceSnapshots)
|
||
.where(eq(sourceSnapshots.id, input.snapshotId))
|
||
.limit(1)
|
||
if (rows.length === 0) {
|
||
throw new ImportError('SNAPSHOT_NOT_FOUND', `快照不存在:${input.snapshotId}`)
|
||
}
|
||
const row = rows[0] as SnapshotRow
|
||
if (row.siteId !== site.id) {
|
||
throw new ImportError(
|
||
'SNAPSHOT_NOT_FOUND',
|
||
`快照 ${input.snapshotId} 不属于站点 ${input.siteKey}`,
|
||
)
|
||
}
|
||
if (row.kind !== 'wxr') {
|
||
throw new ImportError(
|
||
'NO_WXR_SNAPSHOT',
|
||
`快照 ${input.snapshotId} 的 kind 是 ${row.kind},不是 wxr——主输入必须是 wxr 快照`,
|
||
)
|
||
}
|
||
wxrRow = row
|
||
} else {
|
||
const connectionName = input.connectionName ?? 'default'
|
||
// connection 判重键 (site_id, name) 无库级唯一约束(Plan 1 决定,register 是唯一写入方)
|
||
const connRows = await db
|
||
.select({ id: sourceConnections.id })
|
||
.from(sourceConnections)
|
||
.where(and(eq(sourceConnections.siteId, site.id), eq(sourceConnections.name, connectionName)))
|
||
.limit(1)
|
||
if (connRows.length > 0) {
|
||
wxrRow = await latestSnapshot(db, connRows[0]!.id, 'wxr')
|
||
}
|
||
if (wxrRow === null) {
|
||
throw new ImportError(
|
||
'NO_WXR_SNAPSHOT',
|
||
`站点 ${input.siteKey}(connection=${connectionName})没有任何 wxr 快照——先运行 sf source:register --wxr`,
|
||
)
|
||
}
|
||
}
|
||
const connectionId = wxrRow.sourceConnectionId
|
||
|
||
// 阶段 2:侧快照(前置事实 13)——挂在 wxr 快照的 connection 下按 kind 解析最新
|
||
const mediaBundleRow = await latestSnapshot(db, connectionId, 'media_bundle')
|
||
const manualUrlListRow = await latestSnapshot(db, connectionId, 'manual_url_list')
|
||
const wxrSnapshot = toRef(wxrRow)
|
||
const mediaBundleSnapshot = mediaBundleRow === null ? null : toRef(mediaBundleRow)
|
||
const manualUrlListSnapshot = manualUrlListRow === null ? null : toRef(manualUrlListRow)
|
||
|
||
// 阶段 3:幂等建 job(queued)——命中既有 key 直接复用返回(不重跑)
|
||
const idempotencyKey = input.idempotencyKey ?? defaultIdempotencyKey(input.siteKey, wxrSnapshot.id)
|
||
const resolved = {
|
||
siteKey: input.siteKey,
|
||
connectionId,
|
||
wxrSnapshot,
|
||
mediaBundleSnapshot,
|
||
manualUrlListSnapshot,
|
||
}
|
||
const existing = await reuseExisting(db, idempotencyKey)
|
||
if (existing !== null) {
|
||
return { ...existing, ...resolved, durationMs: performance.now() - startedAt }
|
||
}
|
||
let jobId: string
|
||
try {
|
||
const inserted = await db
|
||
.insert(importJobs)
|
||
.values({
|
||
siteId: site.id,
|
||
sourceSnapshotId: wxrSnapshot.id,
|
||
status: 'queued',
|
||
mappingVersion: MAPPING_VERSION,
|
||
idempotencyKey,
|
||
})
|
||
.returning({ id: importJobs.id })
|
||
jobId = inserted[0]!.id
|
||
} catch (err) {
|
||
// 并发同 key:唯一约束收敛——重查复用(插入撞 23505 说明行已存在,重查必命中)
|
||
if (isUniqueViolation(err)) {
|
||
const raced = await reuseExisting(db, idempotencyKey)
|
||
if (raced !== null) {
|
||
return { ...raced, ...resolved, durationMs: performance.now() - startedAt }
|
||
}
|
||
}
|
||
throw err
|
||
}
|
||
|
||
// 阶段 4:状态推进 queued → running(attempt 记首次执行)
|
||
await db
|
||
.update(importJobs)
|
||
.set({ status: 'running', startedAt: new Date(), attempt: 1 })
|
||
.where(eq(importJobs.id, jobId))
|
||
|
||
// 阶段 5:管道(本任务缺省空管道;Task 6 接真实写入器)——抛错 → failed 终态
|
||
let failed: { code: string; summary: string } | null = null
|
||
const ctx: ImportJobContext = {
|
||
db,
|
||
jobId,
|
||
siteId: site.id,
|
||
siteKey: site.key,
|
||
defaultLocale: site.defaultLocale,
|
||
connectionId,
|
||
wxrSnapshot,
|
||
mediaBundleSnapshot,
|
||
manualUrlListSnapshot,
|
||
versions: VERSIONS,
|
||
}
|
||
try {
|
||
await (input.pipeline ?? emptyPipeline)(ctx)
|
||
} catch (err) {
|
||
failed = {
|
||
code: err instanceof ImportError ? err.code : 'PIPELINE_FAILED',
|
||
summary: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
|
||
}
|
||
}
|
||
|
||
// 阶段 6:终态落行(report_path 同步入行)→ 报告写盘
|
||
const reportRoot = resolve(input.reportRoot ?? join(REPO_ROOT, 'artifacts', 'import'))
|
||
const reportDir = join(reportRoot, jobId, 'reports')
|
||
const reportPath = join(reportDir, 'import.json')
|
||
const finishedAt = new Date()
|
||
const finalStatus: ImportJobStatus = failed === null ? 'succeeded' : 'failed'
|
||
await db
|
||
.update(importJobs)
|
||
.set(
|
||
failed === null
|
||
? { status: finalStatus, finishedAt, reportPath }
|
||
: {
|
||
status: finalStatus,
|
||
finishedAt,
|
||
reportPath,
|
||
errorCode: failed.code,
|
||
errorSummary: failed.summary,
|
||
},
|
||
)
|
||
.where(eq(importJobs.id, jobId))
|
||
const report = {
|
||
jobId,
|
||
generatedAt: finishedAt.toISOString(),
|
||
status: finalStatus,
|
||
siteKey: site.key,
|
||
siteId: site.id,
|
||
connectionId,
|
||
snapshots: {
|
||
wxr: slimRef(wxrSnapshot),
|
||
media_bundle: mediaBundleSnapshot === null ? null : slimRef(mediaBundleSnapshot),
|
||
manual_url_list: manualUrlListSnapshot === null ? null : slimRef(manualUrlListSnapshot),
|
||
},
|
||
versions: VERSIONS,
|
||
error: failed,
|
||
durationMs: performance.now() - startedAt,
|
||
}
|
||
await mkdir(reportDir, { recursive: true })
|
||
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8')
|
||
|
||
return {
|
||
jobId,
|
||
status: finalStatus,
|
||
reused: false,
|
||
siteId: site.id,
|
||
siteKey: site.key,
|
||
connectionId,
|
||
wxrSnapshot,
|
||
mediaBundleSnapshot,
|
||
manualUrlListSnapshot,
|
||
versions: VERSIONS,
|
||
reportPath,
|
||
durationMs: performance.now() - startedAt,
|
||
}
|
||
}
|
||
|
||
/** 既有 job 的复用返回(不动行、不跑管道);无则 null */
|
||
async function reuseExisting(
|
||
db: ImportDb,
|
||
idempotencyKey: string,
|
||
): Promise<Pick<CreateImportJobResult, 'jobId' | 'status' | 'reused' | 'siteId' | 'reportPath' | 'versions'> | null> {
|
||
const found = await db
|
||
.select({
|
||
id: importJobs.id,
|
||
status: importJobs.status,
|
||
siteId: importJobs.siteId,
|
||
reportPath: importJobs.reportPath,
|
||
})
|
||
.from(importJobs)
|
||
.where(eq(importJobs.idempotencyKey, idempotencyKey))
|
||
.limit(1)
|
||
if (found.length === 0) return null
|
||
const row = found[0]!
|
||
return {
|
||
jobId: row.id,
|
||
status: row.status,
|
||
reused: true,
|
||
siteId: row.siteId,
|
||
reportPath: row.reportPath,
|
||
versions: VERSIONS,
|
||
}
|
||
}
|
||
|
||
/** 报告里的快照引用瘦身版:不含 metadata(media_bundle 清单可达数千条,报告只留身份锚点) */
|
||
function slimRef(ref: SnapshotRef): {
|
||
id: string
|
||
kind: string
|
||
sha256: string
|
||
byteSize: number
|
||
storagePath: string
|
||
} {
|
||
return { id: ref.id, kind: ref.kind, sha256: ref.sha256, byteSize: ref.byteSize, storagePath: ref.storagePath }
|
||
}
|
||
|
||
/** Task 1 空管道:直接成功(Task 6 以真实写入器替换缺省值) */
|
||
async function emptyPipeline(_ctx: ImportJobContext): Promise<void> {}
|