浅念
0e8b2b5356
Plan 3 Task 1：sf import 命令骨架 + import_jobs 生命周期（空管道，Task 6 接真实管道）。 - import.service.ts（零装饰器）：createImportJob——站点/快照解析（缺省按 (connection, kind='wxr') 选最新；三快照绑定按前置事实 13 记入报告，job 行不扩列）、幂等建行 （默认 key=site+wxr快照+三语义版本；重复提交复用既有 job 不重跑，并发撞唯一约束 重查收敛）、状态推进 queued→running→succeeded/failed（attempt/started_at/finished_at/ error_code/error_summary）、报告落 artifacts/import/<jobId>/reports/import.json （全四版本块：adapter/mapping/sanitizer/parser；job 行落 mapping_version） - 可预期错误带 code：SITE_NOT_FOUND / SNAPSHOT_NOT_FOUND / NO_WXR_SNAPSHOT / PIPELINE_FAILED——解析期失败零 job 行 - adapter 事件加法式补 postId（前置事实 12）：unsupported/error 事件可选携带 String(postId)，缺失时键缺席不硬造 source_id；row 事件与报告聚合零影响、不触发版本 bump - canonicalJson 提取至 apps/api/src/common/canonical.ts 共享导出（前置事实 6）， register 侧改 import 零行为变更（既有清单哈希测试守护） - sf CLI + import 命令（--site/--snapshot/--key/--connection），输出 job 摘要与版本块 - 测试：tests/integration/import-job.test.ts（5 例：生命周期观察/最新快照解析/幂等复用 与并发收敛/可预期失败零落行/failed 路径）+ adapter postId 3 例 + canonical 单测 6 例 （265→279 全绿）；typecheck 四段链/lint 绿；临时库 CLI 冒烟（register→import→复用）通过 Co-Authored-By: Claude Code <noreply@anthropic.com>
412 lines
17 KiB
TypeScript
412 lines
17 KiB
TypeScript
// Plan 3 Task 1 失败测试（先于实现）：import 命令骨架 + import_jobs 生命周期。
|
||
// 覆盖契约（对照计划 Task 1 与前置事实 6/12/13）：
|
||
// 1. register（wxr + media + media-map 三快照）→ createImportJob：job 行 created(queued)→
|
||
// running→succeeded 全程可观察（管道钩子内读 running 中间态）、mapping_version 落
|
||
// MAPPING_VERSION、报告记录全四版本块 + 三快照绑定（wxr 主输入 / media_bundle /
|
||
// manual_url_list 按 (connection,kind) 解析最新记入报告，job 行不扩列——前置事实 13）；
|
||
// 2. 无 --snapshot：按 (connection, kind='wxr') 选最新快照；
|
||
// 3. idempotency_key 唯一：重复提交同默认 key（site+snapshot+版本）→ 复用既有 job 不重跑
|
||
// （reused 标记 + 管道零调用 + 恰一行）；显式 key 覆盖默认；并发同 key 收敛同 job；
|
||
// 4. 无 wxr 快照（站点仅有非 wxr 输入 / 显式 id 指向非 wxr 快照）→ NO_WXR_SNAPSHOT；
|
||
// 显式 id 不存在或属他站 → SNAPSHOT_NOT_FOUND；站点不存在 → SITE_NOT_FOUND；均零 job 行；
|
||
// 5. failed 路径：管道抛错 → running→failed（error_code/error_summary/finished_at），
|
||
// 重提交同 key 复用失败 job 不重跑。
|
||
// 运行于 globalSetup 临时库（ADR-015）；storage/报告根用 os.tmpdir 临时目录，finally 清理。
|
||
import { existsSync } from 'node:fs'
|
||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||
import { tmpdir } from 'node:os'
|
||
import { join, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { describe, expect, it } from 'vitest'
|
||
import { ADAPTER_VERSION, MAPPING_VERSION, PARSER_VERSION, SANITIZER_VERSION } from '@sf/adapter-wp'
|
||
import {
|
||
ImportError,
|
||
createImportJob,
|
||
type CreateImportJobInput,
|
||
} from '../../apps/api/src/modules/import/import.service.js'
|
||
import { registerSource } from '../../apps/api/src/modules/source/register.service.js'
|
||
import { rand, withImmutableCleanup, withTestDb, type Db } from './helpers.js'
|
||
|
||
const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url))
|
||
const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml')
|
||
|
||
interface Ctx {
|
||
work: string
|
||
storageRoot: string
|
||
registerReportRoot: string
|
||
importReportRoot: string
|
||
mediaDir: string
|
||
mediaMapFile: string
|
||
allowedRoots: string[]
|
||
}
|
||
|
||
/** 独立媒体树 + 侧车 JSON（形态同 source-register.test.ts） */
|
||
async function makeCtx(): Promise<Ctx> {
|
||
const work = await mkdtemp(join(tmpdir(), 'sf-import-'))
|
||
const mediaDir = join(work, 'media')
|
||
await mkdir(join(mediaDir, 'sub'), { recursive: true })
|
||
await writeFile(join(mediaDir, 'a.png'), 'sf-import-media-a')
|
||
await writeFile(join(mediaDir, 'sub', 'b.png'), 'sf-import-media-b')
|
||
const mediaMapFile = join(work, 'media-map.json')
|
||
await writeFile(
|
||
mediaMapFile,
|
||
`${JSON.stringify([
|
||
{ sourceUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png', filePath: 'a.png' },
|
||
])}\n`,
|
||
'utf8',
|
||
)
|
||
return {
|
||
work,
|
||
storageRoot: join(work, 'storage'),
|
||
registerReportRoot: join(work, 'artifacts', 'register'),
|
||
importReportRoot: join(work, 'artifacts', 'import'),
|
||
mediaDir,
|
||
mediaMapFile,
|
||
allowedRoots: [FIXTURES_ROOT, work],
|
||
}
|
||
}
|
||
|
||
/** 服务写入的 job 行清场：import_jobs 可变直删；快照不可变走触发器开关；FK 反序 */
|
||
async function cleanupSiteRows(db: Db, siteId: string): Promise<void> {
|
||
await db.execute(`delete from import_jobs where site_id = '${siteId}'`)
|
||
await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`)
|
||
await db.execute(`delete from source_connections where site_id = '${siteId}'`)
|
||
await db.execute(`delete from sites where id = '${siteId}'`)
|
||
}
|
||
|
||
/** 登记三快照（wxr + media_bundle + manual_url_list）的便捷封装 */
|
||
async function registerAll(
|
||
db: Db,
|
||
ctx: Ctx,
|
||
siteKey: string,
|
||
): Promise<Awaited<ReturnType<typeof registerSource>>> {
|
||
return registerSource(db, {
|
||
siteKey,
|
||
wxrPath: WXR_FIXTURE,
|
||
mediaDir: ctx.mediaDir,
|
||
mediaMapPath: ctx.mediaMapFile,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.registerReportRoot,
|
||
})
|
||
}
|
||
|
||
describe('import_jobs 生命周期（Task 1 骨架）', () => {
|
||
it('三快照 register → createImportJob：queued→running→succeeded、mapping_version、报告含版本块与三快照绑定', async () => {
|
||
await withTestDb(async (db) => {
|
||
const ctx = await makeCtx()
|
||
try {
|
||
const key = `sfimp-${rand()}`
|
||
const reg = await registerAll(db, ctx, key)
|
||
try {
|
||
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
|
||
const media = reg.snapshots.find((s) => s.kind === 'media_bundle')!
|
||
const map = reg.snapshots.find((s) => s.kind === 'manual_url_list')!
|
||
|
||
// 管道钩子内观察中间态：此刻 job 已 created(queued)→running（attempt 记首次执行）
|
||
let midStatus: string | undefined
|
||
let midAttempt: string | number | undefined
|
||
let midStartedAt: Date | null | undefined
|
||
let midFinishedAt: Date | null | undefined
|
||
const result = await createImportJob(db, {
|
||
siteKey: key,
|
||
snapshotId: wxr.id,
|
||
reportRoot: ctx.importReportRoot,
|
||
pipeline: async (jobCtx) => {
|
||
expect(jobCtx.jobId).toBeTruthy()
|
||
const rows = (await db.execute(
|
||
`select status, attempt, started_at, finished_at from import_jobs where id = '${jobCtx.jobId}'`,
|
||
)) as any
|
||
midStatus = rows[0].status
|
||
midAttempt = rows[0].attempt
|
||
midStartedAt = rows[0].started_at
|
||
midFinishedAt = rows[0].finished_at
|
||
},
|
||
})
|
||
|
||
expect(result.reused).toBe(false)
|
||
expect(result.status).toBe('succeeded')
|
||
expect(midStatus).toBe('running')
|
||
expect(Number(midAttempt)).toBe(1)
|
||
expect(midStartedAt).not.toBeNull()
|
||
expect(midFinishedAt).toBeNull()
|
||
|
||
// 终态行：身份/版本/时间戳/report_path
|
||
const job = (await db.execute(
|
||
`select status, mapping_version, source_snapshot_id, started_at, finished_at, report_path, attempt, error_code
|
||
from import_jobs where id = '${result.jobId}'`,
|
||
)) as any
|
||
expect(job[0].status).toBe('succeeded')
|
||
expect(job[0].mapping_version).toBe(MAPPING_VERSION)
|
||
expect(job[0].source_snapshot_id).toBe(wxr.id)
|
||
expect(job[0].started_at).not.toBeNull()
|
||
expect(job[0].finished_at).not.toBeNull()
|
||
expect(job[0].error_code).toBeNull()
|
||
expect(Number(job[0].attempt)).toBe(1)
|
||
|
||
// 报告：全四版本块 + 三快照绑定（前置事实 13：侧快照只入报告，job 行不扩列）
|
||
const reportPath = join(ctx.importReportRoot, result.jobId, 'reports', 'import.json')
|
||
expect(job[0].report_path).toBe(reportPath)
|
||
expect(existsSync(reportPath)).toBe(true)
|
||
const report = JSON.parse(await readFile(reportPath, 'utf8')) as {
|
||
jobId: string
|
||
status: string
|
||
versions: Record<string, string>
|
||
snapshots: {
|
||
wxr: { id: string }
|
||
media_bundle: { id: string } | null
|
||
manual_url_list: { id: string } | null
|
||
}
|
||
}
|
||
expect(report.jobId).toBe(result.jobId)
|
||
expect(report.status).toBe('succeeded')
|
||
expect(report.versions).toStrictEqual({
|
||
adapter: ADAPTER_VERSION,
|
||
mapping: MAPPING_VERSION,
|
||
sanitizer: SANITIZER_VERSION,
|
||
parser: PARSER_VERSION,
|
||
})
|
||
expect(report.snapshots.wxr.id).toBe(wxr.id)
|
||
expect(report.snapshots.media_bundle?.id).toBe(media.id)
|
||
expect(report.snapshots.manual_url_list?.id).toBe(map.id)
|
||
} finally {
|
||
await cleanupSiteRows(db, reg.siteId)
|
||
}
|
||
} finally {
|
||
await rm(ctx.work, { recursive: true, force: true })
|
||
}
|
||
})
|
||
})
|
||
|
||
it('无 snapshotId：按 (connection, kind=wxr) 选最新；同默认 key 重提交复用不重跑', async () => {
|
||
await withTestDb(async (db) => {
|
||
const ctx = await makeCtx()
|
||
try {
|
||
const key = `sfimp-${rand()}`
|
||
const r1 = await registerSource(db, {
|
||
siteKey: key,
|
||
wxrPath: WXR_FIXTURE,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.registerReportRoot,
|
||
})
|
||
try {
|
||
// 第二个 wxr 快照：内容加尾随换行（sha 不同、仍合法 XML）；稍歇保证 capturedAt 严格更晚
|
||
const v2 = join(ctx.work, 'mini-v2.xml')
|
||
await writeFile(v2, `${await readFile(WXR_FIXTURE, 'utf8')}\n`, 'utf8')
|
||
await new Promise((r) => setTimeout(r, 25))
|
||
const r2 = await registerSource(db, {
|
||
siteKey: key,
|
||
wxrPath: v2,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.registerReportRoot,
|
||
})
|
||
const wxr1 = r1.snapshots.find((s) => s.kind === 'wxr')!
|
||
const wxr2 = r2.snapshots.find((s) => s.kind === 'wxr')!
|
||
expect(wxr2.id).not.toBe(wxr1.id)
|
||
|
||
let pipelineCalls = 0
|
||
const input: CreateImportJobInput = {
|
||
siteKey: key,
|
||
reportRoot: ctx.importReportRoot,
|
||
pipeline: async () => {
|
||
pipelineCalls++
|
||
},
|
||
}
|
||
const a = await createImportJob(db, input)
|
||
expect(a.reused).toBe(false)
|
||
expect(a.status).toBe('succeeded')
|
||
// 最新快照 = 第二个（v2）
|
||
const job = (await db.execute(
|
||
`select source_snapshot_id from import_jobs where id = '${a.jobId}'`,
|
||
)) as any
|
||
expect(job[0].source_snapshot_id).toBe(wxr2.id)
|
||
|
||
// 同默认 key 重提交：复用既有 job、管道零追加调用
|
||
const b = await createImportJob(db, input)
|
||
expect(b.reused).toBe(true)
|
||
expect(b.jobId).toBe(a.jobId)
|
||
expect(b.status).toBe('succeeded')
|
||
expect(pipelineCalls).toBe(1)
|
||
const count = (await db.execute(
|
||
`select count(*)::int as n from import_jobs where site_id = '${r1.siteId}'`,
|
||
)) as any
|
||
expect(count[0].n).toBe(1)
|
||
|
||
// 显式 idempotencyKey 覆盖默认 key：同 key 复用、异 key（回默认）不新建重复
|
||
const c = await createImportJob(db, { ...input, idempotencyKey: 'manual-key-1' })
|
||
expect(c.reused).toBe(false)
|
||
const d = await createImportJob(db, { ...input, idempotencyKey: 'manual-key-1' })
|
||
expect(d.reused).toBe(true)
|
||
expect(d.jobId).toBe(c.jobId)
|
||
expect(pipelineCalls).toBe(2)
|
||
} finally {
|
||
await cleanupSiteRows(db, r1.siteId)
|
||
}
|
||
} finally {
|
||
await rm(ctx.work, { recursive: true, force: true })
|
||
}
|
||
})
|
||
})
|
||
|
||
it('并发同默认 key：恰一 job 行、双方同 jobId、管道恰执行一次（唯一约束收敛）', async () => {
|
||
await withTestDb(async (db) => {
|
||
const ctx = await makeCtx()
|
||
try {
|
||
const key = `sfimp-${rand()}`
|
||
const reg = await registerAll(db, ctx, key)
|
||
try {
|
||
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
|
||
let pipelineCalls = 0
|
||
const input: CreateImportJobInput = {
|
||
siteKey: key,
|
||
snapshotId: wxr.id,
|
||
reportRoot: ctx.importReportRoot,
|
||
pipeline: async () => {
|
||
pipelineCalls++
|
||
},
|
||
}
|
||
const [a, b] = await Promise.all([createImportJob(db, input), createImportJob(db, input)])
|
||
expect(a.jobId).toBe(b.jobId)
|
||
expect(Number(a.reused) + Number(b.reused)).toBe(1)
|
||
expect(pipelineCalls).toBe(1)
|
||
const count = (await db.execute(
|
||
`select count(*)::int as n from import_jobs where site_id = '${reg.siteId}'`,
|
||
)) as any
|
||
expect(count[0].n).toBe(1)
|
||
} finally {
|
||
await cleanupSiteRows(db, reg.siteId)
|
||
}
|
||
} finally {
|
||
await rm(ctx.work, { recursive: true, force: true })
|
||
}
|
||
})
|
||
})
|
||
|
||
it('可预期失败均零 job 行：SITE_NOT_FOUND / SNAPSHOT_NOT_FOUND（不存在或属他站）/ NO_WXR_SNAPSHOT（无非 wxr 主输入或显式指向非 wxr）', async () => {
|
||
await withTestDb(async (db) => {
|
||
const ctx = await makeCtx()
|
||
try {
|
||
// 站点 A：仅 media-map 输入（有 connection、无非 wxr 主输入）
|
||
const keyA = `sfimp-${rand()}`
|
||
const regA = await registerSource(db, {
|
||
siteKey: keyA,
|
||
mediaMapPath: ctx.mediaMapFile,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.registerReportRoot,
|
||
})
|
||
// 站点 B：三快照齐全
|
||
const keyB = `sfimp-${rand()}`
|
||
const regB = await registerAll(db, ctx, keyB)
|
||
try {
|
||
const mapSnapA = regA.snapshots[0]!
|
||
const mediaSnapB = regB.snapshots.find((s) => s.kind === 'media_bundle')!
|
||
const cases: { label: string; input: CreateImportJobInput; code: string }[] = [
|
||
{
|
||
label: '站点不存在',
|
||
input: { siteKey: `sfimp-no-such-${rand()}` },
|
||
code: 'SITE_NOT_FOUND',
|
||
},
|
||
{
|
||
label: '仅有 manual_url_list 输入的站点（无任何 wxr 快照）',
|
||
input: { siteKey: keyA, reportRoot: ctx.importReportRoot },
|
||
code: 'NO_WXR_SNAPSHOT',
|
||
},
|
||
{
|
||
label: '显式 snapshotId 指向非 wxr 快照（media_bundle）',
|
||
input: { siteKey: keyB, snapshotId: mediaSnapB.id, reportRoot: ctx.importReportRoot },
|
||
code: 'NO_WXR_SNAPSHOT',
|
||
},
|
||
{
|
||
label: '显式 snapshotId 不存在',
|
||
input: {
|
||
siteKey: keyB,
|
||
snapshotId: '00000000-0000-4000-8000-000000000000',
|
||
reportRoot: ctx.importReportRoot,
|
||
},
|
||
code: 'SNAPSHOT_NOT_FOUND',
|
||
},
|
||
{
|
||
label: '显式 snapshotId 属于他站',
|
||
input: { siteKey: keyB, snapshotId: mapSnapA.id, reportRoot: ctx.importReportRoot },
|
||
code: 'SNAPSHOT_NOT_FOUND',
|
||
},
|
||
]
|
||
for (const c of cases) {
|
||
const err: ImportError = await createImportJob(db, c.input).then(
|
||
() => {
|
||
throw new Error(`用例「${c.label}」不应成功`)
|
||
},
|
||
(e: ImportError) => e,
|
||
)
|
||
expect(err, c.label).toBeInstanceOf(ImportError)
|
||
expect(err.code, c.label).toBe(c.code)
|
||
expect(err.message, c.label).not.toBe('')
|
||
}
|
||
// 零 job 行（两站点均无）
|
||
const count = (await db.execute(
|
||
`select count(*)::int as n from import_jobs where site_id in ('${regA.siteId}', '${regB.siteId}')`,
|
||
)) as any
|
||
expect(count[0].n).toBe(0)
|
||
} finally {
|
||
await cleanupSiteRows(db, regB.siteId)
|
||
await cleanupSiteRows(db, regA.siteId)
|
||
}
|
||
} finally {
|
||
await rm(ctx.work, { recursive: true, force: true })
|
||
}
|
||
})
|
||
})
|
||
|
||
it('failed 路径：管道抛错 → running→failed（error_code/error_summary/finished_at）；重提交同 key 复用失败 job 不重跑', async () => {
|
||
await withTestDb(async (db) => {
|
||
const ctx = await makeCtx()
|
||
try {
|
||
const key = `sfimp-${rand()}`
|
||
const reg = await registerAll(db, ctx, key)
|
||
try {
|
||
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
|
||
let pipelineCalls = 0
|
||
const input: CreateImportJobInput = {
|
||
siteKey: key,
|
||
snapshotId: wxr.id,
|
||
reportRoot: ctx.importReportRoot,
|
||
pipeline: async () => {
|
||
pipelineCalls++
|
||
throw new Error('模拟管道失败：断链 attachment')
|
||
},
|
||
}
|
||
const a = await createImportJob(db, input)
|
||
expect(a.reused).toBe(false)
|
||
expect(a.status).toBe('failed')
|
||
const job = (await db.execute(
|
||
`select status, error_code, error_summary, finished_at, started_at from import_jobs where id = '${a.jobId}'`,
|
||
)) as any
|
||
expect(job[0].status).toBe('failed')
|
||
expect(job[0].error_code).toBe('PIPELINE_FAILED')
|
||
expect(String(job[0].error_summary)).toContain('模拟管道失败')
|
||
expect(job[0].started_at).not.toBeNull()
|
||
expect(job[0].finished_at).not.toBeNull()
|
||
|
||
// 同 key 重提交：复用失败 job（不重跑、不改行）
|
||
const b = await createImportJob(db, input)
|
||
expect(b.reused).toBe(true)
|
||
expect(b.jobId).toBe(a.jobId)
|
||
expect(b.status).toBe('failed')
|
||
expect(pipelineCalls).toBe(1)
|
||
const count = (await db.execute(
|
||
`select count(*)::int as n from import_jobs where site_id = '${reg.siteId}'`,
|
||
)) as any
|
||
expect(count[0].n).toBe(1)
|
||
} finally {
|
||
await cleanupSiteRows(db, reg.siteId)
|
||
}
|
||
} finally {
|
||
await rm(ctx.work, { recursive: true, force: true })
|
||
}
|
||
})
|
||
})
|
||
})
|