Files
cms-website/tests/integration/source-register.test.ts
T

391 lines
18 KiB
TypeScript
Raw Normal View History

// Plan 2 Task 7 失败测试(先于实现):source:register 服务——content-addressed 快照登记。
// 覆盖六组契约:
// 1. 三类输入(wxr/sitemap/media 目录)一次登记 → site 按 key 自动建、source_connections
// 恰 1 行((site_id,name) 应用层判重)、source_snapshots 恰 3 行且 sha256 与源文件一致;
// 2. media_bundle 清单哈希定义钉死:排序相对路径('/' 分隔)+ 每文件 sha256 → canonical
// JSON(键排序、无空白)→ 再 sha256;metadata 含完整清单(path/bytes/sha256)、不含 mtime;
// 3. 重复 register 同输入 → 复用既有 snapshot(UNIQUE(site,kind,sha256),不新增行),
// 媒体目录重跑哈希一致(Windows 分隔符归一化 + mtime 不参与);
// 4. 副本布局:storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(整树),
// 副本内容与原输入一致(后续任务只读副本);
// 5. 路径安全:白名单外 / 越界遍历 / ENOENT / 类型不符均清晰报错且零落库;
// 6. 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机)。
// 运行于 globalSetup 临时库(ADR-015);storage/报告根用 os.tmpdir 临时目录,finally 清理。
// 注:符号链接跳过用目录 junction 验证(Windows 免特权;非 Windows 退化为普通符号链接);
// 创建失败(极端沙箱)时该断言静默放行——junction 创建不是本测试的被测契约。
import { createHash } from 'node:crypto'
import { existsSync } from 'node:fs'
import { mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import {
registerSource,
SourceRegisterError,
type RegisterInput,
type RegisteredSnapshot,
} from '../../apps/api/src/modules/source/register.service.js'
import { rand, withImmutableCleanup, withTestDb, type Db } from './helpers.js'
const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url))
const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml')
const SITEMAP_FIXTURE = resolve(FIXTURES_ROOT, 'sitemap', 'mini-sitemap.xml')
const THIS_FILE = fileURLToPath(import.meta.url)
const sha256 = (data: Buffer | string): string => createHash('sha256').update(data).digest('hex')
/** 测试侧独立实现的 canonical JSON(键排序、无空白)——不 import 实现,钉死定义 */
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`
if (value !== null && typeof value === 'object') {
const record = value as Record<string, unknown>
const body = Object.keys(record)
.sort()
.map((k) => `${JSON.stringify(k)}:${canonicalJson(record[k])}`)
.join(',')
return `{${body}}`
}
return JSON.stringify(value)
}
interface MediaFile {
path: string
bytes: number
content: Buffer
}
interface Ctx {
work: string
storageRoot: string
reportRoot: string
mediaDir: string
mediaFiles: MediaFile[]
allowedRoots: string[]
}
/** 独立媒体树:a.png + sub/b.png(内容固定,期望哈希可在测试内独立重算) */
async function makeMediaTree(work: string): Promise<{ mediaDir: string; mediaFiles: MediaFile[] }> {
const mediaDir = join(work, 'media')
await mkdir(join(mediaDir, 'sub'), { recursive: true })
const a = Buffer.from('sf-media-a-content')
const b = Buffer.from('sf-media-sub-b-content')
await writeFile(join(mediaDir, 'a.png'), a)
await writeFile(join(mediaDir, 'sub', 'b.png'), b)
return {
mediaDir,
mediaFiles: [
{ path: 'a.png', bytes: a.length, content: a },
{ path: 'sub/b.png', bytes: b.length, content: b },
],
}
}
async function withCtx(fn: (ctx: Ctx) => Promise<void>): Promise<void> {
const work = await mkdtemp(join(tmpdir(), 'sf-register-'))
const { mediaDir, mediaFiles } = await makeMediaTree(work)
const ctx: Ctx = {
work,
storageRoot: join(work, 'storage'),
reportRoot: join(work, 'artifacts', 'register'),
mediaDir,
mediaFiles,
allowedRoots: [FIXTURES_ROOT, work],
}
try {
await fn(ctx)
} finally {
await rm(work, { recursive: true, force: true })
}
}
function makeInput(ctx: Ctx, siteKey: string): RegisterInput {
return {
siteKey,
wxrPath: WXR_FIXTURE,
sitemapPath: SITEMAP_FIXTURE,
mediaDir: ctx.mediaDir,
allowedRoots: ctx.allowedRoots,
storageRoot: ctx.storageRoot,
reportRoot: ctx.reportRoot,
}
}
/** 服务创建的 site 行清场:快照不可变走触发器开关,FK 反序删 connection → site */
async function cleanupSiteRows(db: Db, siteId: string): Promise<void> {
await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`)
await db.execute(`delete from source_connections where site_id = '${siteId}'`)
await db.execute(`delete from sites where id = '${siteId}'`)
}
describe('source:register 服务', () => {
it('三类输入一次登记:site 自动创建、connection 1 行、snapshot 3 行(sha256 与源一致)', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
// site 按 key 自动 upsert(不存在则建,默认列落库)
const siteRows = (await db.execute(`select id, key, default_locale from sites where key = '${key}'`)) as any
expect(siteRows).toHaveLength(1)
expect(siteRows[0].id).toBe(r.siteId)
expect(siteRows[0].default_locale).toBe('en')
// connection 恰 1 行((site_id,name) 应用层判重)
const connRows = (await db.execute(
`select id, kind, name from source_connections where site_id = '${r.siteId}'`,
)) as any
expect(connRows).toHaveLength(1)
expect(connRows[0].kind).toBe('wxr')
expect(connRows[0].name).toBe('default')
expect(connRows[0].id).toBe(r.connectionId)
// snapshot 恰 3 行,kind 各异,sha256 与源文件一致,storage_path 指向存在的副本
const snapRows = (await db.execute(
`select kind, sha256, byte_size, storage_path from source_snapshots where site_id = '${r.siteId}'`,
)) as { kind: string; sha256: string; storage_path: string }[]
expect(snapRows).toHaveLength(3)
const byKind = new Map(snapRows.map((row) => [row.kind, row]))
expect(byKind.get('wxr')?.sha256).toBe(sha256(await readFile(WXR_FIXTURE)))
expect(byKind.get('sitemap')?.sha256).toBe(sha256(await readFile(SITEMAP_FIXTURE)))
for (const row of snapRows) {
expect(existsSync(join(ctx.storageRoot, row.storage_path))).toBe(true)
}
expect(r.snapshots).toHaveLength(3)
expect(new Set(r.snapshots.map((s: RegisteredSnapshot) => s.kind))).toEqual(
new Set(['wxr', 'sitemap', 'media_bundle']),
)
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('media_bundle 清单哈希 = 排序路径+每文件 sha256 的 canonical JSON 再哈希;metadata 无 mtime', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
const bundle = r.snapshots.find((s: RegisteredSnapshot) => s.kind === 'media_bundle')!
// 测试侧独立重算(排序 + canonical JSON 键序无空白)
const entries = ctx.mediaFiles
.map((f) => ({ path: f.path, sha256: sha256(f.content) }))
.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
expect(bundle.sha256).toBe(sha256(canonicalJson(entries)))
// metadata 含完整清单(path/bytes/sha256),不含 mtime
const manifest = (bundle.metadata as { manifest: { path: string; bytes: number; sha256: string }[] }).manifest
expect([...manifest].sort((a, b) => (a.path < b.path ? -1 : 1))).toEqual([
{ path: 'a.png', bytes: ctx.mediaFiles[0]!.bytes, sha256: sha256(ctx.mediaFiles[0]!.content) },
{ path: 'sub/b.png', bytes: ctx.mediaFiles[1]!.bytes, sha256: sha256(ctx.mediaFiles[1]!.content) },
])
expect(JSON.stringify(bundle.metadata)).not.toContain('mtime')
// byteSize = 清单文件字节数之和
expect(bundle.byteSize).toBe(ctx.mediaFiles.reduce((sum, f) => sum + f.bytes, 0))
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('重复 register 同输入:复用既有 snapshot/connection/site,不新增行;mtime 变化不影响哈希', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r1 = await registerSource(db, makeInput(ctx, key))
try {
// 播动 mtime(证明清单哈希不含 mtime)后重跑
const t = new Date(Date.now() + 60_000)
await utimes(join(ctx.mediaDir, 'a.png'), t, t)
await utimes(WXR_FIXTURE, t, t).catch(() => undefined) // 夹具可能只读;媒体树已足够证明
const r2 = await registerSource(db, makeInput(ctx, key))
expect(r2.siteId).toBe(r1.siteId)
expect(r2.connectionId).toBe(r1.connectionId)
expect(r2.connectionCreated).toBe(false)
expect(r2.snapshots).toHaveLength(3)
for (const s2 of r2.snapshots) {
const s1 = r1.snapshots.find((s: RegisteredSnapshot) => s.kind === s2.kind)!
expect(s2.id).toBe(s1.id)
expect(s2.sha256).toBe(s1.sha256)
expect(s2.reused).toBe(true)
}
const counts = (await db.execute(
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns,
(select count(*) from sites where key = '${key}') as sites`,
)) as any
expect(counts[0]).toEqual({ snaps: '3', conns: '1', sites: '1' })
} finally {
await cleanupSiteRows(db, r1.siteId)
}
})
})
})
it('副本布局 content-addressed:文件落 <kind>/<sha2>/<sha>,媒体整树落 <sha2>/<sha>/,内容一致', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
for (const s of r.snapshots) {
const sha2 = s.sha256.slice(0, 2)
expect(s.storagePath).toBe(`snapshots/${s.kind}/${sha2}/${s.sha256}`)
const dest = join(ctx.storageRoot, s.storagePath)
if (s.kind === 'media_bundle') {
// 整树:目录内相对结构保留,内容逐文件一致
expect(existsSync(join(dest, 'a.png'))).toBe(true)
expect(existsSync(join(dest, 'sub', 'b.png'))).toBe(true)
expect(await readFile(join(dest, 'a.png'))).toEqual(ctx.mediaFiles[0]!.content)
expect(await readFile(join(dest, 'sub', 'b.png'))).toEqual(ctx.mediaFiles[1]!.content)
} else {
const source = s.kind === 'wxr' ? WXR_FIXTURE : SITEMAP_FIXTURE
expect(await readFile(dest)).toEqual(await readFile(source))
}
// 副本路径 ≠ 源路径(后续任务只读副本)
expect(dest).not.toBe(s.sourcePath)
}
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('路径安全:白名单外/越界遍历/ENOENT/类型不符/无输入 均清晰报错且零落库', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const base: RegisterInput = {
siteKey: key,
allowedRoots: [ctx.work], // 注意:本用例故意不含 FIXTURES_ROOT
storageRoot: ctx.storageRoot,
reportRoot: ctx.reportRoot,
}
const cases: { label: string; input: RegisterInput; code: string }[] = [
{
label: '白名单外的绝对路径(本测试文件自身)',
input: { ...base, wxrPath: THIS_FILE },
code: 'PATH_NOT_ALLOWED',
},
{
label: '越界遍历 ../(白名单只含 incoming/,目标逃到 work 根)',
input: {
...base,
wxrPath: join(ctx.work, 'incoming', '..', 'escape.txt'),
allowedRoots: [join(ctx.work, 'incoming')],
},
code: 'PATH_NOT_ALLOWED',
},
{
label: 'ENOENT',
input: { ...base, wxrPath: join(ctx.work, 'not-exist.xml') },
code: 'INPUT_NOT_FOUND',
},
{
label: 'wxr 输入是目录',
input: { ...base, wxrPath: ctx.mediaDir },
code: 'INPUT_NOT_FILE',
},
{
label: 'media 输入是文件',
// 白名单放行 FIXTURES_ROOT 以穿过白名单检查,命中类型检查
input: { ...base, mediaDir: WXR_FIXTURE, allowedRoots: [ctx.work, FIXTURES_ROOT] },
code: 'INPUT_NOT_DIR',
},
{
label: '无任何输入',
input: { ...base },
code: 'NO_INPUTS',
},
]
// 越界遍历用例的配套文件:work 根下放 escape.txt,incoming/ 只是白名单根
await writeFile(join(ctx.work, 'escape.txt'), 'escape')
await mkdir(join(ctx.work, 'incoming'), { recursive: true })
for (const c of cases) {
const err: SourceRegisterError = await registerSource(db, c.input).then(
() => {
throw new Error(`用例「${c.label}」不应成功`)
},
(e: SourceRegisterError) => e,
)
expect(err, c.label).toBeInstanceOf(SourceRegisterError)
expect(err.code, c.label).toBe(c.code)
expect(err.message, c.label).toMatch(/越界|不存在|白名单|目录|文件|输入|路径/)
}
// 零落库:site 从未创建
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
expect(sites).toHaveLength(0)
})
})
})
it('符号链接(junction)在媒体枚举中被跳过并记录,清单不受其内容影响', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
// 白名单外目录放一个文件,junction 指过去——若被跟随既破坏白名单也污染清单
const outside = join(ctx.work, 'outside-pkg')
await mkdir(outside, { recursive: true })
await writeFile(join(outside, 'rogue.png'), 'rogue')
try {
await symlink(outside, join(ctx.mediaDir, 'linked-pkg'), 'junction')
} catch {
return // Windows 无开发者模式且非管理员:junction 创建失败,本用例静默放行
}
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
const bundle = r.snapshots.find((s: RegisteredSnapshot) => s.kind === 'media_bundle')!
const meta = bundle.metadata as { manifest: { path: string }[]; skippedSymlinks: string[] }
expect(meta.manifest.map((m) => m.path).sort()).toEqual(['a.png', 'sub/b.png'])
expect(meta.skippedSymlinks).toContain('linked-pkg')
// 副本树不含 junction 内容
expect(existsSync(join(ctx.storageRoot, bundle.storagePath, 'linked-pkg'))).toBe(false)
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('报告落盘 <reportRoot>/<runId>/reports/register.json:输入清单/哈希/大小/耗时', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
expect(r.reportPath).toBe(join(ctx.reportRoot, r.runId, 'reports', 'register.json'))
expect(existsSync(r.reportPath)).toBe(true)
const report = JSON.parse(await readFile(r.reportPath, 'utf8')) as {
runId: string
siteKey: string
siteId: string
inputs: { kind: string; sourcePath: string; sha256: string; byteSize: number; durationMs: number }[]
totals: { snapshots: number; reused: number; byteSize: number }
durationMs: number
}
expect(report.runId).toBe(r.runId)
expect(report.siteKey).toBe(key)
expect(report.siteId).toBe(r.siteId)
expect(report.inputs).toHaveLength(3)
const wxrInput = report.inputs.find((i) => i.kind === 'wxr')!
expect(wxrInput.sha256).toBe(sha256(await readFile(WXR_FIXTURE)))
expect(wxrInput.byteSize).toBe((await readFile(WXR_FIXTURE)).length)
expect(wxrInput.durationMs).toBeGreaterThanOrEqual(0)
expect(report.totals.snapshots).toBe(3)
expect(report.totals.reused).toBe(0)
expect(report.durationMs).toBeGreaterThanOrEqual(0)
// runId 形态:时间戳 + 随机(两次运行不同)
expect(r.runId).toMatch(/^\d{8}T\d{6}-[a-z0-9]+$/)
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
})