浅念
a227cfac49
Plan 3 Task 0（评审 Issue 1）：旧 cms 库 media_assets 的 source_url→file_path 映射只读导出为侧车 JSON（859 条，覆盖率对照 WXR 855 attachment_url：精确命中 836 / 归一命中 0 / 未命中 19，97.8%），register 新增 --media-map 输入通道 （kind=manual_url_list 快照，形状校验 INVALID_MEDIA_MAP，metadata 含条目数， 幂等复用）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
609 lines
28 KiB
TypeScript
609 lines
28 KiB
TypeScript
// Plan 2 Task 7 失败测试（先于实现）：source:register 服务——content-addressed 快照登记。
|
||
// 覆盖六组契约：
|
||
// 1. 三类输入（wxr/sitemap/media 目录）一次登记 → site 按 key 自动建、source_connections
|
||
// 恰 1 行（(site_id,name) 应用层判重）、source_snapshots 恰 3 行且 sha256 与源文件一致；
|
||
// 2. media_bundle 清单哈希定义钉死：排序相对路径（'/' 分隔）+ 每文件 sha256 → canonical
|
||
// JSON（键排序、无空白）→ 再 sha256；metadata 含完整清单（path/bytes/sha256）、不含 mtime；
|
||
// 3. 重复 register 同输入 → 复用既有 snapshot（UNIQUE(site,kind,sha256)，不新增行），
|
||
// 媒体目录重跑哈希一致（Windows 分隔符归一化 + mtime 不参与）；
|
||
// 4. 副本布局：storage/snapshots/<kind>/<sha256前2>/<sha256>（文件）或 <sha256>/（整树），
|
||
// 副本内容与原输入一致（后续任务只读副本）；
|
||
// 5. 路径安全：白名单外 / 越界遍历 / **中间段链接逃逸（整路径 realpath 后判定）** /
|
||
// ENOENT / 类型不符均清晰报错且零落库；
|
||
// 6. 报告落 artifacts/register/<runId>/reports/register.json（runId=时间戳+随机）；
|
||
// 7. 并发登记同输入恰好一组行（DB 单事务 + 唯一约束/advisory lock 收敛）、双方同 id；
|
||
// 8. 空媒体目录拒绝（EMPTY_MEDIA_DIR）、metadata 固化 storageRoot、siteCreated 语义；
|
||
// 9. media-map 侧车输入（Plan 3 Task 0）：kind=manual_url_list 快照、sha256=文件哈希、
|
||
// metadata 含条目数、幂等重跑复用；非法 JSON（非数组/条目字段缺失）→ INVALID_MEDIA_MAP 零落库。
|
||
// 运行于 globalSetup 临时库（ADR-015）；storage/报告根用 os.tmpdir 临时目录，finally 清理。
|
||
// 注：符号链接跳过用目录 junction 验证（Windows 免特权；非 Windows 退化为普通符号链接）；
|
||
// 创建失败（极端沙箱）时该断言静默放行——junction 创建不是本测试的被测契约。
|
||
import { createHash } from 'node:crypto'
|
||
import { existsSync } from 'node:fs'
|
||
import { mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from 'node:fs/promises'
|
||
import { tmpdir } from 'node:os'
|
||
import { join, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { describe, expect, it } from 'vitest'
|
||
import {
|
||
isUniqueViolation,
|
||
registerSource,
|
||
SourceRegisterError,
|
||
type RegisterInput,
|
||
type RegisteredSnapshot,
|
||
} from '../../apps/api/src/modules/source/register.service.js'
|
||
import { rand, withImmutableCleanup, withTestDb, type Db } from './helpers.js'
|
||
|
||
const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url))
|
||
const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml')
|
||
const SITEMAP_FIXTURE = resolve(FIXTURES_ROOT, 'sitemap', 'mini-sitemap.xml')
|
||
const THIS_FILE = fileURLToPath(import.meta.url)
|
||
|
||
const sha256 = (data: Buffer | string): string => createHash('sha256').update(data).digest('hex')
|
||
|
||
/** 测试侧独立实现的 canonical JSON（键排序、无空白）——不 import 实现，钉死定义 */
|
||
function canonicalJson(value: unknown): string {
|
||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`
|
||
if (value !== null && typeof value === 'object') {
|
||
const record = value as Record<string, unknown>
|
||
const body = Object.keys(record)
|
||
.sort()
|
||
.map((k) => `${JSON.stringify(k)}:${canonicalJson(record[k])}`)
|
||
.join(',')
|
||
return `{${body}}`
|
||
}
|
||
return JSON.stringify(value)
|
||
}
|
||
|
||
interface MediaFile {
|
||
path: string
|
||
bytes: number
|
||
content: Buffer
|
||
}
|
||
|
||
interface Ctx {
|
||
work: string
|
||
storageRoot: string
|
||
reportRoot: string
|
||
mediaDir: string
|
||
mediaFiles: MediaFile[]
|
||
allowedRoots: string[]
|
||
}
|
||
|
||
/** 独立媒体树：a.png + sub/b.png（内容固定，期望哈希可在测试内独立重算） */
|
||
async function makeMediaTree(work: string): Promise<{ mediaDir: string; mediaFiles: MediaFile[] }> {
|
||
const mediaDir = join(work, 'media')
|
||
await mkdir(join(mediaDir, 'sub'), { recursive: true })
|
||
const a = Buffer.from('sf-media-a-content')
|
||
const b = Buffer.from('sf-media-sub-b-content')
|
||
await writeFile(join(mediaDir, 'a.png'), a)
|
||
await writeFile(join(mediaDir, 'sub', 'b.png'), b)
|
||
return {
|
||
mediaDir,
|
||
mediaFiles: [
|
||
{ path: 'a.png', bytes: a.length, content: a },
|
||
{ path: 'sub/b.png', bytes: b.length, content: b },
|
||
],
|
||
}
|
||
}
|
||
|
||
async function withCtx(fn: (ctx: Ctx) => Promise<void>): Promise<void> {
|
||
const work = await mkdtemp(join(tmpdir(), 'sf-register-'))
|
||
const { mediaDir, mediaFiles } = await makeMediaTree(work)
|
||
const ctx: Ctx = {
|
||
work,
|
||
storageRoot: join(work, 'storage'),
|
||
reportRoot: join(work, 'artifacts', 'register'),
|
||
mediaDir,
|
||
mediaFiles,
|
||
allowedRoots: [FIXTURES_ROOT, work],
|
||
}
|
||
try {
|
||
await fn(ctx)
|
||
} finally {
|
||
await rm(work, { recursive: true, force: true })
|
||
}
|
||
}
|
||
|
||
function makeInput(ctx: Ctx, siteKey: string): RegisterInput {
|
||
return {
|
||
siteKey,
|
||
wxrPath: WXR_FIXTURE,
|
||
sitemapPath: SITEMAP_FIXTURE,
|
||
mediaDir: ctx.mediaDir,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.reportRoot,
|
||
}
|
||
}
|
||
|
||
/** 服务创建的 site 行清场：快照不可变走触发器开关，FK 反序删 connection → site */
|
||
async function cleanupSiteRows(db: Db, siteId: string): Promise<void> {
|
||
await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`)
|
||
await db.execute(`delete from source_connections where site_id = '${siteId}'`)
|
||
await db.execute(`delete from sites where id = '${siteId}'`)
|
||
}
|
||
|
||
describe('source:register 服务', () => {
|
||
it('三类输入一次登记：site 自动创建、connection 1 行、snapshot 3 行（sha256 与源一致）', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const r = await registerSource(db, makeInput(ctx, key))
|
||
try {
|
||
// site 按 key 自动 upsert（不存在则建，默认列落库）
|
||
const siteRows = (await db.execute(`select id, key, default_locale from sites where key = '${key}'`)) as any
|
||
expect(siteRows).toHaveLength(1)
|
||
expect(siteRows[0].id).toBe(r.siteId)
|
||
expect(siteRows[0].default_locale).toBe('en')
|
||
|
||
// connection 恰 1 行（(site_id,name) 应用层判重）
|
||
const connRows = (await db.execute(
|
||
`select id, kind, name from source_connections where site_id = '${r.siteId}'`,
|
||
)) as any
|
||
expect(connRows).toHaveLength(1)
|
||
expect(connRows[0].kind).toBe('wxr')
|
||
expect(connRows[0].name).toBe('default')
|
||
expect(connRows[0].id).toBe(r.connectionId)
|
||
|
||
// snapshot 恰 3 行，kind 各异，sha256 与源文件一致，storage_path 指向存在的副本
|
||
const snapRows = (await db.execute(
|
||
`select kind, sha256, byte_size, storage_path from source_snapshots where site_id = '${r.siteId}'`,
|
||
)) as { kind: string; sha256: string; storage_path: string }[]
|
||
expect(snapRows).toHaveLength(3)
|
||
const byKind = new Map(snapRows.map((row) => [row.kind, row]))
|
||
expect(byKind.get('wxr')?.sha256).toBe(sha256(await readFile(WXR_FIXTURE)))
|
||
expect(byKind.get('sitemap')?.sha256).toBe(sha256(await readFile(SITEMAP_FIXTURE)))
|
||
for (const row of snapRows) {
|
||
expect(existsSync(join(ctx.storageRoot, row.storage_path))).toBe(true)
|
||
}
|
||
expect(r.snapshots).toHaveLength(3)
|
||
expect(new Set(r.snapshots.map((s: RegisteredSnapshot) => s.kind))).toEqual(
|
||
new Set(['wxr', 'sitemap', 'media_bundle']),
|
||
)
|
||
} finally {
|
||
await cleanupSiteRows(db, r.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('media_bundle 清单哈希 = 排序路径+每文件 sha256 的 canonical JSON 再哈希；metadata 无 mtime', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const r = await registerSource(db, makeInput(ctx, key))
|
||
try {
|
||
const bundle = r.snapshots.find((s: RegisteredSnapshot) => s.kind === 'media_bundle')!
|
||
// 测试侧独立重算（排序 + canonical JSON 键序无空白）
|
||
const entries = ctx.mediaFiles
|
||
.map((f) => ({ path: f.path, sha256: sha256(f.content) }))
|
||
.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
|
||
expect(bundle.sha256).toBe(sha256(canonicalJson(entries)))
|
||
// metadata 含完整清单（path/bytes/sha256），不含 mtime
|
||
const manifest = (bundle.metadata as { manifest: { path: string; bytes: number; sha256: string }[] }).manifest
|
||
expect([...manifest].sort((a, b) => (a.path < b.path ? -1 : 1))).toEqual([
|
||
{ path: 'a.png', bytes: ctx.mediaFiles[0]!.bytes, sha256: sha256(ctx.mediaFiles[0]!.content) },
|
||
{ path: 'sub/b.png', bytes: ctx.mediaFiles[1]!.bytes, sha256: sha256(ctx.mediaFiles[1]!.content) },
|
||
])
|
||
expect(JSON.stringify(bundle.metadata)).not.toContain('mtime')
|
||
// byteSize = 清单文件字节数之和
|
||
expect(bundle.byteSize).toBe(ctx.mediaFiles.reduce((sum, f) => sum + f.bytes, 0))
|
||
} finally {
|
||
await cleanupSiteRows(db, r.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('重复 register 同输入：复用既有 snapshot/connection/site，不新增行；mtime 变化不影响哈希', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const r1 = await registerSource(db, makeInput(ctx, key))
|
||
try {
|
||
// 播动 mtime（证明清单哈希不含 mtime）后重跑
|
||
const t = new Date(Date.now() + 60_000)
|
||
await utimes(join(ctx.mediaDir, 'a.png'), t, t)
|
||
await utimes(WXR_FIXTURE, t, t).catch(() => undefined) // 夹具可能只读；媒体树已足够证明
|
||
|
||
const r2 = await registerSource(db, makeInput(ctx, key))
|
||
expect(r2.siteId).toBe(r1.siteId)
|
||
expect(r2.siteCreated).toBe(false)
|
||
expect(r1.siteCreated).toBe(true)
|
||
expect(r2.connectionId).toBe(r1.connectionId)
|
||
expect(r2.connectionCreated).toBe(false)
|
||
expect(r2.snapshots).toHaveLength(3)
|
||
for (const s2 of r2.snapshots) {
|
||
const s1 = r1.snapshots.find((s: RegisteredSnapshot) => s.kind === s2.kind)!
|
||
expect(s2.id).toBe(s1.id)
|
||
expect(s2.sha256).toBe(s1.sha256)
|
||
expect(s2.reused).toBe(true)
|
||
}
|
||
const counts = (await db.execute(
|
||
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
|
||
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns,
|
||
(select count(*) from sites where key = '${key}') as sites`,
|
||
)) as any
|
||
expect(counts[0]).toEqual({ snaps: '3', conns: '1', sites: '1' })
|
||
} finally {
|
||
await cleanupSiteRows(db, r1.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('副本布局 content-addressed：文件落 <kind>/<sha2>/<sha>，媒体整树落 <sha2>/<sha>/，内容一致', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const r = await registerSource(db, makeInput(ctx, key))
|
||
try {
|
||
for (const s of r.snapshots) {
|
||
const sha2 = s.sha256.slice(0, 2)
|
||
expect(s.storagePath).toBe(`snapshots/${s.kind}/${sha2}/${s.sha256}`)
|
||
const dest = join(ctx.storageRoot, s.storagePath)
|
||
if (s.kind === 'media_bundle') {
|
||
// 整树：目录内相对结构保留，内容逐文件一致
|
||
expect(existsSync(join(dest, 'a.png'))).toBe(true)
|
||
expect(existsSync(join(dest, 'sub', 'b.png'))).toBe(true)
|
||
expect(await readFile(join(dest, 'a.png'))).toEqual(ctx.mediaFiles[0]!.content)
|
||
expect(await readFile(join(dest, 'sub', 'b.png'))).toEqual(ctx.mediaFiles[1]!.content)
|
||
} else {
|
||
const source = s.kind === 'wxr' ? WXR_FIXTURE : SITEMAP_FIXTURE
|
||
expect(await readFile(dest)).toEqual(await readFile(source))
|
||
}
|
||
// 副本路径 ≠ 源路径（后续任务只读副本）
|
||
expect(dest).not.toBe(s.sourcePath)
|
||
// storageRoot 固化进 metadata（快照不可变，事后无法补写）
|
||
expect((s.metadata as { storageRoot: string }).storageRoot).toBe(resolve(ctx.storageRoot))
|
||
}
|
||
} finally {
|
||
await cleanupSiteRows(db, r.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('路径安全：白名单外/越界遍历/ENOENT/类型不符/无输入 均清晰报错且零落库', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const base: RegisterInput = {
|
||
siteKey: key,
|
||
allowedRoots: [ctx.work], // 注意：本用例故意不含 FIXTURES_ROOT
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.reportRoot,
|
||
}
|
||
const cases: { label: string; input: RegisterInput; code: string }[] = [
|
||
{
|
||
label: '白名单外的绝对路径（本测试文件自身）',
|
||
input: { ...base, wxrPath: THIS_FILE },
|
||
code: 'PATH_NOT_ALLOWED',
|
||
},
|
||
{
|
||
label: '越界遍历 ../（白名单只含 incoming/，目标逃到 work 根）',
|
||
input: {
|
||
...base,
|
||
wxrPath: join(ctx.work, 'incoming', '..', 'escape.txt'),
|
||
allowedRoots: [join(ctx.work, 'incoming')],
|
||
},
|
||
code: 'PATH_NOT_ALLOWED',
|
||
},
|
||
{
|
||
label: 'ENOENT',
|
||
input: { ...base, wxrPath: join(ctx.work, 'not-exist.xml') },
|
||
code: 'INPUT_NOT_FOUND',
|
||
},
|
||
{
|
||
label: 'wxr 输入是目录',
|
||
input: { ...base, wxrPath: ctx.mediaDir },
|
||
code: 'INPUT_NOT_FILE',
|
||
},
|
||
{
|
||
label: 'media 输入是文件',
|
||
// 白名单放行 FIXTURES_ROOT 以穿过白名单检查，命中类型检查
|
||
input: { ...base, mediaDir: WXR_FIXTURE, allowedRoots: [ctx.work, FIXTURES_ROOT] },
|
||
code: 'INPUT_NOT_DIR',
|
||
},
|
||
{
|
||
label: '无任何输入',
|
||
input: { ...base },
|
||
code: 'NO_INPUTS',
|
||
},
|
||
{
|
||
label: '空媒体目录',
|
||
input: { ...base, mediaDir: join(ctx.work, 'empty-media') },
|
||
code: 'EMPTY_MEDIA_DIR',
|
||
},
|
||
]
|
||
// 用例配套布局：work 根放 escape.txt（越界目标）、incoming/（收窄白名单根）、empty-media/（空目录）
|
||
await writeFile(join(ctx.work, 'escape.txt'), 'escape')
|
||
await mkdir(join(ctx.work, 'incoming'), { recursive: true })
|
||
await mkdir(join(ctx.work, 'empty-media'), { recursive: true })
|
||
|
||
for (const c of cases) {
|
||
const err: SourceRegisterError = await registerSource(db, c.input).then(
|
||
() => {
|
||
throw new Error(`用例「${c.label}」不应成功`)
|
||
},
|
||
(e: SourceRegisterError) => e,
|
||
)
|
||
expect(err, c.label).toBeInstanceOf(SourceRegisterError)
|
||
expect(err.code, c.label).toBe(c.code)
|
||
expect(err.message, c.label).toMatch(/越界|不存在|白名单|目录|文件|输入|路径/)
|
||
}
|
||
// 零落库：site 从未创建
|
||
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
|
||
expect(sites).toHaveLength(0)
|
||
})
|
||
})
|
||
})
|
||
|
||
it('中间段 junction 逃逸被拒：目录段含指向白名单外的链接，整路径按 realpath 判定', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
// 布局（复现评审 PoC 形态）：白名单根 incoming/ 内放 junction，目标 pkg/ 在白名单外；
|
||
// 文件物理落在 pkg/escape.xml——词法前缀判定会把 incoming/junc/escape.xml 误判为界内
|
||
const pkg = join(ctx.work, 'pkg-outside')
|
||
const incoming = join(ctx.work, 'incoming')
|
||
await mkdir(pkg, { recursive: true })
|
||
await mkdir(incoming, { recursive: true })
|
||
await writeFile(join(pkg, 'escape.xml'), 'escaped')
|
||
try {
|
||
await symlink(pkg, join(incoming, 'junc'), 'junction')
|
||
} catch {
|
||
return // junction 创建失败（极端沙箱）：静默放行，创建能力不是被测契约
|
||
}
|
||
const key = `sfreg-${rand()}`
|
||
const err: SourceRegisterError = await registerSource(db, {
|
||
siteKey: key,
|
||
wxrPath: join(incoming, 'junc', 'escape.xml'),
|
||
allowedRoots: [incoming],
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.reportRoot,
|
||
}).then(
|
||
() => {
|
||
throw new Error('中间段 junction 逃逸不应成功')
|
||
},
|
||
(e: SourceRegisterError) => e,
|
||
)
|
||
expect(err).toBeInstanceOf(SourceRegisterError)
|
||
expect(err.code).toBe('PATH_NOT_ALLOWED')
|
||
expect(err.message).toContain('越界')
|
||
// 零落库
|
||
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
|
||
expect(sites).toHaveLength(0)
|
||
})
|
||
})
|
||
})
|
||
|
||
it('符号链接（junction）在媒体枚举中被跳过并记录，清单不受其内容影响', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
// 白名单外目录放一个文件，junction 指过去——若被跟随既破坏白名单也污染清单
|
||
const outside = join(ctx.work, 'outside-pkg')
|
||
await mkdir(outside, { recursive: true })
|
||
await writeFile(join(outside, 'rogue.png'), 'rogue')
|
||
try {
|
||
await symlink(outside, join(ctx.mediaDir, 'linked-pkg'), 'junction')
|
||
} catch {
|
||
return // Windows 无开发者模式且非管理员：junction 创建失败，本用例静默放行
|
||
}
|
||
const key = `sfreg-${rand()}`
|
||
const r = await registerSource(db, makeInput(ctx, key))
|
||
try {
|
||
const bundle = r.snapshots.find((s: RegisteredSnapshot) => s.kind === 'media_bundle')!
|
||
const meta = bundle.metadata as { manifest: { path: string }[]; skippedSymlinks: string[] }
|
||
expect(meta.manifest.map((m) => m.path).sort()).toEqual(['a.png', 'sub/b.png'])
|
||
expect(meta.skippedSymlinks).toContain('linked-pkg')
|
||
// 副本树不含 junction 内容
|
||
expect(existsSync(join(ctx.storageRoot, bundle.storagePath, 'linked-pkg'))).toBe(false)
|
||
} finally {
|
||
await cleanupSiteRows(db, r.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('报告落盘 <reportRoot>/<runId>/reports/register.json：输入清单/哈希/大小/耗时', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const r = await registerSource(db, makeInput(ctx, key))
|
||
try {
|
||
expect(r.reportPath).toBe(join(ctx.reportRoot, r.runId, 'reports', 'register.json'))
|
||
expect(existsSync(r.reportPath)).toBe(true)
|
||
const report = JSON.parse(await readFile(r.reportPath, 'utf8')) as {
|
||
runId: string
|
||
siteKey: string
|
||
siteId: string
|
||
storageRoot: string
|
||
inputs: { kind: string; sourcePath: string; sha256: string; byteSize: number; storageRoot: string; durationMs: number }[]
|
||
totals: { snapshots: number; reused: number; byteSize: number }
|
||
durationMs: number
|
||
}
|
||
expect(report.runId).toBe(r.runId)
|
||
expect(report.siteKey).toBe(key)
|
||
expect(report.siteId).toBe(r.siteId)
|
||
expect(report.storageRoot).toBe(resolve(ctx.storageRoot))
|
||
expect(report.inputs).toHaveLength(3)
|
||
const wxrInput = report.inputs.find((i) => i.kind === 'wxr')!
|
||
expect(wxrInput.sha256).toBe(sha256(await readFile(WXR_FIXTURE)))
|
||
expect(wxrInput.byteSize).toBe((await readFile(WXR_FIXTURE)).length)
|
||
expect(wxrInput.storageRoot).toBe(resolve(ctx.storageRoot))
|
||
expect(wxrInput.durationMs).toBeGreaterThanOrEqual(0)
|
||
expect(report.totals.snapshots).toBe(3)
|
||
expect(report.totals.reused).toBe(0)
|
||
expect(report.durationMs).toBeGreaterThanOrEqual(0)
|
||
// runId 形态：时间戳 + 随机（两次运行不同）
|
||
expect(r.runId).toMatch(/^\d{8}T\d{6}-[a-z0-9]+$/)
|
||
} finally {
|
||
await cleanupSiteRows(db, r.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('并发 register 同输入：恰好一组行，双方返回同一组 id（事务 + 唯一约束/advisory lock 收敛）', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const input: RegisterInput = {
|
||
siteKey: key,
|
||
wxrPath: WXR_FIXTURE,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.reportRoot,
|
||
}
|
||
const [r1, r2] = await Promise.all([registerSource(db, input), registerSource(db, input)])
|
||
try {
|
||
// 并发终态契约：同 id、同快照；"谁新建"分支按竞态分配，但恰好各一方
|
||
expect(r1.siteId).toBe(r2.siteId)
|
||
expect(r1.connectionId).toBe(r2.connectionId)
|
||
expect(r1.snapshots.map((s: RegisteredSnapshot) => s.id)).toEqual(
|
||
r2.snapshots.map((s: RegisteredSnapshot) => s.id),
|
||
)
|
||
expect(Number(r1.siteCreated) + Number(r2.siteCreated)).toBe(1)
|
||
expect(Number(r1.connectionCreated) + Number(r2.connectionCreated)).toBe(1)
|
||
expect(
|
||
r1.snapshots.map((s: RegisteredSnapshot) => !s.reused).filter(Boolean).length +
|
||
r2.snapshots.map((s: RegisteredSnapshot) => !s.reused).filter(Boolean).length,
|
||
).toBe(1)
|
||
const counts = (await db.execute(
|
||
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
|
||
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns,
|
||
(select count(*) from sites where key = '${key}') as sites`,
|
||
)) as any
|
||
expect(counts[0]).toEqual({ snaps: '1', conns: '1', sites: '1' })
|
||
} finally {
|
||
await cleanupSiteRows(db, r1.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('media-map 输入：kind=manual_url_list 快照（sha256=文件哈希、metadata 含条目数），幂等重跑复用', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
// 侧车 JSON（Plan 3 Task 0 形态）：[{sourceUrl, filePath}] 数组——真实导出脚本的输出同构
|
||
const entries = [
|
||
{
|
||
sourceUrl: 'https://hesupipe.com/wp-content/uploads/2019/12/Amazon.png',
|
||
filePath: 'f249747a-50d3-48ed-a28a-981cf1d5ee7f.png',
|
||
},
|
||
{
|
||
sourceUrl: 'https://hesupipe.com/wp-content/uploads/2019/12/map-location.jpg',
|
||
filePath: '0e6d0fd7-8d5f-4f57-a541-40a8f2cb9c02.jpg',
|
||
},
|
||
]
|
||
const mapContent = `${JSON.stringify(entries, null, 2)}\n`
|
||
const mapFile = join(ctx.work, 'media-map-hesupipe.json')
|
||
await writeFile(mapFile, mapContent, 'utf8')
|
||
|
||
const key = `sfreg-${rand()}`
|
||
const makeMapInput = (): RegisterInput => ({
|
||
siteKey: key,
|
||
mediaMapPath: mapFile,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.reportRoot,
|
||
})
|
||
const r1 = await registerSource(db, makeMapInput())
|
||
try {
|
||
expect(r1.snapshots).toHaveLength(1)
|
||
const s = r1.snapshots[0]!
|
||
expect(s.kind).toBe('manual_url_list')
|
||
// sha256 = 文件内容哈希（与 wxr/sitemap 文件输入同口径）
|
||
expect(s.sha256).toBe(sha256(mapContent))
|
||
expect(s.byteSize).toBe(Buffer.byteLength(mapContent))
|
||
expect(s.storagePath).toBe(`snapshots/manual_url_list/${s.sha256.slice(0, 2)}/${s.sha256}`)
|
||
expect(existsSync(join(ctx.storageRoot, s.storagePath))).toBe(true)
|
||
expect(await readFile(join(ctx.storageRoot, s.storagePath), 'utf8')).toBe(mapContent)
|
||
// metadata 含条目数（Task 0 契约）与 sourceName；storageRoot 固化
|
||
expect(s.metadata).toMatchObject({ entryCount: 2, sourceName: 'media-map-hesupipe.json' })
|
||
expect((s.metadata as { storageRoot: string }).storageRoot).toBe(resolve(ctx.storageRoot))
|
||
|
||
// DB 行：kind 枚举值落库正确、metadata jsonb 含条目数
|
||
const rows = (await db.execute(
|
||
`select kind, sha256, byte_size, storage_path, metadata from source_snapshots where site_id = '${r1.siteId}'`,
|
||
)) as { kind: string; sha256: string; byte_size: string; storage_path: string; metadata: { entryCount: number } }[]
|
||
expect(rows).toHaveLength(1)
|
||
expect(rows[0]!.kind).toBe('manual_url_list')
|
||
expect(rows[0]!.sha256).toBe(sha256(mapContent))
|
||
expect(Number(rows[0]!.byte_size)).toBe(Buffer.byteLength(mapContent))
|
||
expect(rows[0]!.storage_path).toBe(s.storagePath)
|
||
expect(rows[0]!.metadata.entryCount).toBe(2)
|
||
|
||
// 幂等重跑：UNIQUE(site,kind,sha256) 复用既有行，不新增
|
||
const r2 = await registerSource(db, makeMapInput())
|
||
expect(r2.siteId).toBe(r1.siteId)
|
||
expect(r2.snapshots).toHaveLength(1)
|
||
expect(r2.snapshots[0]!.id).toBe(s.id)
|
||
expect(r2.snapshots[0]!.reused).toBe(true)
|
||
const counts = (await db.execute(
|
||
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
|
||
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns`,
|
||
)) as any
|
||
expect(counts[0]).toEqual({ snaps: '1', conns: '1' })
|
||
} finally {
|
||
await cleanupSiteRows(db, r1.siteId)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('media-map 非法内容（非 JSON / 非数组 / 条目字段缺失或非串）：INVALID_MEDIA_MAP 报错且零落库', async () => {
|
||
await withTestDb(async (db) => {
|
||
await withCtx(async (ctx) => {
|
||
const key = `sfreg-${rand()}`
|
||
const cases: { label: string; content: string }[] = [
|
||
{ label: '非 JSON 文本', content: 'not json at all\n' },
|
||
{ label: 'JSON 但非数组', content: '{"sourceUrl": "x"}\n' },
|
||
{ label: '数组但条目缺 filePath', content: '[{"sourceUrl": "https://a.example/1.png"}]\n' },
|
||
{ label: '数组但条目字段非字符串', content: '[{"sourceUrl": 1, "filePath": "a.png"}]\n' },
|
||
]
|
||
for (const c of cases) {
|
||
const mapFile = join(ctx.work, `bad-map-${cases.indexOf(c)}.json`)
|
||
await writeFile(mapFile, c.content, 'utf8')
|
||
const err: SourceRegisterError = await registerSource(db, {
|
||
siteKey: key,
|
||
mediaMapPath: mapFile,
|
||
allowedRoots: ctx.allowedRoots,
|
||
storageRoot: ctx.storageRoot,
|
||
reportRoot: ctx.reportRoot,
|
||
}).then(
|
||
() => {
|
||
throw new Error(`用例「${c.label}」不应成功`)
|
||
},
|
||
(e: SourceRegisterError) => e,
|
||
)
|
||
expect(err, c.label).toBeInstanceOf(SourceRegisterError)
|
||
expect(err.code, c.label).toBe('INVALID_MEDIA_MAP')
|
||
expect(err.message, c.label).toMatch(/media-map/)
|
||
// 零落库：site 从未创建
|
||
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
|
||
expect(sites, c.label).toHaveLength(0)
|
||
}
|
||
})
|
||
})
|
||
})
|
||
|
||
it('isUniqueViolation 形状识别：drizzle 包裹（cause.code）/裸 postgres code/其他均正确', () => {
|
||
// drizzle 会把 postgres 错误包成 DrizzleQueryError：message 只剩 "Failed query"，真 code 在 cause
|
||
const drizzleWrapped = Object.assign(new Error('Failed query: insert into sites ...'), {
|
||
cause: Object.assign(new Error('duplicate key value violates unique constraint "sites_key_unique"'), {
|
||
code: '23505',
|
||
}),
|
||
})
|
||
expect(isUniqueViolation(drizzleWrapped)).toBe(true)
|
||
// 裸 postgres 错误（code 直挂）
|
||
expect(isUniqueViolation(Object.assign(new Error('dup'), { code: '23505' }))).toBe(true)
|
||
// 其他错误码 / 无 code → false
|
||
expect(isUniqueViolation(Object.assign(new Error('x'), { cause: { code: '23508' } }))).toBe(false)
|
||
expect(isUniqueViolation(Object.assign(new Error('x'), { code: '42P01' }))).toBe(false)
|
||
expect(isUniqueViolation(new Error('plain'))).toBe(false)
|
||
expect(isUniqueViolation(null)).toBe(false)
|
||
})
|
||
})
|