feat(source): 输入登记服务与 sf CLI(content-addressed 快照)

- registerSource(db, input):wxr/sitemap/媒体目录三类输入登记为 content-addressed
  快照——流式 copy+sha256 单遍双算(64KB 块),副本落
  storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(媒体整树),
  后续任务只读副本;UNIQUE(site,kind,sha256) 幂等(重复登记复用既有行)
- media_bundle 哈希钉死:排序相对路径('/' 归一)+ 每文件 sha256 → canonical
  JSON(键排序无空白)→ 再 sha256;不含 mtime、不依赖遍历顺序;完整清单
  (path/bytes/sha256)入 metadata jsonb;符号链接(含 junction)跳过并记录
- site 按 key select-then-insert upsert(23505 竞态重查);connection 判重键
  (site_id,name) 为应用层 select-then-insert(表无库级唯一,注释注明)
- 路径安全:输入 realpath 后须在 allowedRoots 白名单内(缺省 storage/incoming),
  ENOENT/越界/类型不符抛带 code 的 SourceRegisterError;全部输入先校验后摄取,
  任何失败零落库零复制
- 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机;
  输入清单/哈希/大小/耗时)
- scripts/sf.ts(commander):wxr:probe 只读诊断(任意可读路径,IO/损坏文件出
  报告不崩——Task 6 fatalError 契约)出 JSON/HTML 报告;source:register 调服务
  出摘要。CLI 机制钉死:根 devDep tsx + 根 script sf + commander 根依赖,TS 源码
  直连(@sf/* 经 tsconfig paths,不经 dist)
- source.module.ts:Nest 注册 SourceRegisterService(standalone context 用);
  register.service.ts 刻意零装饰器(CLI/测试直 import 不拉 reflect-metadata)
- launder 类型垫片改经 triple-slash 引用随 html.ts 进任意编译程序(apps/api 与
  根程序直连适配器源码的新场景);apps/api 增 @sf/adapter-wp workspace 依赖与
  paths 映射(register.service 消费 ADAPTER_VERSION 单源)
- 测试 +7(tests/integration/source-register.test.ts,临时库 + tmpdir storage/
  报告根 finally 清理):三输入登记/清单哈希定义对拍/重复登记幂等/mtime 排除/
  副本布局与内容一致/路径安全六分支/junction 跳过/报告契约;全套 260 绿

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-10-21 12:31:15 +08:00
parent a535cde270
commit 586d715455
10 changed files with 1246 additions and 3 deletions
+1
View File
@@ -12,6 +12,7 @@
"dependencies": {
"@nestjs/common": "^10.4.22",
"@nestjs/core": "^10.4.22",
"@sf/adapter-wp": "workspace:*",
"drizzle-orm": "^0.45.4",
"postgres": "^3.4.9",
"reflect-metadata": "^0.2.2",
@@ -0,0 +1,654 @@
// source:register 输入登记服务(Plan 2 Task 7,spec §7.1 / ADR-004/008):
// 把 WXR / sitemap / 媒体目录三类输入登记为 content-addressed 快照——
// 副本落 storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(媒体整树),
// 源输入文件从此不再被后续任务读取(导入只寻址副本)。
//
// 本文件刻意保持 **零装饰器**:CLI(scripts/sf.ts 经 tsx)与集成测试直接 import 本文件,
// 不应因此拉起 Nest 装饰器执行(@Module/@Injectable 需 reflect-metadata 先行,见
// source.module.ts 的 standalone 说明)。Nest 侧由 SourceRegisterService 薄包装注入。
//
// 哈希定义(钉死,测试独立重算对拍):
// - 文件输入:内容 sha256(流式,64KB 块,copy 与 hash 单遍双算);
// - media_bundle:排序相对路径(分隔符统一 '/')+ 每文件 sha256 → canonical JSON
// (对象键排序、无空白)→ 对该 JSON 串再 sha256。不含 mtime、不依赖遍历顺序。
//
// 幂等:source_snapshots UNIQUE(site_id, kind, sha256)——同输入重复 register 复用既有行;
// site 按 key select-then-insert upsert;connection 判重键 (site_id, name) **无库级唯一
// 约束**,靠应用层 select-then-insert 判重(本服务是唯一写入方;引入并发写入方时须补库级约束)。
//
// 路径安全:输入路径(realpath 后)必须位于 allowedRoots 白名单内——storage/incoming 是
// 常规通道,显式白名单供测试/工具放行夹具目录;越界与 ENOENT 均抛带 reason code 的
// SourceRegisterError。wxr:probe(CLI 只读诊断)不走本服务,无白名单。
import { createHash, randomUUID } from 'node:crypto'
import { createReadStream, createWriteStream, existsSync } from 'node:fs'
import { lstat, mkdir, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises'
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { pipeline } from 'node:stream/promises'
import { and, eq } from 'drizzle-orm'
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
import { ADAPTER_VERSION } from '@sf/adapter-wp'
import { sites, sourceConnections, sourceSnapshots } from '../../db/schema.js'
import type * as schema from '../../db/schema.js'
/** 本文件位于 <repo>/apps/api/src/modules/source/ → 上五级是仓库根 */
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..')
/** 流式读写块大小(copy 与 hash 同块;64KB 与 maxWxrBytes 限额家族同一量级约定) */
const CHUNK_SIZE = 64 * 1024
export type SourceDb = NodePgDatabase<typeof schema>
export type SourceSnapshotKind = 'wxr' | 'sitemap' | 'media_bundle'
/** 各 kind 快照的 adapter_version:wxr 用适配器包单源版本;sitemap/media 尚无解析器,占位钉死 */
const SNAPSHOT_ADAPTER_VERSION: Record<SourceSnapshotKind, string> = {
wxr: `adapter-wp@${ADAPTER_VERSION}`,
sitemap: 'sitemap@0',
media_bundle: 'media-bundle@1',
}
export type SourceRegisterErrorCode =
| 'NO_INPUTS'
| 'PATH_NOT_ALLOWED'
| 'INPUT_NOT_FOUND'
| 'INPUT_NOT_FILE'
| 'INPUT_NOT_DIR'
| 'INPUT_READ_FAILED'
| 'COPY_FAILED'
/** 登记失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */
export class SourceRegisterError extends Error {
readonly code: SourceRegisterErrorCode
constructor(code: SourceRegisterErrorCode, message: string) {
super(message)
this.name = 'SourceRegisterError'
this.code = code
}
}
export interface RegisterSiteDefaults {
name?: string
defaultLocale?: string
productionOrigin?: string
}
export interface RegisterInput {
siteKey: string
wxrPath?: string
sitemapPath?: string
mediaDir?: string
/** 输入路径白名单(realpath 后必须位于其中之一);缺省 [repoRoot/storage/incoming] */
allowedRoots?: string[]
/** 快照存储根(副本落 <storageRoot>/snapshots/...);缺省 repoRoot/storage */
storageRoot?: string
/** 报告根(落 <reportRoot>/<runId>/reports/register.json);缺省 repoRoot/artifacts/register */
reportRoot?: string
/** 连接名——(site_id, name) 应用层判重键的 name 侧;缺省 'default' */
connectionName?: string
/** 站点不存在时的建站默认列 */
siteDefaults?: RegisterSiteDefaults
}
export interface RegisteredSnapshot {
kind: SourceSnapshotKind
/** snapshot 行 id */
id: string
sha256: string
byteSize: number
/** 相对 storageRoot 的副本路径(posix 分隔符,跨平台稳定) */
storagePath: string
/** 原输入绝对路径(审计用;副本才是后续任务的读取对象) */
sourcePath: string
/** 本次是否新插入行(false = UNIQUE(site,kind,sha256) 复用既有快照) */
reused: boolean
/** 本次是否写入副本文件(false = content-addressed 副本已存在) */
copied: boolean
durationMs: number
metadata: Record<string, unknown>
}
export interface RegisterResult {
runId: string
siteKey: string
siteId: string
connectionId: string
connectionCreated: boolean
snapshots: RegisteredSnapshot[]
reportPath: string
durationMs: number
}
export interface MediaManifestEntry {
/** 相对媒体目录根的路径,分隔符统一 '/' */
path: string
bytes: number
sha256: string
}
// ---- 哈希与副本 ----
/** canonical JSON(对象键递归排序、无空白)——media_bundle 清单哈希的序列化基形 */
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`
if (value !== null && typeof value === 'object') {
const record = value as Record<string, unknown>
const body = Object.keys(record)
.sort()
.map((k) => `${JSON.stringify(k)}:${canonicalJson(record[k])}`)
.join(',')
return `{${body}}`
}
return JSON.stringify(value)
}
/** 流式单遍 copy+sha256 双算:读 64KB 块 → 哈希更新 → 写目标(pipeline 管背压与错误传播) */
async function copyAndHash(src: string, dest: string): Promise<{ sha256: string; bytes: number }> {
const hash = createHash('sha256')
let bytes = 0
const through = async function* (chunks: AsyncIterable<Buffer>) {
for await (const chunk of chunks) {
hash.update(chunk)
bytes += chunk.length
yield chunk
}
}
try {
await pipeline(
createReadStream(src, { highWaterMark: CHUNK_SIZE }),
through,
createWriteStream(dest, { highWaterMark: CHUNK_SIZE }),
)
} catch (err) {
await rm(dest, { force: true })
throw err
}
return { sha256: hash.digest('hex'), bytes }
}
/** 流式哈希单文件(媒体扫描阶段用——只读不复制) */
async function hashFile(src: string): Promise<{ sha256: string; bytes: number }> {
const hash = createHash('sha256')
let bytes = 0
for await (const chunk of createReadStream(src, { highWaterMark: CHUNK_SIZE })) {
hash.update(chunk)
bytes += chunk.length
}
return { sha256: hash.digest('hex'), bytes }
}
interface MediaScan {
/** 按 path 字典序排序(哈希定义要求与遍历顺序无关) */
files: MediaManifestEntry[]
/** 被跳过的符号链接(含 junction)与其他非常规文件的相对路径 */
skipped: string[]
}
/** 递归枚举媒体目录:跳过符号链接(记录之),文件逐个流式哈希 */
async function scanMediaDir(root: string): Promise<MediaScan> {
const files: MediaManifestEntry[] = []
const skipped: string[] = []
const walk = async (abs: string, rel: string): Promise<void> => {
for (const entry of await readdir(abs, { withFileTypes: true })) {
const childRel = rel === '' ? entry.name : `${rel}/${entry.name}`
const childAbs = join(abs, entry.name)
if (entry.isSymbolicLink()) {
skipped.push(childRel)
continue
}
if (entry.isDirectory()) {
await walk(childAbs, childRel)
continue
}
if (!entry.isFile()) {
skipped.push(childRel)
continue
}
const { sha256, bytes } = await hashFile(childAbs)
files.push({ path: childRel, bytes, sha256 })
}
}
await walk(root, '')
files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
return { files, skipped }
}
/** 并发写同一 content-addressed 目标的兜底:rename 撞已有副本视为命中(幂等),删临时件 */
async function renameIntoPlace(tmp: string, dest: string): Promise<boolean> {
if (existsSync(dest)) {
await rm(tmp, { recursive: true, force: true })
return false
}
await mkdir(dirname(dest), { recursive: true })
try {
await rename(tmp, dest)
return true
} catch (err) {
if (existsSync(dest)) {
await rm(tmp, { recursive: true, force: true })
return false
}
throw err
}
}
interface Ingested {
kind: SourceSnapshotKind
sourcePath: string
sha256: string
byteSize: number
storagePath: string
copied: boolean
metadata: Record<string, unknown>
durationMs: number
}
/** 文件输入(wxr/sitemap):temp 副本上单遍 copy+hash → 按 sha 定名落位(已存在则命中复用) */
async function ingestFile(
kind: 'wxr' | 'sitemap',
src: string,
storageRoot: string,
): Promise<Ingested> {
const t0 = performance.now()
const tmpDir = join(storageRoot, 'snapshots', kind, '.tmp')
await mkdir(tmpDir, { recursive: true })
const tmp = join(tmpDir, `${randomUUID()}.part`)
let sha256: string
let bytes: number
try {
;({ sha256, bytes } = await copyAndHash(src, tmp))
} catch (err) {
throw new SourceRegisterError(
'INPUT_READ_FAILED',
`${kind} 输入读取失败:${src}(${err instanceof Error ? err.message : String(err)})`,
)
}
const rel = `snapshots/${kind}/${sha256.slice(0, 2)}/${sha256}`
let copied: boolean
try {
copied = await renameIntoPlace(tmp, join(storageRoot, rel))
} catch (err) {
throw new SourceRegisterError(
'COPY_FAILED',
`${kind} 快照副本落位失败:${join(storageRoot, rel)}(${err instanceof Error ? err.message : String(err)})`,
)
}
return {
kind,
sourcePath: src,
sha256,
byteSize: bytes,
storagePath: rel,
copied,
metadata: { sourceName: basename(src) },
durationMs: performance.now() - t0,
}
}
/** 媒体目录输入:扫描(每文件哈希)→ 清单哈希 → 缺副本时整树复制到 <sha>/ 目录 */
async function ingestMediaDir(srcDir: string, storageRoot: string): Promise<Ingested> {
const t0 = performance.now()
let scan: MediaScan
try {
scan = await scanMediaDir(srcDir)
} catch (err) {
throw new SourceRegisterError(
'INPUT_READ_FAILED',
`media 目录扫描失败:${srcDir}(${err instanceof Error ? err.message : String(err)})`,
)
}
// 清单哈希输入只含 path+sha256(bytes 只进 metadata,不进哈希——定义钉死)
const manifestHashInput = scan.files.map((f) => ({ path: f.path, sha256: f.sha256 }))
const sha256 = createHash('sha256').update(canonicalJson(manifestHashInput)).digest('hex')
const totalBytes = scan.files.reduce((sum, f) => sum + f.bytes, 0)
const rel = `snapshots/media_bundle/${sha256.slice(0, 2)}/${sha256}`
const dest = join(storageRoot, rel)
let copied = false
if (!existsSync(dest)) {
const tmpRoot = join(storageRoot, 'snapshots', 'media_bundle', '.tmp')
await mkdir(tmpRoot, { recursive: true })
const tmp = join(tmpRoot, randomUUID())
try {
for (const f of scan.files) {
const destFile = join(tmp, f.path)
await mkdir(dirname(destFile), { recursive: true })
await pipeline(
createReadStream(join(srcDir, f.path), { highWaterMark: CHUNK_SIZE }),
createWriteStream(destFile, { highWaterMark: CHUNK_SIZE }),
)
}
copied = await renameIntoPlace(tmp, dest)
} catch (err) {
await rm(tmp, { recursive: true, force: true })
throw new SourceRegisterError(
'COPY_FAILED',
`media_bundle 快照副本落位失败:${dest}(${err instanceof Error ? err.message : String(err)})`,
)
}
}
return {
kind: 'media_bundle',
sourcePath: srcDir,
sha256,
byteSize: totalBytes,
storagePath: rel,
copied,
metadata: {
fileCount: scan.files.length,
totalBytes,
manifest: scan.files,
skippedSymlinks: scan.skipped,
},
durationMs: performance.now() - t0,
}
}
// ---- 路径安全 ----
/** realpath(不存在则回退词法 resolve)——白名单根可能尚未创建(如首次运行无 storage/incoming) */
async function realpathOrResolve(p: string): Promise<string> {
try {
return await realpath(p)
} catch {
return resolve(p)
}
}
/** 路径是否位于某根目录内(双方均 realpath 后比较;首段 .. 或绝对化即越界) */
function isUnderRoot(target: string, root: string): boolean {
const rel = relative(root, target)
if (rel === '') return true
if (isAbsolute(rel)) return false
return rel.split(/[\\/]/)[0] !== '..'
}
async function validateInputPath(
inputPath: string,
expected: 'file' | 'directory',
allowedRoots: string[],
label: string,
): Promise<string> {
const abs = resolve(inputPath)
let stat
try {
stat = await lstat(abs)
} catch (err) {
const code = (err as NodeJS.ErrnoException).code
if (code === 'ENOENT' || code === 'ENOTDIR') {
throw new SourceRegisterError('INPUT_NOT_FOUND', `${label} 输入不存在:${abs}`)
}
throw new SourceRegisterError(
'INPUT_READ_FAILED',
`${label} 输入无法访问(${code ?? 'unknown'}):${abs}`,
)
}
// 输入本身是符号链接:以 realpath 后的目标做白名单与类型判定(防白名单内链接指向外界)
let real: string
if (stat.isSymbolicLink()) {
try {
real = await realpath(abs)
} catch {
// 断链(目标不存在):realpath 抛 ENOENT——归入清晰的"输入不存在",而非裸系统错误
throw new SourceRegisterError('INPUT_NOT_FOUND', `${label} 输入不存在(符号链接目标缺失):${abs}`)
}
} else {
real = abs
}
let allowed = false
for (const root of allowedRoots) {
const realRoot = await realpathOrResolve(resolve(root))
if (isUnderRoot(real, realRoot)) {
allowed = true
break
}
}
if (!allowed) {
throw new SourceRegisterError(
'PATH_NOT_ALLOWED',
`${label} 输入路径越界:${abs}(不在允许目录:${allowedRoots.join(' ; ')})`,
)
}
const realStat = await lstat(real)
if (expected === 'file' && !realStat.isFile()) {
throw new SourceRegisterError('INPUT_NOT_FILE', `${label} 输入不是文件:${abs}`)
}
if (expected === 'directory' && !realStat.isDirectory()) {
throw new SourceRegisterError('INPUT_NOT_DIR', `${label} 输入不是目录:${abs}`)
}
return real
}
// ---- DB 写入(drizzle) ----
/** postgres unique violation(23505)——drizzle 会包成 DrizzleQueryError,真 code 在 cause */
function isUniqueViolation(err: unknown): boolean {
const e = err as { code?: unknown; cause?: { code?: unknown } }
const code = e?.cause?.code ?? e?.code
return code === '23505'
}
/** site upsert:select-then-insert;插入撞 key 唯一约束(并发竞态)则重查一次 */
async function ensureSite(db: SourceDb, siteKey: string, defaults: RegisterSiteDefaults): Promise<string> {
const found = await db.select({ id: sites.id }).from(sites).where(eq(sites.key, siteKey)).limit(1)
if (found.length > 0) return found[0]!.id
try {
const inserted = await db
.insert(sites)
.values({
key: siteKey,
name: defaults.name ?? siteKey,
defaultLocale: defaults.defaultLocale ?? 'en',
productionOrigin: defaults.productionOrigin ?? `https://${siteKey}.example`,
})
.returning({ id: sites.id })
return inserted[0]!.id
} catch (err) {
if (isUniqueViolation(err)) {
const again = await db.select({ id: sites.id }).from(sites).where(eq(sites.key, siteKey)).limit(1)
if (again.length > 0) return again[0]!.id
}
throw err
}
}
/**
* connection 判重:键 (site_id, name)。**表无此库级唯一约束**(Plan 1 schema 决定),
* 判重在应用层 select-then-insert——本服务是 source_connections 唯一写入方;出现第二个
* 写入方(REST 通道等)时必须先补库级唯一索引再谈并发。
* configuration 保持空对象:Connection 只描述来源类别(ADR-004),文件事实属于 snapshots。
*/
async function ensureConnection(
db: SourceDb,
siteId: string,
name: string,
): Promise<{ id: string; created: boolean }> {
const found = await db
.select({ id: sourceConnections.id })
.from(sourceConnections)
.where(and(eq(sourceConnections.siteId, siteId), eq(sourceConnections.name, name)))
.limit(1)
if (found.length > 0) return { id: found[0]!.id, created: false }
try {
const inserted = await db
.insert(sourceConnections)
.values({ siteId, kind: 'wxr', name, configuration: {} })
.returning({ id: sourceConnections.id })
return { id: inserted[0]!.id, created: true }
} catch (err) {
if (isUniqueViolation(err)) {
// (site_id,name) 无库级唯一——此分支只可能是站点级联竞态;重查兜底
const again = await db
.select({ id: sourceConnections.id })
.from(sourceConnections)
.where(and(eq(sourceConnections.siteId, siteId), eq(sourceConnections.name, name)))
.limit(1)
if (again.length > 0) return { id: again[0]!.id, created: false }
}
throw err
}
}
/** snapshot 幂等落库:UNIQUE(site_id, kind, sha256) 命中复用;不可变表只 INSERT */
async function ensureSnapshot(
db: SourceDb,
args: { siteId: string; connectionId: string; ingested: Ingested; capturedAt: Date },
): Promise<{ id: string; reused: boolean }> {
const { siteId, connectionId, ingested, capturedAt } = args
const found = await db
.select({ id: sourceSnapshots.id })
.from(sourceSnapshots)
.where(
and(
eq(sourceSnapshots.siteId, siteId),
eq(sourceSnapshots.kind, ingested.kind),
eq(sourceSnapshots.sha256, ingested.sha256),
),
)
.limit(1)
if (found.length > 0) return { id: found[0]!.id, reused: true }
try {
const inserted = await db
.insert(sourceSnapshots)
.values({
siteId,
sourceConnectionId: connectionId,
kind: ingested.kind,
storagePath: ingested.storagePath,
sha256: ingested.sha256,
byteSize: ingested.byteSize,
adapterVersion: SNAPSHOT_ADAPTER_VERSION[ingested.kind],
capturedAt,
metadata: ingested.metadata,
})
.returning({ id: sourceSnapshots.id })
return { id: inserted[0]!.id, reused: false }
} catch (err) {
if (isUniqueViolation(err)) {
const again = await db
.select({ id: sourceSnapshots.id })
.from(sourceSnapshots)
.where(
and(
eq(sourceSnapshots.siteId, siteId),
eq(sourceSnapshots.kind, ingested.kind),
eq(sourceSnapshots.sha256, ingested.sha256),
),
)
.limit(1)
if (again.length > 0) return { id: again[0]!.id, reused: true }
}
throw err
}
}
// ---- runId 与报告 ----
/** runId = 紧凑时间戳 + 6 位随机(同秒多次运行可区分) */
function makeRunId(): string {
const ts = new Date()
const compact = `${ts.getFullYear()}${String(ts.getMonth() + 1).padStart(2, '0')}${String(
ts.getDate(),
).padStart(2, '0')}T${String(ts.getHours()).padStart(2, '0')}${String(ts.getMinutes()).padStart(
2,
'0',
)}${String(ts.getSeconds()).padStart(2, '0')}`
return `${compact}-${Math.random().toString(36).slice(2, 8)}`
}
// ---- 主入口 ----
/**
* 登记来源输入:校验(全部输入先过白名单/存在性,任何失败零落库零复制)→
* 逐输入摄取(copy+hash)→ site/connection/snapshot 落库 → 报告写盘。
* 返回 RegisteredSnapshot[] 供 CLI 摘要与调用方断言。
*/
export async function registerSource(db: SourceDb, input: RegisterInput): Promise<RegisterResult> {
const startedAt = performance.now()
const allowedRoots = input.allowedRoots ?? [join(REPO_ROOT, 'storage', 'incoming')]
const storageRoot = input.storageRoot ?? join(REPO_ROOT, 'storage')
const reportRoot = input.reportRoot ?? join(REPO_ROOT, 'artifacts', 'register')
// 阶段 0:全部输入先校验(fail fast,不留下半程副本)
let wxrReal: string | undefined
let sitemapReal: string | undefined
let mediaReal: string | undefined
if (input.wxrPath !== undefined) wxrReal = await validateInputPath(input.wxrPath, 'file', allowedRoots, 'wxr')
if (input.sitemapPath !== undefined)
sitemapReal = await validateInputPath(input.sitemapPath, 'file', allowedRoots, 'sitemap')
if (input.mediaDir !== undefined)
mediaReal = await validateInputPath(input.mediaDir, 'directory', allowedRoots, 'media')
if (wxrReal === undefined && sitemapReal === undefined && mediaReal === undefined) {
throw new SourceRegisterError('NO_INPUTS', '未提供任何输入(wxrPath / sitemapPath / mediaDir 至少其一)')
}
// 阶段 1:摄取(hash + 副本落位)——顺序固定 wxr → sitemap → media(报告输入清单稳定序)
const ingested: Ingested[] = []
if (wxrReal !== undefined) ingested.push(await ingestFile('wxr', wxrReal, storageRoot))
if (sitemapReal !== undefined) ingested.push(await ingestFile('sitemap', sitemapReal, storageRoot))
if (mediaReal !== undefined) ingested.push(await ingestMediaDir(mediaReal, storageRoot))
// 阶段 2:DB 写入
const capturedAt = new Date()
const siteId = await ensureSite(db, input.siteKey, input.siteDefaults ?? {})
const connection = await ensureConnection(db, siteId, input.connectionName ?? 'default')
const snapshots: RegisteredSnapshot[] = []
for (const item of ingested) {
const { id, reused } = await ensureSnapshot(db, { siteId, connectionId: connection.id, ingested: item, capturedAt })
snapshots.push({
kind: item.kind,
id,
sha256: item.sha256,
byteSize: item.byteSize,
storagePath: item.storagePath,
sourcePath: item.sourcePath,
reused,
copied: item.copied,
durationMs: item.durationMs,
metadata: item.metadata,
})
}
// 阶段 3:报告(阶段 2 无 import_jobs,报告按 runId 归档——<reportRoot>/<runId>/reports/)
const runId = makeRunId()
const durationMs = performance.now() - startedAt
const report = {
runId,
generatedAt: new Date().toISOString(),
siteKey: input.siteKey,
siteId,
connectionId: connection.id,
connectionCreated: connection.created,
inputs: snapshots.map((s) => ({
kind: s.kind,
sourcePath: s.sourcePath,
sha256: s.sha256,
byteSize: s.byteSize,
storagePath: s.storagePath,
reused: s.reused,
copied: s.copied,
durationMs: s.durationMs,
metadata: s.metadata,
})),
totals: {
snapshots: snapshots.length,
reused: snapshots.filter((s) => s.reused).length,
byteSize: snapshots.reduce((sum, s) => sum + s.byteSize, 0),
},
durationMs,
}
const reportDir = join(reportRoot, runId, 'reports')
await mkdir(reportDir, { recursive: true })
const reportPath = join(reportDir, 'register.json')
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8')
return {
runId,
siteKey: input.siteKey,
siteId,
connectionId: connection.id,
connectionCreated: connection.created,
snapshots,
reportPath,
durationMs,
}
}
@@ -0,0 +1,25 @@
// source 域 Nest 模块(Plan 2 Task 7):注册 SourceRegisterService 供 standalone context
// 使用(NestFactory.createApplicationContext(SourceModule)——本切片 API 不监听端口,ADR-002)。
// 注意:本文件是仓库中第一个会执行 Nest 装饰器的模块文件——import 'reflect-metadata' 置于
// 顶部(装饰器依赖 Reflect.defineMetadata;CLI/测试直接 import register.service.ts 纯函数
// 文件,不经本模块,故无需提前加载 reflect-metadata)。
import 'reflect-metadata'
import { Injectable, Module } from '@nestjs/common'
import { registerSource, type RegisterInput, type RegisterResult, type SourceDb } from './register.service.js'
/**
* registerSource 的 Nest 注入壳:逻辑全在 register.service.ts 的独立函数里,
* 本类只提供 DI 语境下的调用面(standalone context / 后续 REST 通道复用同一实现)。
*/
@Injectable()
export class SourceRegisterService {
register(db: SourceDb, input: RegisterInput): Promise<RegisterResult> {
return registerSource(db, input)
}
}
@Module({
providers: [SourceRegisterService],
exports: [SourceRegisterService],
})
export class SourceModule {}
+6 -1
View File
@@ -3,7 +3,12 @@
"compilerOptions": {
"outDir": "dist",
"experimentalDecorators": true,
"emitDecoratorMetadata": true
"emitDecoratorMetadata": true,
"baseUrl": ".",
"paths": {
"@sf/contracts": ["../../packages/contracts/src/index.ts"],
"@sf/adapter-wp": ["../../packages/adapter-wordpress/src/index.ts"]
}
},
"include": ["src", "test"]
}
+6 -1
View File
@@ -12,12 +12,17 @@
"lint": "eslint .",
"db:up": "docker compose -f infra/docker-compose.yml up -d --wait",
"db:migrate": "pnpm --filter @sf/api db:migrate",
"db:test": "bash scripts/migrate-test.sh"
"db:test": "bash scripts/migrate-test.sh",
"sf": "tsx scripts/sf.ts"
},
"dependencies": {
"commander": "^14.0.0"
},
"devDependencies": {
"@eslint/js": "^9.39.0",
"@types/node": "^22",
"eslint": "^9.39.0",
"tsx": "^4.19.0",
"typescript": "^5.6.0",
"typescript-eslint": "^8.46.4",
"vitest": "^2.1.0"
@@ -42,6 +42,12 @@
// "allowIframe: false"的等价物就是 allowedTags 缺席(本库无此独立开关)。
// - style 属性与 <style> 标签均拒绝(ADR-009:内联样式是风险面,清洗后进产出的
// 内容不得携带 CSS;未来若放开须走 allowedStyles 显式子集评审)。
// 挂载 launder 的环境模块声明(本包自带的类型垫片):declare module 只能经 d.ts
// 进入程序、无法用 import 表达,故用 triple-slash 引用使其随本文件进任何编译程序
// ——含 apps/api 与根程序经 tsconfig paths 直连本包源码的场景(Plan 2 Task 7 起;
// 否则垫片只在本包自身 tsconfig 的 include 下生效)。
// eslint-disable-next-line @typescript-eslint/triple-slash-reference -- 垫片无 import 形态
/// <reference path="../types/launder.d.ts" />
// - class/id 透传:WP 对齐类(alignleft 等)与锚点依赖之;class 值是惰性文本,
// 安全边界在标签/属性白名单,不 class 名单(阶段 3 主题自行取舍是否消费)。
// - **追踪像素策略(本切片钉死)**:外链 <img>(含 1x1 追踪像素形态)一律保留——
+16
View File
@@ -7,6 +7,10 @@ settings:
importers:
.:
dependencies:
commander:
specifier: ^14.0.0
version: 14.0.3
devDependencies:
'@eslint/js':
specifier: ^9.39.0
@@ -17,6 +21,9 @@ importers:
eslint:
specifier: ^9.39.0
version: 9.39.5
tsx:
specifier: ^4.19.0
version: 4.23.15
typescript:
specifier: ^5.6.0
version: 5.9.3
@@ -35,6 +42,9 @@ importers:
'@nestjs/core':
specifier: ^10.4.22
version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2)
'@sf/adapter-wp':
specifier: workspace:*
version: link:../../packages/adapter-wordpress
drizzle-orm:
specifier: ^0.45.4
version: 0.45.4(postgres@3.4.9)
@@ -1095,6 +1105,10 @@ packages:
color-name@1.1.4:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
commander@14.0.3:
resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}
engines: {node: '>=20'}
concat-map@0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
@@ -2489,6 +2503,8 @@ snapshots:
color-name@1.1.4: {}
commander@14.0.3: {}
concat-map@0.0.1: {}
consola@2.15.3: {}
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env node
// sf CLI(Plan 2 Task 7):commander 两命令——
// - wxr:probe:只读诊断(任意可读路径;Task 8 直读 cms 用此通道),出 JSON/HTML 报告不入库;
// - source:register:调 registerSource 登记 content-addressed 快照(路径白名单校验在服务内)。
// 执行机制(Plan 2 评审钉死):根 devDep tsx 直吃 TS 源码(含 apps/api 的 Nest 源码与
// packages 的装饰器/TS 源文件),@sf/* 经根/包 tsconfig paths 直连 src——无需预构建 dist。
// 注意:CLI 只 import register.service.ts(零装饰器纯函数文件),不拉起 Nest 模块——
// 反射元数据依赖由此避免;source.module.ts 留给 standalone context。
import { createReadStream } from 'node:fs'
import { mkdir, writeFile } from 'node:fs/promises'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { Command } from 'commander'
import { renderReportHtml, runWxrAdapterReport, type WxrAdapterReport } from '@sf/adapter-wp'
import { getDb } from '../apps/api/src/db/client.js'
import { registerSource, SourceRegisterError, type RegisterInput } from '../apps/api/src/modules/source/register.service.js'
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
/** runId = 紧凑时间戳 + 6 位随机(与 register.service 同形;报告目录按 runId 归档) */
function makeRunId(): string {
const ts = new Date()
const p = (n: number) => String(n).padStart(2, '0')
const compact = `${ts.getFullYear()}${p(ts.getMonth() + 1)}${p(ts.getDate())}T${p(ts.getHours())}${p(ts.getMinutes())}${p(ts.getSeconds())}`
return `${compact}-${Math.random().toString(36).slice(2, 8)}`
}
/** 探测报告摘要(stdout):版本块之外的计数/清洗/耗时 + 致命错误 */
function printProbeSummary(report: WxrAdapterReport): void {
console.log(` rows=${report.totals.rows} terms=${report.totals.terms} unsupported=${report.totals.unsupported} failed=${report.totals.failed}`)
console.log(` changedRows=${report.sanitize.changedRows} removed-tag=${report.sanitize.byDiffType['removed-tag']} removed-attr=${report.sanitize.byDiffType['removed-attr']}`)
console.log(` timing: parse=${report.timing.parseMs.toFixed(1)}ms sanitize=${report.timing.sanitizeMs.toFixed(1)}ms total=${report.timing.totalMs.toFixed(1)}ms`)
for (const type of Object.keys(report.counts).sort()) {
const c = report.counts[type]!
console.log(` ${type}: created=${c.created} skipped=${c.skipped} failed=${c.failed} unsupported=${c.unsupported}`)
}
if (report.fatalError !== null) {
console.error(` fatalError: ${report.fatalError}`)
}
}
const program = new Command()
program
.name('sf')
.description('SiteForge CLI(阶段 2:来源输入登记与 WXR 探测)')
.version('0.1.0')
program
.command('wxr:probe')
.description('只读诊断:WXR 全链解析(parse→map→sanitize)并出报告;不入库、不写业务存储,接受任意可读路径')
.requiredOption('--wxr <path>', 'WXR 文件路径(任意可读路径;损坏文件出报告不崩溃)')
.option('--out <dir>', '报告输出根目录', join(REPO_ROOT, 'artifacts', 'probe'))
.action(async (opts: { wxr: string; out: string }) => {
const wxrPath = resolve(opts.wxr)
const runId = makeRunId()
// Task 6 fatalError 契约:IO/损坏文件错误被 runWxrAdapterReport 收敛进报告——
// 这里永远能产出报告文件(createReadStream 的 ENOENT 同样走 fatalError 分支)
const report = await runWxrAdapterReport(createReadStream(wxrPath))
const dir = join(resolve(opts.out), runId)
await mkdir(dir, { recursive: true })
await writeFile(join(dir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`, 'utf8')
await writeFile(join(dir, 'report.html'), `${renderReportHtml(report)}\n`, 'utf8')
console.log(`[wxr:probe] runId=${runId}`)
console.log(` input: ${wxrPath}`)
printProbeSummary(report)
console.log(` report: ${join(dir, 'report.json')}`)
if (report.fatalError !== null) process.exitCode = 1
})
program
.command('source:register')
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录,可多选)')
.requiredOption('--site <key>', '站点 key(不存在则自动创建)')
.option('--wxr <path>', 'WXR 文件路径')
.option('--sitemap <path>', 'sitemap 文件路径')
.option('--media <dir>', '媒体目录路径(已解包目录,递归登记)')
.option('--allow <dir...>', '额外输入白名单根目录(可多个;默认含 storage/incoming 与当前目录)', [])
.option('--name <name>', '新建站点时的显示名(缺省 = site key)')
.option('--locale <locale>', '新建站点时的默认 locale(缺省 en)')
.option('--origin <origin>', '新建站点时的生产域名(缺省 https://<key>.example)')
.action(async (opts: {
site: string
wxr?: string
sitemap?: string
media?: string
allow: string[]
name?: string
locale?: string
origin?: string
}) => {
if (opts.wxr === undefined && opts.sitemap === undefined && opts.media === undefined) {
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media 之一')
}
// CLI 是可信操作者通道:白名单 = storage/incoming(常规通道)+ 操作者所在 cwd(显式
// 于此运行的动作本身即放行依据)+ --allow 追加根。服务内的白名单校验(PATH_NOT_ALLOWED)
// 防的是未来程序化/REST 通道的越界读取。
const input: RegisterInput = {
siteKey: opts.site,
allowedRoots: [join(REPO_ROOT, 'storage', 'incoming'), process.cwd(), ...opts.allow.map((d) => resolve(d))],
storageRoot: join(REPO_ROOT, 'storage'),
reportRoot: join(REPO_ROOT, 'artifacts', 'register'),
}
if (opts.wxr !== undefined) input.wxrPath = opts.wxr
if (opts.sitemap !== undefined) input.sitemapPath = opts.sitemap
if (opts.media !== undefined) input.mediaDir = opts.media
if (opts.name !== undefined || opts.locale !== undefined || opts.origin !== undefined) {
const defaults: NonNullable<RegisterInput['siteDefaults']> = {}
if (opts.name !== undefined) defaults.name = opts.name
if (opts.locale !== undefined) defaults.defaultLocale = opts.locale
if (opts.origin !== undefined) defaults.productionOrigin = opts.origin
input.siteDefaults = defaults
}
// getDb 走 DATABASE_URL_TEST ?? DATABASE_URL 回退链——对临时库冒烟只需 env 覆盖,无需改代码
const db = getDb()
try {
const result = await registerSource(db, input)
console.log(`[source:register] runId=${result.runId} site=${result.siteKey} (${result.siteId})`)
console.log(
` connection=${result.connectionId}${result.connectionCreated ? ' (created)' : ' (existing)'}`,
)
for (const s of result.snapshots) {
console.log(
` ${s.kind}: sha256=${s.sha256.slice(0, 12)}… bytes=${s.byteSize} ${s.reused ? 'snapshot复用' : 'snapshot新建'}${s.copied ? '' : ' 副本已存在'} (${s.durationMs.toFixed(1)}ms)`,
)
}
console.log(` report: ${result.reportPath} (total ${result.durationMs.toFixed(1)}ms)`)
} finally {
await (db as { $client: { end: () => Promise<void> } }).$client.end()
}
})
// parseAsync:action 内抛错走此 catch(commander 默认会 process.exit(1) 并打印堆栈,
// 这里收敛为单行错误信息 + exitCode——SourceRegisterError 打 code,其余原样呈现)
program.parseAsync(process.argv).catch((err: unknown) => {
if (err instanceof SourceRegisterError) {
console.error(`[sf] ${err.code}: ${err.message}`)
} else {
console.error(`[sf] ${err instanceof Error ? err.message : String(err)}`)
}
process.exitCode = 1
})
+390
View File
@@ -0,0 +1,390 @@
// Plan 2 Task 7 失败测试(先于实现):source:register 服务——content-addressed 快照登记。
// 覆盖六组契约:
// 1. 三类输入(wxr/sitemap/media 目录)一次登记 → site 按 key 自动建、source_connections
// 恰 1 行((site_id,name) 应用层判重)、source_snapshots 恰 3 行且 sha256 与源文件一致;
// 2. media_bundle 清单哈希定义钉死:排序相对路径('/' 分隔)+ 每文件 sha256 → canonical
// JSON(键排序、无空白)→ 再 sha256;metadata 含完整清单(path/bytes/sha256)、不含 mtime;
// 3. 重复 register 同输入 → 复用既有 snapshot(UNIQUE(site,kind,sha256),不新增行),
// 媒体目录重跑哈希一致(Windows 分隔符归一化 + mtime 不参与);
// 4. 副本布局:storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(整树),
// 副本内容与原输入一致(后续任务只读副本);
// 5. 路径安全:白名单外 / 越界遍历 / ENOENT / 类型不符均清晰报错且零落库;
// 6. 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机)。
// 运行于 globalSetup 临时库(ADR-015);storage/报告根用 os.tmpdir 临时目录,finally 清理。
// 注:符号链接跳过用目录 junction 验证(Windows 免特权;非 Windows 退化为普通符号链接);
// 创建失败(极端沙箱)时该断言静默放行——junction 创建不是本测试的被测契约。
import { createHash } from 'node:crypto'
import { existsSync } from 'node:fs'
import { mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import {
registerSource,
SourceRegisterError,
type RegisterInput,
type RegisteredSnapshot,
} from '../../apps/api/src/modules/source/register.service.js'
import { rand, withImmutableCleanup, withTestDb, type Db } from './helpers.js'
const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url))
const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml')
const SITEMAP_FIXTURE = resolve(FIXTURES_ROOT, 'sitemap', 'mini-sitemap.xml')
const THIS_FILE = fileURLToPath(import.meta.url)
const sha256 = (data: Buffer | string): string => createHash('sha256').update(data).digest('hex')
/** 测试侧独立实现的 canonical JSON(键排序、无空白)——不 import 实现,钉死定义 */
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`
if (value !== null && typeof value === 'object') {
const record = value as Record<string, unknown>
const body = Object.keys(record)
.sort()
.map((k) => `${JSON.stringify(k)}:${canonicalJson(record[k])}`)
.join(',')
return `{${body}}`
}
return JSON.stringify(value)
}
interface MediaFile {
path: string
bytes: number
content: Buffer
}
interface Ctx {
work: string
storageRoot: string
reportRoot: string
mediaDir: string
mediaFiles: MediaFile[]
allowedRoots: string[]
}
/** 独立媒体树:a.png + sub/b.png(内容固定,期望哈希可在测试内独立重算) */
async function makeMediaTree(work: string): Promise<{ mediaDir: string; mediaFiles: MediaFile[] }> {
const mediaDir = join(work, 'media')
await mkdir(join(mediaDir, 'sub'), { recursive: true })
const a = Buffer.from('sf-media-a-content')
const b = Buffer.from('sf-media-sub-b-content')
await writeFile(join(mediaDir, 'a.png'), a)
await writeFile(join(mediaDir, 'sub', 'b.png'), b)
return {
mediaDir,
mediaFiles: [
{ path: 'a.png', bytes: a.length, content: a },
{ path: 'sub/b.png', bytes: b.length, content: b },
],
}
}
async function withCtx(fn: (ctx: Ctx) => Promise<void>): Promise<void> {
const work = await mkdtemp(join(tmpdir(), 'sf-register-'))
const { mediaDir, mediaFiles } = await makeMediaTree(work)
const ctx: Ctx = {
work,
storageRoot: join(work, 'storage'),
reportRoot: join(work, 'artifacts', 'register'),
mediaDir,
mediaFiles,
allowedRoots: [FIXTURES_ROOT, work],
}
try {
await fn(ctx)
} finally {
await rm(work, { recursive: true, force: true })
}
}
function makeInput(ctx: Ctx, siteKey: string): RegisterInput {
return {
siteKey,
wxrPath: WXR_FIXTURE,
sitemapPath: SITEMAP_FIXTURE,
mediaDir: ctx.mediaDir,
allowedRoots: ctx.allowedRoots,
storageRoot: ctx.storageRoot,
reportRoot: ctx.reportRoot,
}
}
/** 服务创建的 site 行清场:快照不可变走触发器开关,FK 反序删 connection → site */
async function cleanupSiteRows(db: Db, siteId: string): Promise<void> {
await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`)
await db.execute(`delete from source_connections where site_id = '${siteId}'`)
await db.execute(`delete from sites where id = '${siteId}'`)
}
describe('source:register 服务', () => {
it('三类输入一次登记:site 自动创建、connection 1 行、snapshot 3 行(sha256 与源一致)', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
// site 按 key 自动 upsert(不存在则建,默认列落库)
const siteRows = (await db.execute(`select id, key, default_locale from sites where key = '${key}'`)) as any
expect(siteRows).toHaveLength(1)
expect(siteRows[0].id).toBe(r.siteId)
expect(siteRows[0].default_locale).toBe('en')
// connection 恰 1 行((site_id,name) 应用层判重)
const connRows = (await db.execute(
`select id, kind, name from source_connections where site_id = '${r.siteId}'`,
)) as any
expect(connRows).toHaveLength(1)
expect(connRows[0].kind).toBe('wxr')
expect(connRows[0].name).toBe('default')
expect(connRows[0].id).toBe(r.connectionId)
// snapshot 恰 3 行,kind 各异,sha256 与源文件一致,storage_path 指向存在的副本
const snapRows = (await db.execute(
`select kind, sha256, byte_size, storage_path from source_snapshots where site_id = '${r.siteId}'`,
)) as { kind: string; sha256: string; storage_path: string }[]
expect(snapRows).toHaveLength(3)
const byKind = new Map(snapRows.map((row) => [row.kind, row]))
expect(byKind.get('wxr')?.sha256).toBe(sha256(await readFile(WXR_FIXTURE)))
expect(byKind.get('sitemap')?.sha256).toBe(sha256(await readFile(SITEMAP_FIXTURE)))
for (const row of snapRows) {
expect(existsSync(join(ctx.storageRoot, row.storage_path))).toBe(true)
}
expect(r.snapshots).toHaveLength(3)
expect(new Set(r.snapshots.map((s: RegisteredSnapshot) => s.kind))).toEqual(
new Set(['wxr', 'sitemap', 'media_bundle']),
)
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('media_bundle 清单哈希 = 排序路径+每文件 sha256 的 canonical JSON 再哈希;metadata 无 mtime', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
const bundle = r.snapshots.find((s: RegisteredSnapshot) => s.kind === 'media_bundle')!
// 测试侧独立重算(排序 + canonical JSON 键序无空白)
const entries = ctx.mediaFiles
.map((f) => ({ path: f.path, sha256: sha256(f.content) }))
.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
expect(bundle.sha256).toBe(sha256(canonicalJson(entries)))
// metadata 含完整清单(path/bytes/sha256),不含 mtime
const manifest = (bundle.metadata as { manifest: { path: string; bytes: number; sha256: string }[] }).manifest
expect([...manifest].sort((a, b) => (a.path < b.path ? -1 : 1))).toEqual([
{ path: 'a.png', bytes: ctx.mediaFiles[0]!.bytes, sha256: sha256(ctx.mediaFiles[0]!.content) },
{ path: 'sub/b.png', bytes: ctx.mediaFiles[1]!.bytes, sha256: sha256(ctx.mediaFiles[1]!.content) },
])
expect(JSON.stringify(bundle.metadata)).not.toContain('mtime')
// byteSize = 清单文件字节数之和
expect(bundle.byteSize).toBe(ctx.mediaFiles.reduce((sum, f) => sum + f.bytes, 0))
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('重复 register 同输入:复用既有 snapshot/connection/site,不新增行;mtime 变化不影响哈希', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r1 = await registerSource(db, makeInput(ctx, key))
try {
// 播动 mtime(证明清单哈希不含 mtime)后重跑
const t = new Date(Date.now() + 60_000)
await utimes(join(ctx.mediaDir, 'a.png'), t, t)
await utimes(WXR_FIXTURE, t, t).catch(() => undefined) // 夹具可能只读;媒体树已足够证明
const r2 = await registerSource(db, makeInput(ctx, key))
expect(r2.siteId).toBe(r1.siteId)
expect(r2.connectionId).toBe(r1.connectionId)
expect(r2.connectionCreated).toBe(false)
expect(r2.snapshots).toHaveLength(3)
for (const s2 of r2.snapshots) {
const s1 = r1.snapshots.find((s: RegisteredSnapshot) => s.kind === s2.kind)!
expect(s2.id).toBe(s1.id)
expect(s2.sha256).toBe(s1.sha256)
expect(s2.reused).toBe(true)
}
const counts = (await db.execute(
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns,
(select count(*) from sites where key = '${key}') as sites`,
)) as any
expect(counts[0]).toEqual({ snaps: '3', conns: '1', sites: '1' })
} finally {
await cleanupSiteRows(db, r1.siteId)
}
})
})
})
it('副本布局 content-addressed:文件落 <kind>/<sha2>/<sha>,媒体整树落 <sha2>/<sha>/,内容一致', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
for (const s of r.snapshots) {
const sha2 = s.sha256.slice(0, 2)
expect(s.storagePath).toBe(`snapshots/${s.kind}/${sha2}/${s.sha256}`)
const dest = join(ctx.storageRoot, s.storagePath)
if (s.kind === 'media_bundle') {
// 整树:目录内相对结构保留,内容逐文件一致
expect(existsSync(join(dest, 'a.png'))).toBe(true)
expect(existsSync(join(dest, 'sub', 'b.png'))).toBe(true)
expect(await readFile(join(dest, 'a.png'))).toEqual(ctx.mediaFiles[0]!.content)
expect(await readFile(join(dest, 'sub', 'b.png'))).toEqual(ctx.mediaFiles[1]!.content)
} else {
const source = s.kind === 'wxr' ? WXR_FIXTURE : SITEMAP_FIXTURE
expect(await readFile(dest)).toEqual(await readFile(source))
}
// 副本路径 ≠ 源路径(后续任务只读副本)
expect(dest).not.toBe(s.sourcePath)
}
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('路径安全:白名单外/越界遍历/ENOENT/类型不符/无输入 均清晰报错且零落库', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const base: RegisterInput = {
siteKey: key,
allowedRoots: [ctx.work], // 注意:本用例故意不含 FIXTURES_ROOT
storageRoot: ctx.storageRoot,
reportRoot: ctx.reportRoot,
}
const cases: { label: string; input: RegisterInput; code: string }[] = [
{
label: '白名单外的绝对路径(本测试文件自身)',
input: { ...base, wxrPath: THIS_FILE },
code: 'PATH_NOT_ALLOWED',
},
{
label: '越界遍历 ../(白名单只含 incoming/,目标逃到 work 根)',
input: {
...base,
wxrPath: join(ctx.work, 'incoming', '..', 'escape.txt'),
allowedRoots: [join(ctx.work, 'incoming')],
},
code: 'PATH_NOT_ALLOWED',
},
{
label: 'ENOENT',
input: { ...base, wxrPath: join(ctx.work, 'not-exist.xml') },
code: 'INPUT_NOT_FOUND',
},
{
label: 'wxr 输入是目录',
input: { ...base, wxrPath: ctx.mediaDir },
code: 'INPUT_NOT_FILE',
},
{
label: 'media 输入是文件',
// 白名单放行 FIXTURES_ROOT 以穿过白名单检查,命中类型检查
input: { ...base, mediaDir: WXR_FIXTURE, allowedRoots: [ctx.work, FIXTURES_ROOT] },
code: 'INPUT_NOT_DIR',
},
{
label: '无任何输入',
input: { ...base },
code: 'NO_INPUTS',
},
]
// 越界遍历用例的配套文件:work 根下放 escape.txt,incoming/ 只是白名单根
await writeFile(join(ctx.work, 'escape.txt'), 'escape')
await mkdir(join(ctx.work, 'incoming'), { recursive: true })
for (const c of cases) {
const err: SourceRegisterError = await registerSource(db, c.input).then(
() => {
throw new Error(`用例「${c.label}」不应成功`)
},
(e: SourceRegisterError) => e,
)
expect(err, c.label).toBeInstanceOf(SourceRegisterError)
expect(err.code, c.label).toBe(c.code)
expect(err.message, c.label).toMatch(/越界|不存在|白名单|目录|文件|输入|路径/)
}
// 零落库:site 从未创建
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
expect(sites).toHaveLength(0)
})
})
})
it('符号链接(junction)在媒体枚举中被跳过并记录,清单不受其内容影响', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
// 白名单外目录放一个文件,junction 指过去——若被跟随既破坏白名单也污染清单
const outside = join(ctx.work, 'outside-pkg')
await mkdir(outside, { recursive: true })
await writeFile(join(outside, 'rogue.png'), 'rogue')
try {
await symlink(outside, join(ctx.mediaDir, 'linked-pkg'), 'junction')
} catch {
return // Windows 无开发者模式且非管理员:junction 创建失败,本用例静默放行
}
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
const bundle = r.snapshots.find((s: RegisteredSnapshot) => s.kind === 'media_bundle')!
const meta = bundle.metadata as { manifest: { path: string }[]; skippedSymlinks: string[] }
expect(meta.manifest.map((m) => m.path).sort()).toEqual(['a.png', 'sub/b.png'])
expect(meta.skippedSymlinks).toContain('linked-pkg')
// 副本树不含 junction 内容
expect(existsSync(join(ctx.storageRoot, bundle.storagePath, 'linked-pkg'))).toBe(false)
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
it('报告落盘 <reportRoot>/<runId>/reports/register.json:输入清单/哈希/大小/耗时', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const r = await registerSource(db, makeInput(ctx, key))
try {
expect(r.reportPath).toBe(join(ctx.reportRoot, r.runId, 'reports', 'register.json'))
expect(existsSync(r.reportPath)).toBe(true)
const report = JSON.parse(await readFile(r.reportPath, 'utf8')) as {
runId: string
siteKey: string
siteId: string
inputs: { kind: string; sourcePath: string; sha256: string; byteSize: number; durationMs: number }[]
totals: { snapshots: number; reused: number; byteSize: number }
durationMs: number
}
expect(report.runId).toBe(r.runId)
expect(report.siteKey).toBe(key)
expect(report.siteId).toBe(r.siteId)
expect(report.inputs).toHaveLength(3)
const wxrInput = report.inputs.find((i) => i.kind === 'wxr')!
expect(wxrInput.sha256).toBe(sha256(await readFile(WXR_FIXTURE)))
expect(wxrInput.byteSize).toBe((await readFile(WXR_FIXTURE)).length)
expect(wxrInput.durationMs).toBeGreaterThanOrEqual(0)
expect(report.totals.snapshots).toBe(3)
expect(report.totals.reused).toBe(0)
expect(report.durationMs).toBeGreaterThanOrEqual(0)
// runId 形态:时间戳 + 随机(两次运行不同)
expect(r.runId).toMatch(/^\d{8}T\d{6}-[a-z0-9]+$/)
} finally {
await cleanupSiteRows(db, r.siteId)
}
})
})
})
})
+1 -1
View File
@@ -9,5 +9,5 @@
"@sf/adapter-wp": ["packages/adapter-wordpress/src/index.ts"]
}
},
"include": ["vitest.config.ts", "tests/**/*.ts"]
"include": ["vitest.config.ts", "tests/**/*.ts", "scripts/**/*.ts"]
}