586d715455
- registerSource(db, input):wxr/sitemap/媒体目录三类输入登记为 content-addressed 快照——流式 copy+sha256 单遍双算(64KB 块),副本落 storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(媒体整树), 后续任务只读副本;UNIQUE(site,kind,sha256) 幂等(重复登记复用既有行) - media_bundle 哈希钉死:排序相对路径('/' 归一)+ 每文件 sha256 → canonical JSON(键排序无空白)→ 再 sha256;不含 mtime、不依赖遍历顺序;完整清单 (path/bytes/sha256)入 metadata jsonb;符号链接(含 junction)跳过并记录 - site 按 key select-then-insert upsert(23505 竞态重查);connection 判重键 (site_id,name) 为应用层 select-then-insert(表无库级唯一,注释注明) - 路径安全:输入 realpath 后须在 allowedRoots 白名单内(缺省 storage/incoming), ENOENT/越界/类型不符抛带 code 的 SourceRegisterError;全部输入先校验后摄取, 任何失败零落库零复制 - 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机; 输入清单/哈希/大小/耗时) - scripts/sf.ts(commander):wxr:probe 只读诊断(任意可读路径,IO/损坏文件出 报告不崩——Task 6 fatalError 契约)出 JSON/HTML 报告;source:register 调服务 出摘要。CLI 机制钉死:根 devDep tsx + 根 script sf + commander 根依赖,TS 源码 直连(@sf/* 经 tsconfig paths,不经 dist) - source.module.ts:Nest 注册 SourceRegisterService(standalone context 用); register.service.ts 刻意零装饰器(CLI/测试直 import 不拉 reflect-metadata) - launder 类型垫片改经 triple-slash 引用随 html.ts 进任意编译程序(apps/api 与 根程序直连适配器源码的新场景);apps/api 增 @sf/adapter-wp workspace 依赖与 paths 映射(register.service 消费 ADAPTER_VERSION 单源) - 测试 +7(tests/integration/source-register.test.ts,临时库 + tmpdir storage/ 报告根 finally 清理):三输入登记/清单哈希定义对拍/重复登记幂等/mtime 排除/ 副本布局与内容一致/路径安全六分支/junction 跳过/报告契约;全套 260 绿 Co-Authored-By: Claude Code <noreply@anthropic.com>
142 lines
7.7 KiB
JavaScript
142 lines
7.7 KiB
JavaScript
#!/usr/bin/env node
|
||
// sf CLI(Plan 2 Task 7):commander 两命令——
|
||
// - wxr:probe:只读诊断(任意可读路径;Task 8 直读 cms 用此通道),出 JSON/HTML 报告不入库;
|
||
// - source:register:调 registerSource 登记 content-addressed 快照(路径白名单校验在服务内)。
|
||
// 执行机制(Plan 2 评审钉死):根 devDep tsx 直吃 TS 源码(含 apps/api 的 Nest 源码与
|
||
// packages 的装饰器/TS 源文件),@sf/* 经根/包 tsconfig paths 直连 src——无需预构建 dist。
|
||
// 注意:CLI 只 import register.service.ts(零装饰器纯函数文件),不拉起 Nest 模块——
|
||
// 反射元数据依赖由此避免;source.module.ts 留给 standalone context。
|
||
import { createReadStream } from 'node:fs'
|
||
import { mkdir, writeFile } from 'node:fs/promises'
|
||
import { dirname, join, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { Command } from 'commander'
|
||
import { renderReportHtml, runWxrAdapterReport, type WxrAdapterReport } from '@sf/adapter-wp'
|
||
import { getDb } from '../apps/api/src/db/client.js'
|
||
import { registerSource, SourceRegisterError, type RegisterInput } from '../apps/api/src/modules/source/register.service.js'
|
||
|
||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||
|
||
/** runId = 紧凑时间戳 + 6 位随机(与 register.service 同形;报告目录按 runId 归档) */
|
||
function makeRunId(): string {
|
||
const ts = new Date()
|
||
const p = (n: number) => String(n).padStart(2, '0')
|
||
const compact = `${ts.getFullYear()}${p(ts.getMonth() + 1)}${p(ts.getDate())}T${p(ts.getHours())}${p(ts.getMinutes())}${p(ts.getSeconds())}`
|
||
return `${compact}-${Math.random().toString(36).slice(2, 8)}`
|
||
}
|
||
|
||
/** 探测报告摘要(stdout):版本块之外的计数/清洗/耗时 + 致命错误 */
|
||
function printProbeSummary(report: WxrAdapterReport): void {
|
||
console.log(` rows=${report.totals.rows} terms=${report.totals.terms} unsupported=${report.totals.unsupported} failed=${report.totals.failed}`)
|
||
console.log(` changedRows=${report.sanitize.changedRows} removed-tag=${report.sanitize.byDiffType['removed-tag']} removed-attr=${report.sanitize.byDiffType['removed-attr']}`)
|
||
console.log(` timing: parse=${report.timing.parseMs.toFixed(1)}ms sanitize=${report.timing.sanitizeMs.toFixed(1)}ms total=${report.timing.totalMs.toFixed(1)}ms`)
|
||
for (const type of Object.keys(report.counts).sort()) {
|
||
const c = report.counts[type]!
|
||
console.log(` ${type}: created=${c.created} skipped=${c.skipped} failed=${c.failed} unsupported=${c.unsupported}`)
|
||
}
|
||
if (report.fatalError !== null) {
|
||
console.error(` fatalError: ${report.fatalError}`)
|
||
}
|
||
}
|
||
|
||
const program = new Command()
|
||
program
|
||
.name('sf')
|
||
.description('SiteForge CLI(阶段 2:来源输入登记与 WXR 探测)')
|
||
.version('0.1.0')
|
||
|
||
program
|
||
.command('wxr:probe')
|
||
.description('只读诊断:WXR 全链解析(parse→map→sanitize)并出报告;不入库、不写业务存储,接受任意可读路径')
|
||
.requiredOption('--wxr <path>', 'WXR 文件路径(任意可读路径;损坏文件出报告不崩溃)')
|
||
.option('--out <dir>', '报告输出根目录', join(REPO_ROOT, 'artifacts', 'probe'))
|
||
.action(async (opts: { wxr: string; out: string }) => {
|
||
const wxrPath = resolve(opts.wxr)
|
||
const runId = makeRunId()
|
||
// Task 6 fatalError 契约:IO/损坏文件错误被 runWxrAdapterReport 收敛进报告——
|
||
// 这里永远能产出报告文件(createReadStream 的 ENOENT 同样走 fatalError 分支)
|
||
const report = await runWxrAdapterReport(createReadStream(wxrPath))
|
||
const dir = join(resolve(opts.out), runId)
|
||
await mkdir(dir, { recursive: true })
|
||
await writeFile(join(dir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`, 'utf8')
|
||
await writeFile(join(dir, 'report.html'), `${renderReportHtml(report)}\n`, 'utf8')
|
||
console.log(`[wxr:probe] runId=${runId}`)
|
||
console.log(` input: ${wxrPath}`)
|
||
printProbeSummary(report)
|
||
console.log(` report: ${join(dir, 'report.json')}`)
|
||
if (report.fatalError !== null) process.exitCode = 1
|
||
})
|
||
|
||
program
|
||
.command('source:register')
|
||
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录,可多选)')
|
||
.requiredOption('--site <key>', '站点 key(不存在则自动创建)')
|
||
.option('--wxr <path>', 'WXR 文件路径')
|
||
.option('--sitemap <path>', 'sitemap 文件路径')
|
||
.option('--media <dir>', '媒体目录路径(已解包目录,递归登记)')
|
||
.option('--allow <dir...>', '额外输入白名单根目录(可多个;默认含 storage/incoming 与当前目录)', [])
|
||
.option('--name <name>', '新建站点时的显示名(缺省 = site key)')
|
||
.option('--locale <locale>', '新建站点时的默认 locale(缺省 en)')
|
||
.option('--origin <origin>', '新建站点时的生产域名(缺省 https://<key>.example)')
|
||
.action(async (opts: {
|
||
site: string
|
||
wxr?: string
|
||
sitemap?: string
|
||
media?: string
|
||
allow: string[]
|
||
name?: string
|
||
locale?: string
|
||
origin?: string
|
||
}) => {
|
||
if (opts.wxr === undefined && opts.sitemap === undefined && opts.media === undefined) {
|
||
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media 之一')
|
||
}
|
||
// CLI 是可信操作者通道:白名单 = storage/incoming(常规通道)+ 操作者所在 cwd(显式
|
||
// 于此运行的动作本身即放行依据)+ --allow 追加根。服务内的白名单校验(PATH_NOT_ALLOWED)
|
||
// 防的是未来程序化/REST 通道的越界读取。
|
||
const input: RegisterInput = {
|
||
siteKey: opts.site,
|
||
allowedRoots: [join(REPO_ROOT, 'storage', 'incoming'), process.cwd(), ...opts.allow.map((d) => resolve(d))],
|
||
storageRoot: join(REPO_ROOT, 'storage'),
|
||
reportRoot: join(REPO_ROOT, 'artifacts', 'register'),
|
||
}
|
||
if (opts.wxr !== undefined) input.wxrPath = opts.wxr
|
||
if (opts.sitemap !== undefined) input.sitemapPath = opts.sitemap
|
||
if (opts.media !== undefined) input.mediaDir = opts.media
|
||
if (opts.name !== undefined || opts.locale !== undefined || opts.origin !== undefined) {
|
||
const defaults: NonNullable<RegisterInput['siteDefaults']> = {}
|
||
if (opts.name !== undefined) defaults.name = opts.name
|
||
if (opts.locale !== undefined) defaults.defaultLocale = opts.locale
|
||
if (opts.origin !== undefined) defaults.productionOrigin = opts.origin
|
||
input.siteDefaults = defaults
|
||
}
|
||
// getDb 走 DATABASE_URL_TEST ?? DATABASE_URL 回退链——对临时库冒烟只需 env 覆盖,无需改代码
|
||
const db = getDb()
|
||
try {
|
||
const result = await registerSource(db, input)
|
||
console.log(`[source:register] runId=${result.runId} site=${result.siteKey} (${result.siteId})`)
|
||
console.log(
|
||
` connection=${result.connectionId}${result.connectionCreated ? ' (created)' : ' (existing)'}`,
|
||
)
|
||
for (const s of result.snapshots) {
|
||
console.log(
|
||
` ${s.kind}: sha256=${s.sha256.slice(0, 12)}… bytes=${s.byteSize} ${s.reused ? 'snapshot复用' : 'snapshot新建'}${s.copied ? '' : ' 副本已存在'} (${s.durationMs.toFixed(1)}ms)`,
|
||
)
|
||
}
|
||
console.log(` report: ${result.reportPath} (total ${result.durationMs.toFixed(1)}ms)`)
|
||
} finally {
|
||
await (db as { $client: { end: () => Promise<void> } }).$client.end()
|
||
}
|
||
})
|
||
|
||
// parseAsync:action 内抛错走此 catch(commander 默认会 process.exit(1) 并打印堆栈,
|
||
// 这里收敛为单行错误信息 + exitCode——SourceRegisterError 打 code,其余原样呈现)
|
||
program.parseAsync(process.argv).catch((err: unknown) => {
|
||
if (err instanceof SourceRegisterError) {
|
||
console.error(`[sf] ${err.code}: ${err.message}`)
|
||
} else {
|
||
console.error(`[sf] ${err instanceof Error ? err.message : String(err)}`)
|
||
}
|
||
process.exitCode = 1
|
||
})
|