Files
cms-website/scripts/sf.ts
T
浅念 586d715455 feat(source): 输入登记服务与 sf CLI(content-addressed 快照)
- registerSource(db, input):wxr/sitemap/媒体目录三类输入登记为 content-addressed
  快照——流式 copy+sha256 单遍双算(64KB 块),副本落
  storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(媒体整树),
  后续任务只读副本;UNIQUE(site,kind,sha256) 幂等(重复登记复用既有行)
- media_bundle 哈希钉死:排序相对路径('/' 归一)+ 每文件 sha256 → canonical
  JSON(键排序无空白)→ 再 sha256;不含 mtime、不依赖遍历顺序;完整清单
  (path/bytes/sha256)入 metadata jsonb;符号链接(含 junction)跳过并记录
- site 按 key select-then-insert upsert(23505 竞态重查);connection 判重键
  (site_id,name) 为应用层 select-then-insert(表无库级唯一,注释注明)
- 路径安全:输入 realpath 后须在 allowedRoots 白名单内(缺省 storage/incoming),
  ENOENT/越界/类型不符抛带 code 的 SourceRegisterError;全部输入先校验后摄取,
  任何失败零落库零复制
- 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机;
  输入清单/哈希/大小/耗时)
- scripts/sf.ts(commander):wxr:probe 只读诊断(任意可读路径,IO/损坏文件出
  报告不崩——Task 6 fatalError 契约)出 JSON/HTML 报告;source:register 调服务
  出摘要。CLI 机制钉死:根 devDep tsx + 根 script sf + commander 根依赖,TS 源码
  直连(@sf/* 经 tsconfig paths,不经 dist)
- source.module.ts:Nest 注册 SourceRegisterService(standalone context 用);
  register.service.ts 刻意零装饰器(CLI/测试直 import 不拉 reflect-metadata)
- launder 类型垫片改经 triple-slash 引用随 html.ts 进任意编译程序(apps/api 与
  根程序直连适配器源码的新场景);apps/api 增 @sf/adapter-wp workspace 依赖与
  paths 映射(register.service 消费 ADAPTER_VERSION 单源)
- 测试 +7(tests/integration/source-register.test.ts,临时库 + tmpdir storage/
  报告根 finally 清理):三输入登记/清单哈希定义对拍/重复登记幂等/mtime 排除/
  副本布局与内容一致/路径安全六分支/junction 跳过/报告契约;全套 260 绿

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-21 12:31:15 +08:00

142 lines
7.7 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
// sf CLI(Plan 2 Task 7):commander 两命令——
// - wxr:probe:只读诊断(任意可读路径;Task 8 直读 cms 用此通道),出 JSON/HTML 报告不入库;
// - source:register:调 registerSource 登记 content-addressed 快照(路径白名单校验在服务内)。
// 执行机制(Plan 2 评审钉死):根 devDep tsx 直吃 TS 源码(含 apps/api 的 Nest 源码与
// packages 的装饰器/TS 源文件),@sf/* 经根/包 tsconfig paths 直连 src——无需预构建 dist。
// 注意:CLI 只 import register.service.ts(零装饰器纯函数文件),不拉起 Nest 模块——
// 反射元数据依赖由此避免;source.module.ts 留给 standalone context。
import { createReadStream } from 'node:fs'
import { mkdir, writeFile } from 'node:fs/promises'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { Command } from 'commander'
import { renderReportHtml, runWxrAdapterReport, type WxrAdapterReport } from '@sf/adapter-wp'
import { getDb } from '../apps/api/src/db/client.js'
import { registerSource, SourceRegisterError, type RegisterInput } from '../apps/api/src/modules/source/register.service.js'
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
/** runId = 紧凑时间戳 + 6 位随机(与 register.service 同形;报告目录按 runId 归档) */
function makeRunId(): string {
const ts = new Date()
const p = (n: number) => String(n).padStart(2, '0')
const compact = `${ts.getFullYear()}${p(ts.getMonth() + 1)}${p(ts.getDate())}T${p(ts.getHours())}${p(ts.getMinutes())}${p(ts.getSeconds())}`
return `${compact}-${Math.random().toString(36).slice(2, 8)}`
}
/** 探测报告摘要(stdout):版本块之外的计数/清洗/耗时 + 致命错误 */
function printProbeSummary(report: WxrAdapterReport): void {
console.log(` rows=${report.totals.rows} terms=${report.totals.terms} unsupported=${report.totals.unsupported} failed=${report.totals.failed}`)
console.log(` changedRows=${report.sanitize.changedRows} removed-tag=${report.sanitize.byDiffType['removed-tag']} removed-attr=${report.sanitize.byDiffType['removed-attr']}`)
console.log(` timing: parse=${report.timing.parseMs.toFixed(1)}ms sanitize=${report.timing.sanitizeMs.toFixed(1)}ms total=${report.timing.totalMs.toFixed(1)}ms`)
for (const type of Object.keys(report.counts).sort()) {
const c = report.counts[type]!
console.log(` ${type}: created=${c.created} skipped=${c.skipped} failed=${c.failed} unsupported=${c.unsupported}`)
}
if (report.fatalError !== null) {
console.error(` fatalError: ${report.fatalError}`)
}
}
const program = new Command()
program
.name('sf')
.description('SiteForge CLI(阶段 2:来源输入登记与 WXR 探测)')
.version('0.1.0')
program
.command('wxr:probe')
.description('只读诊断:WXR 全链解析(parse→map→sanitize)并出报告;不入库、不写业务存储,接受任意可读路径')
.requiredOption('--wxr <path>', 'WXR 文件路径(任意可读路径;损坏文件出报告不崩溃)')
.option('--out <dir>', '报告输出根目录', join(REPO_ROOT, 'artifacts', 'probe'))
.action(async (opts: { wxr: string; out: string }) => {
const wxrPath = resolve(opts.wxr)
const runId = makeRunId()
// Task 6 fatalError 契约:IO/损坏文件错误被 runWxrAdapterReport 收敛进报告——
// 这里永远能产出报告文件(createReadStream 的 ENOENT 同样走 fatalError 分支)
const report = await runWxrAdapterReport(createReadStream(wxrPath))
const dir = join(resolve(opts.out), runId)
await mkdir(dir, { recursive: true })
await writeFile(join(dir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`, 'utf8')
await writeFile(join(dir, 'report.html'), `${renderReportHtml(report)}\n`, 'utf8')
console.log(`[wxr:probe] runId=${runId}`)
console.log(` input: ${wxrPath}`)
printProbeSummary(report)
console.log(` report: ${join(dir, 'report.json')}`)
if (report.fatalError !== null) process.exitCode = 1
})
program
.command('source:register')
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录,可多选)')
.requiredOption('--site <key>', '站点 key(不存在则自动创建)')
.option('--wxr <path>', 'WXR 文件路径')
.option('--sitemap <path>', 'sitemap 文件路径')
.option('--media <dir>', '媒体目录路径(已解包目录,递归登记)')
.option('--allow <dir...>', '额外输入白名单根目录(可多个;默认含 storage/incoming 与当前目录)', [])
.option('--name <name>', '新建站点时的显示名(缺省 = site key)')
.option('--locale <locale>', '新建站点时的默认 locale(缺省 en)')
.option('--origin <origin>', '新建站点时的生产域名(缺省 https://<key>.example)')
.action(async (opts: {
site: string
wxr?: string
sitemap?: string
media?: string
allow: string[]
name?: string
locale?: string
origin?: string
}) => {
if (opts.wxr === undefined && opts.sitemap === undefined && opts.media === undefined) {
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media 之一')
}
// CLI 是可信操作者通道:白名单 = storage/incoming(常规通道)+ 操作者所在 cwd(显式
// 于此运行的动作本身即放行依据)+ --allow 追加根。服务内的白名单校验(PATH_NOT_ALLOWED)
// 防的是未来程序化/REST 通道的越界读取。
const input: RegisterInput = {
siteKey: opts.site,
allowedRoots: [join(REPO_ROOT, 'storage', 'incoming'), process.cwd(), ...opts.allow.map((d) => resolve(d))],
storageRoot: join(REPO_ROOT, 'storage'),
reportRoot: join(REPO_ROOT, 'artifacts', 'register'),
}
if (opts.wxr !== undefined) input.wxrPath = opts.wxr
if (opts.sitemap !== undefined) input.sitemapPath = opts.sitemap
if (opts.media !== undefined) input.mediaDir = opts.media
if (opts.name !== undefined || opts.locale !== undefined || opts.origin !== undefined) {
const defaults: NonNullable<RegisterInput['siteDefaults']> = {}
if (opts.name !== undefined) defaults.name = opts.name
if (opts.locale !== undefined) defaults.defaultLocale = opts.locale
if (opts.origin !== undefined) defaults.productionOrigin = opts.origin
input.siteDefaults = defaults
}
// getDb 走 DATABASE_URL_TEST ?? DATABASE_URL 回退链——对临时库冒烟只需 env 覆盖,无需改代码
const db = getDb()
try {
const result = await registerSource(db, input)
console.log(`[source:register] runId=${result.runId} site=${result.siteKey} (${result.siteId})`)
console.log(
` connection=${result.connectionId}${result.connectionCreated ? ' (created)' : ' (existing)'}`,
)
for (const s of result.snapshots) {
console.log(
` ${s.kind}: sha256=${s.sha256.slice(0, 12)}… bytes=${s.byteSize} ${s.reused ? 'snapshot复用' : 'snapshot新建'}${s.copied ? '' : ' 副本已存在'} (${s.durationMs.toFixed(1)}ms)`,
)
}
console.log(` report: ${result.reportPath} (total ${result.durationMs.toFixed(1)}ms)`)
} finally {
await (db as { $client: { end: () => Promise<void> } }).$client.end()
}
})
// parseAsync:action 内抛错走此 catch(commander 默认会 process.exit(1) 并打印堆栈,
// 这里收敛为单行错误信息 + exitCode——SourceRegisterError 打 code,其余原样呈现)
program.parseAsync(process.argv).catch((err: unknown) => {
if (err instanceof SourceRegisterError) {
console.error(`[sf] ${err.code}: ${err.message}`)
} else {
console.error(`[sf] ${err instanceof Error ? err.message : String(err)}`)
}
process.exitCode = 1
})