浅念
21ef37232d
真实 hesupipe WXR（5,490,399B，SHA-256 与 Plan 1 基线一致）跑 wxr:probe 全链： 计数 18 项对照全过（rows 1047 / terms 451 / unsupported 75 细分一致 / failed 0 / item 总数 1122）；heapUsed max 62.2MiB < 100MiB（setInterval(100ms) 双 max 口径， 全程 167ms 仅 2 样本如实注明）；parseMs 含流 IO 等待口径注明。 nicename 锚点核对：在表 taxonomy 706/706 全命中（nicename≡slug，违例 0）； category/post_tag 共 807 引用因 wp:term 表缺该 taxonomy 不可解析——阶段 3 需按 (domain, nicename) 自建缺失 term，不能假设 term 表完备。 eslint.config.js：scripts/*.mjs 显式 node 全局（no-undef 误报）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
358 lines
15 KiB
JavaScript
358 lines
15 KiB
JavaScript
#!/usr/bin/env node
|
||
// 真实 WXR 冒烟（Plan 2 · Task 8，阶段 2 验收）：
|
||
// 对 hesupipe 真实导出（只读）跑 wxr:probe 同链路（runWxrAdapterReport），计数对照
|
||
// Plan 1 基线 docs/evidence/real-wxr-baseline.md，内存按计划钉死口径采样——
|
||
// setInterval(100ms) 采 process.memoryUsage() 的 heapUsed/rss 双 max。
|
||
//
|
||
// 运行方式（仓库根）：
|
||
// pnpm exec tsx scripts/smoke-real-wxr.mjs [--wxr <path>] [--out <dir>]
|
||
// 与 scripts/sf.ts 同机制：@sf/adapter-wp 经根 tsconfig paths 直连 TS 源码（需 tsx，
|
||
// 纯 node 无法解析 workspace TS 包——证据文档须记实际执行命令原文）。
|
||
//
|
||
// 断言语义（任一不一致 → verdict FAIL + exitCode 1，如实报告不凑解释）：
|
||
// - totals 与基线逐项一致（rows/terms/unsupported/failed 与 item 总数）；
|
||
// - unsupported 按 sourceType（`wp:<post_type>`）细分计数一致；
|
||
// - heapUsed max < 100MB（流式验收线）；fatalError 必须为 null。
|
||
// nicename 锚点核对（Task 4 冻结点）为证据采集而非 pass/fail 断言：不一致逐条列出，
|
||
// 供阶段 3 taxonomyRefs 映射策略决策。
|
||
//
|
||
// 计时口径（Task 6 移交，报告须注明）：parseMs 是 parser.next() await 窗口累计，
|
||
// 含流式 IO 等待（磁盘读/backpressure），非纯 CPU 解析耗时；totalMs 覆盖整个消费循环。
|
||
//
|
||
// 输出：stdout 纯 JSON（机器可消费）；人类可读进度走 stderr。产物落
|
||
// artifacts/probe/<runId>/{report.json,report.html,smoke.json}（与 wxr:probe CLI 同布局）。
|
||
|
||
import { createHash } from 'node:crypto'
|
||
import { createReadStream } from 'node:fs'
|
||
import { mkdir, stat, writeFile } from 'node:fs/promises'
|
||
import { dirname, join, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { parseWxrStream, renderReportHtml, runWxrAdapterReport } from '@sf/adapter-wp'
|
||
import { makeRunId } from '../apps/api/src/modules/source/register.service.js'
|
||
|
||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||
|
||
/** 冒烟对象（缺省）：hesupipe 真实导出，绝对只读 */
|
||
const DEFAULT_WXR = 'D:/project/cms/storage/wxr/fd04d6ba8a1316b6.xml'
|
||
|
||
/** 流式验收线：heapUsed max < 100MB（计划钉死） */
|
||
const HEAP_LIMIT_BYTES = 100 * 1024 * 1024
|
||
|
||
/** 内存采样周期（计划钉死口径：setInterval(100ms) 双 max） */
|
||
const SAMPLE_INTERVAL_MS = 100
|
||
|
||
/** nicename 锚点抽检条数：文档序前 N 个含 domain="category" 引用的 item */
|
||
const NICENAME_SAMPLE_ITEMS = 10
|
||
|
||
/** Plan 1 基线（docs/evidence/real-wxr-baseline.md，对照用冻结值） */
|
||
const BASELINE = {
|
||
fileBytes: 5_490_399,
|
||
sha256: 'f9bc5d4497a1792af37abff78e0096268bda975cb367a956da42bcf9552b9f62',
|
||
itemsTotal: 1122,
|
||
rowsTotal: 1047, // 855 attachment + 129 post + 48 product + 15 page
|
||
terms: 451,
|
||
unsupportedTotal: 75,
|
||
failedTotal: 0,
|
||
/** 产 row 的四类（created+skipped+failed 之和对照） */
|
||
rowsByType: { 'wp:attachment': 855, 'wp:post': 129, 'wp:product': 48, 'wp:page': 15 },
|
||
/** unsupported 细分（`wp:<post_type>` 归桶） */
|
||
unsupportedByType: {
|
||
'wp:nav_menu_item': 68,
|
||
'wp:wpcf7_contact_form': 3,
|
||
'wp:elementor_library': 2,
|
||
'wp:wp_navigation': 2,
|
||
},
|
||
}
|
||
|
||
/** 期望出现的 counts 键全集 = rowsByType ∪ unsupportedByType（多一个少一个都算不一致） */
|
||
const EXPECTED_COUNTS_KEYS = [...Object.keys(BASELINE.rowsByType), ...Object.keys(BASELINE.unsupportedByType)].sort()
|
||
|
||
/** 断言登记（顺序即证据表行序） */
|
||
function makeChecks() {
|
||
const checks = []
|
||
const check = (name, expected, actual, pass) => checks.push({ name, expected, actual, pass })
|
||
const checkEq = (name, expected, actual) => check(name, expected, actual, Object.is(expected, actual))
|
||
return { checks, check, checkEq }
|
||
}
|
||
|
||
/** 流式 sha256（64KB 块，与 register 服务的哈希口径同型） */
|
||
async function sha256File(path) {
|
||
const hash = createHash('sha256')
|
||
for await (const chunk of createReadStream(path, { highWaterMark: 64 * 1024 })) {
|
||
hash.update(chunk)
|
||
}
|
||
return hash.digest('hex')
|
||
}
|
||
|
||
/** --wxr <path> / --wxr=<path> / --out <dir> / --out=<dir>（缺省 DEFAULT_WXR 与 artifacts/probe） */
|
||
function parseArgs(argv) {
|
||
const out = { wxr: DEFAULT_WXR, outDir: join(REPO_ROOT, 'artifacts', 'probe') }
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const arg = argv[i]
|
||
if (arg === '--wxr' || arg === '--out') {
|
||
const value = argv[++i]
|
||
if (value === undefined) throw new Error(`${arg} 缺少值参数`)
|
||
if (arg === '--wxr') out.wxr = value
|
||
else out.outDir = value
|
||
} else if (arg !== undefined && arg.startsWith('--wxr=')) out.wxr = arg.slice(6)
|
||
else if (arg !== undefined && arg.startsWith('--out=')) out.outDir = arg.slice(6)
|
||
else throw new Error(`未知参数：${arg}（支持 --wxr <path> 与 --out <dir>）`)
|
||
}
|
||
return out
|
||
}
|
||
|
||
/** 单域 unresolved nicename 明细固化上限（防 post_tag 这类大域使 smoke.json 失去上界） */
|
||
const UNRESOLVED_LIST_CAP = 50
|
||
|
||
/**
|
||
* nicename 锚点核对（Task 4 冻结点：sourceTermId 以 item 内 @nicename 为锚）。
|
||
* 单遍流式（与 probe 独立的第二遍读，采样器已停）：全量收集 wp:term 的
|
||
* (taxonomy → Set(slug)) 与 item 引用 (domain → Map(nicename → 计数))、
|
||
* 文档序前 10 个含 category 域引用的 item 明细——流结束后统一比对
|
||
* （不依赖 wp:term 与 item 的文档先后顺序）。
|
||
*
|
||
* 证据分两层（阶段 3 taxonomyRefs 映射策略的决策依据）：
|
||
* - 任务要求的 10 条抽检：item 的 category 域 nicename ∈ wp:term(category) 的 slug 集；
|
||
* - 全量分层核对（每个 domain）：区分「taxonomy 未出现在 wp:term 表」（导出缺该
|
||
* taxonomy，锚点无从对拍）与「taxonomy 在表但 nicename 无对应 slug」（真正的
|
||
* nicename≠slug 不一致）。前者是本 WXR 的实际形态（category/post_tag 未导出
|
||
* term 表），如实记录而非记为锚点失败。
|
||
*/
|
||
async function nicenameAudit(wxrPath) {
|
||
const termSlugsByTaxonomy = new Map()
|
||
const domainCounts = new Map()
|
||
const nicenameCountsByDomain = new Map()
|
||
const sampledItems = []
|
||
let itemsSeen = 0
|
||
let itemsWithCategoryRef = 0
|
||
|
||
for await (const ev of parseWxrStream(createReadStream(wxrPath))) {
|
||
if (ev.kind === 'term') {
|
||
const { taxonomy, slug } = ev.term
|
||
let set = termSlugsByTaxonomy.get(taxonomy)
|
||
if (set === undefined) {
|
||
set = new Set()
|
||
termSlugsByTaxonomy.set(taxonomy, set)
|
||
}
|
||
set.add(slug)
|
||
continue
|
||
}
|
||
itemsSeen++
|
||
const categoryRefs = ev.item.categories.filter((c) => c.domain === 'category')
|
||
for (const ref of ev.item.categories) {
|
||
domainCounts.set(ref.domain, (domainCounts.get(ref.domain) ?? 0) + 1)
|
||
let byNicename = nicenameCountsByDomain.get(ref.domain)
|
||
if (byNicename === undefined) {
|
||
byNicename = new Map()
|
||
nicenameCountsByDomain.set(ref.domain, byNicename)
|
||
}
|
||
byNicename.set(ref.nicename, (byNicename.get(ref.nicename) ?? 0) + 1)
|
||
}
|
||
if (categoryRefs.length === 0) continue
|
||
itemsWithCategoryRef++
|
||
if (sampledItems.length < NICENAME_SAMPLE_ITEMS) {
|
||
sampledItems.push({
|
||
rawLocation: ev.item.rawLocation,
|
||
postId: ev.item.postId,
|
||
postType: ev.item.postType,
|
||
refs: categoryRefs.map((ref) => ({ nicename: ref.nicename, value: ref.value })),
|
||
})
|
||
}
|
||
}
|
||
|
||
const categorySlugs = termSlugsByTaxonomy.get('category') ?? new Set()
|
||
|
||
// 抽检 10 条：明细级比对（nicename ∈ category 域 wp:term 的 term_slug 集）
|
||
let sampleMatched = 0
|
||
let sampleTotal = 0
|
||
const sampleMismatches = []
|
||
for (const item of sampledItems) {
|
||
for (const ref of item.refs) {
|
||
sampleTotal++
|
||
const resolved = categorySlugs.has(ref.nicename)
|
||
if (resolved) sampleMatched++
|
||
else sampleMismatches.push({ rawLocation: item.rawLocation, postId: item.postId, ...ref })
|
||
}
|
||
}
|
||
|
||
// 全量分层核对（每个 domain 一行）：taxonomy 在表与否 × nicename 可解析与否
|
||
const perDomain = [...nicenameCountsByDomain.entries()]
|
||
.sort(([a], [b]) => a.localeCompare(b))
|
||
.map(([domain, byNicename]) => {
|
||
const slugs = termSlugsByTaxonomy.get(domain)
|
||
let refs = 0
|
||
let matched = 0
|
||
let distinctUnresolved = 0
|
||
const unresolved = []
|
||
for (const [nicename, count] of byNicename) {
|
||
refs += count
|
||
if (slugs !== undefined && slugs.has(nicename)) {
|
||
matched += count
|
||
} else {
|
||
distinctUnresolved++
|
||
if (unresolved.length < UNRESOLVED_LIST_CAP) unresolved.push({ nicename, refCount: count })
|
||
}
|
||
}
|
||
return {
|
||
domain,
|
||
refs,
|
||
matched,
|
||
rate: refs === 0 ? null : matched / refs,
|
||
taxonomyInTermTable: slugs !== undefined,
|
||
distinctUnresolved,
|
||
unresolvedCapped: unresolved.length < distinctUnresolved,
|
||
unresolved,
|
||
}
|
||
})
|
||
|
||
const conclusion = {
|
||
/** taxonomy 在 wp:term 表中的域的引用数（这些引用可严格对拍 nicename vs slug） */
|
||
refsAgainstExistingTaxonomy: 0,
|
||
matchedAgainstExistingTaxonomy: 0,
|
||
/** taxonomy 不在 wp:term 表的域的引用数（锚点无从对拍——导出缺表，非锚点不一致） */
|
||
refsOfAbsentTaxonomy: 0,
|
||
/** taxonomy 在表但 nicename 无对应 slug 的引用数（真正的 nicename≠slug 违例） */
|
||
nicenameEqualsSlugViolations: 0,
|
||
}
|
||
for (const row of perDomain) {
|
||
if (row.taxonomyInTermTable) {
|
||
conclusion.refsAgainstExistingTaxonomy += row.refs
|
||
conclusion.matchedAgainstExistingTaxonomy += row.matched
|
||
conclusion.nicenameEqualsSlugViolations += row.refs - row.matched
|
||
} else {
|
||
conclusion.refsOfAbsentTaxonomy += row.refs
|
||
}
|
||
}
|
||
|
||
return {
|
||
sampleRule: `文档序前 ${NICENAME_SAMPLE_ITEMS} 个含 domain="category" 引用的 item（共 ${itemsWithCategoryRef} 个 item 含 category 域引用，items 总数 ${itemsSeen}）`,
|
||
sampledItems: sampledItems.map((item) => ({
|
||
...item,
|
||
refs: item.refs.map((ref) => ({ ...ref, resolvedToTermSlug: categorySlugs.has(ref.nicename) })),
|
||
})),
|
||
sampleMatched,
|
||
sampleTotal,
|
||
sampleRate: sampleTotal === 0 ? null : sampleMatched / sampleTotal,
|
||
sampleMismatches,
|
||
perDomain,
|
||
conclusion,
|
||
domainCounts: Object.fromEntries([...domainCounts.entries()].sort(([a], [b]) => a.localeCompare(b))),
|
||
termTaxonomies: Object.fromEntries(
|
||
[...termSlugsByTaxonomy.entries()]
|
||
.sort(([a], [b]) => a.localeCompare(b))
|
||
.map(([taxonomy, set]) => [taxonomy, set.size]),
|
||
),
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
const args = parseArgs(process.argv.slice(2))
|
||
const wxrPath = resolve(args.wxr)
|
||
const { checks, check, checkEq } = makeChecks()
|
||
|
||
process.stderr.write(`[smoke] 输入：${wxrPath}\n`)
|
||
const fileStat = await stat(wxrPath)
|
||
const sha256 = await sha256File(wxrPath)
|
||
checkEq('input.bytes', BASELINE.fileBytes, fileStat.size)
|
||
checkEq('input.sha256', BASELINE.sha256, sha256)
|
||
|
||
// —— probe + 内存采样（计划钉死口径）——
|
||
process.stderr.write('[smoke] 运行 probe（parse→map→sanitize 全链）+ setInterval(100ms) 内存采样…\n')
|
||
let maxHeapUsed = 0
|
||
let maxRss = 0
|
||
let samples = 0
|
||
const sample = () => {
|
||
const mu = process.memoryUsage()
|
||
if (mu.heapUsed > maxHeapUsed) maxHeapUsed = mu.heapUsed
|
||
if (mu.rss > maxRss) maxRss = mu.rss
|
||
samples++
|
||
}
|
||
const sampler = setInterval(sample, SAMPLE_INTERVAL_MS)
|
||
const report = await runWxrAdapterReport(createReadStream(wxrPath))
|
||
clearInterval(sampler)
|
||
sample() // 终末补采一次，覆盖收尾段（报告对象组装等）
|
||
|
||
// —— 计数对照（基线逐项）——
|
||
const t = report.totals
|
||
checkEq('totals.rows', BASELINE.rowsTotal, t.rows)
|
||
checkEq('totals.terms', BASELINE.terms, t.terms)
|
||
checkEq('totals.unsupported', BASELINE.unsupportedTotal, t.unsupported)
|
||
checkEq('totals.failed', BASELINE.failedTotal, t.failed)
|
||
checkEq('items.total(rows+unsupported+failed)', BASELINE.itemsTotal, t.rows + t.unsupported + t.failed)
|
||
|
||
for (const [type, expected] of Object.entries(BASELINE.rowsByType)) {
|
||
const c = report.counts[type]
|
||
checkEq(`rows.${type}(created+skipped+failed)`, expected, c === undefined ? 0 : c.created + c.skipped + c.failed)
|
||
}
|
||
for (const [type, expected] of Object.entries(BASELINE.unsupportedByType)) {
|
||
const c = report.counts[type]
|
||
checkEq(`unsupported.${type}`, expected, c === undefined ? 0 : c.unsupported)
|
||
}
|
||
checkEq(
|
||
'counts.keys(无多余/缺失桶)',
|
||
EXPECTED_COUNTS_KEYS.join(','),
|
||
Object.keys(report.counts).sort().join(','),
|
||
)
|
||
checkEq('probe.fatalError', null, report.fatalError)
|
||
check('memory.heapUsedMax<100MB', `< ${HEAP_LIMIT_BYTES}`, maxHeapUsed, maxHeapUsed < HEAP_LIMIT_BYTES)
|
||
|
||
// —— nicename 锚点核对（证据采集，非 pass/fail 断言）——
|
||
process.stderr.write('[smoke] nicename 锚点核对（第二遍流式）…\n')
|
||
const nicename = await nicenameAudit(wxrPath)
|
||
|
||
const verdict = checks.every((c) => c.pass) ? 'PASS' : 'FAIL'
|
||
|
||
// —— 产物（与 wxr:probe CLI 同布局 + smoke.json）——
|
||
const runId = makeRunId()
|
||
const dir = join(resolve(args.outDir), runId)
|
||
await mkdir(dir, { recursive: true })
|
||
await writeFile(join(dir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`, 'utf8')
|
||
await writeFile(join(dir, 'report.html'), `${renderReportHtml(report)}\n`, 'utf8')
|
||
const smoke = {
|
||
runId,
|
||
generatedAt: new Date().toISOString(),
|
||
nodeVersion: process.version,
|
||
input: { path: wxrPath, bytes: fileStat.size, sha256 },
|
||
probe: {
|
||
totals: report.totals,
|
||
counts: report.counts,
|
||
sanitize: report.sanitize,
|
||
timing: report.timing,
|
||
rowErrors: { recorded: report.rowErrors.length, truncated: report.rowErrorsTruncated },
|
||
fatalError: report.fatalError,
|
||
},
|
||
memory: {
|
||
intervalMs: SAMPLE_INTERVAL_MS,
|
||
samples,
|
||
maxHeapUsedBytes: maxHeapUsed,
|
||
maxRssBytes: maxRss,
|
||
heapLimitBytes: HEAP_LIMIT_BYTES,
|
||
heapUnderLimit: maxHeapUsed < HEAP_LIMIT_BYTES,
|
||
},
|
||
nicenameAudit: nicename,
|
||
checks,
|
||
verdict,
|
||
artifacts: { dir, report: join(dir, 'report.json'), smoke: join(dir, 'smoke.json') },
|
||
}
|
||
await writeFile(join(dir, 'smoke.json'), `${JSON.stringify(smoke, null, 2)}\n`, 'utf8')
|
||
|
||
for (const c of checks) {
|
||
process.stderr.write(` [${c.pass ? 'ok' : 'FAIL'}] ${c.name}: expected=${JSON.stringify(c.expected)} actual=${JSON.stringify(c.actual)}\n`)
|
||
}
|
||
const nc = nicename.conclusion
|
||
process.stderr.write(
|
||
`[smoke] verdict=${verdict}（nicename 抽检 ${nicename.sampleMatched}/${nicename.sampleTotal}；` +
|
||
`在表 taxonomy 对拍 ${nc.matchedAgainstExistingTaxonomy}/${nc.refsAgainstExistingTaxonomy}，` +
|
||
`nicename≠slug 违例 ${nc.nicenameEqualsSlugViolations}，缺表 taxonomy 引用 ${nc.refsOfAbsentTaxonomy}）\n`,
|
||
)
|
||
process.stderr.write(`[smoke] 产物：${join(dir, 'smoke.json')}\n`)
|
||
|
||
process.stdout.write(`${JSON.stringify(smoke, null, 2)}\n`)
|
||
process.exitCode = verdict === 'PASS' ? 0 : 1
|
||
}
|
||
|
||
main().catch((err) => {
|
||
process.stderr.write(`[smoke] 未预期异常：${err instanceof Error ? err.stack : String(err)}\n`)
|
||
process.exitCode = 1
|
||
})
|