358 lines
15 KiB
JavaScript
358 lines
15 KiB
JavaScript
|
#!/usr/bin/env node
|
|||
|
|
// 真实 WXR 冒烟（Plan 2 · Task 8，阶段 2 验收）：
|
|||
|
|
// 对 hesupipe 真实导出（只读）跑 wxr:probe 同链路（runWxrAdapterReport），计数对照
|
|||
|
|
// Plan 1 基线 docs/evidence/real-wxr-baseline.md，内存按计划钉死口径采样——
|
|||
|
|
// setInterval(100ms) 采 process.memoryUsage() 的 heapUsed/rss 双 max。
|
|||
|
|
//
|
|||
|
|
// 运行方式（仓库根）：
|
|||
|
|
// pnpm exec tsx scripts/smoke-real-wxr.mjs [--wxr <path>] [--out <dir>]
|
|||
|
|
// 与 scripts/sf.ts 同机制：@sf/adapter-wp 经根 tsconfig paths 直连 TS 源码（需 tsx，
|
|||
|
|
// 纯 node 无法解析 workspace TS 包——证据文档须记实际执行命令原文）。
|
|||
|
|
//
|
|||
|
|
// 断言语义（任一不一致 → verdict FAIL + exitCode 1，如实报告不凑解释）：
|
|||
|
|
// - totals 与基线逐项一致（rows/terms/unsupported/failed 与 item 总数）；
|
|||
|
|
// - unsupported 按 sourceType（`wp:<post_type>`）细分计数一致；
|
|||
|
|
// - heapUsed max < 100MB（流式验收线）；fatalError 必须为 null。
|
|||
|
|
// nicename 锚点核对（Task 4 冻结点）为证据采集而非 pass/fail 断言：不一致逐条列出，
|
|||
|
|
// 供阶段 3 taxonomyRefs 映射策略决策。
|
|||
|
|
//
|
|||
|
|
// 计时口径（Task 6 移交，报告须注明）：parseMs 是 parser.next() await 窗口累计，
|
|||
|
|
// 含流式 IO 等待（磁盘读/backpressure），非纯 CPU 解析耗时；totalMs 覆盖整个消费循环。
|
|||
|
|
//
|
|||
|
|
// 输出：stdout 纯 JSON（机器可消费）；人类可读进度走 stderr。产物落
|
|||
|
|
// artifacts/probe/<runId>/{report.json,report.html,smoke.json}（与 wxr:probe CLI 同布局）。
|
|||
|
|
|
|||
|
|
import { createHash } from 'node:crypto'
|
|||
|
|
import { createReadStream } from 'node:fs'
|
|||
|
|
import { mkdir, stat, writeFile } from 'node:fs/promises'
|
|||
|
|
import { dirname, join, resolve } from 'node:path'
|
|||
|
|
import { fileURLToPath } from 'node:url'
|
|||
|
|
import { parseWxrStream, renderReportHtml, runWxrAdapterReport } from '@sf/adapter-wp'
|
|||
|
|
import { makeRunId } from '../apps/api/src/modules/source/register.service.js'
|
|||
|
|
|
|||
|
|
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|||
|
|
|
|||
|
|
/** 冒烟对象（缺省）：hesupipe 真实导出，绝对只读 */
|
|||
|
|
const DEFAULT_WXR = 'D:/project/cms/storage/wxr/fd04d6ba8a1316b6.xml'
|
|||
|
|
|
|||
|
|
/** 流式验收线：heapUsed max < 100MB（计划钉死） */
|
|||
|
|
const HEAP_LIMIT_BYTES = 100 * 1024 * 1024
|
|||
|
|
|
|||
|
|
/** 内存采样周期（计划钉死口径：setInterval(100ms) 双 max） */
|
|||
|
|
const SAMPLE_INTERVAL_MS = 100
|
|||
|
|
|
|||
|
|
/** nicename 锚点抽检条数：文档序前 N 个含 domain="category" 引用的 item */
|
|||
|
|
const NICENAME_SAMPLE_ITEMS = 10
|
|||
|
|
|
|||
|
|
/** Plan 1 基线（docs/evidence/real-wxr-baseline.md，对照用冻结值） */
|
|||
|
|
const BASELINE = {
|
|||
|
|
fileBytes: 5_490_399,
|
|||
|
|
sha256: 'f9bc5d4497a1792af37abff78e0096268bda975cb367a956da42bcf9552b9f62',
|
|||
|
|
itemsTotal: 1122,
|
|||
|
|
rowsTotal: 1047, // 855 attachment + 129 post + 48 product + 15 page
|
|||
|
|
terms: 451,
|
|||
|
|
unsupportedTotal: 75,
|
|||
|
|
failedTotal: 0,
|
|||
|
|
/** 产 row 的四类（created+skipped+failed 之和对照） */
|
|||
|
|
rowsByType: { 'wp:attachment': 855, 'wp:post': 129, 'wp:product': 48, 'wp:page': 15 },
|
|||
|
|
/** unsupported 细分（`wp:<post_type>` 归桶） */
|
|||
|
|
unsupportedByType: {
|
|||
|
|
'wp:nav_menu_item': 68,
|
|||
|
|
'wp:wpcf7_contact_form': 3,
|
|||
|
|
'wp:elementor_library': 2,
|
|||
|
|
'wp:wp_navigation': 2,
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 期望出现的 counts 键全集 = rowsByType ∪ unsupportedByType（多一个少一个都算不一致） */
|
|||
|
|
const EXPECTED_COUNTS_KEYS = [...Object.keys(BASELINE.rowsByType), ...Object.keys(BASELINE.unsupportedByType)].sort()
|
|||
|
|
|
|||
|
|
/** 断言登记（顺序即证据表行序） */
|
|||
|
|
function makeChecks() {
|
|||
|
|
const checks = []
|
|||
|
|
const check = (name, expected, actual, pass) => checks.push({ name, expected, actual, pass })
|
|||
|
|
const checkEq = (name, expected, actual) => check(name, expected, actual, Object.is(expected, actual))
|
|||
|
|
return { checks, check, checkEq }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 流式 sha256（64KB 块，与 register 服务的哈希口径同型） */
|
|||
|
|
async function sha256File(path) {
|
|||
|
|
const hash = createHash('sha256')
|
|||
|
|
for await (const chunk of createReadStream(path, { highWaterMark: 64 * 1024 })) {
|
|||
|
|
hash.update(chunk)
|
|||
|
|
}
|
|||
|
|
return hash.digest('hex')
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** --wxr <path> / --wxr=<path> / --out <dir> / --out=<dir>（缺省 DEFAULT_WXR 与 artifacts/probe） */
|
|||
|
|
function parseArgs(argv) {
|
|||
|
|
const out = { wxr: DEFAULT_WXR, outDir: join(REPO_ROOT, 'artifacts', 'probe') }
|
|||
|
|
for (let i = 0; i < argv.length; i++) {
|
|||
|
|
const arg = argv[i]
|
|||
|
|
if (arg === '--wxr' || arg === '--out') {
|
|||
|
|
const value = argv[++i]
|
|||
|
|
if (value === undefined) throw new Error(`${arg} 缺少值参数`)
|
|||
|
|
if (arg === '--wxr') out.wxr = value
|
|||
|
|
else out.outDir = value
|
|||
|
|
} else if (arg !== undefined && arg.startsWith('--wxr=')) out.wxr = arg.slice(6)
|
|||
|
|
else if (arg !== undefined && arg.startsWith('--out=')) out.outDir = arg.slice(6)
|
|||
|
|
else throw new Error(`未知参数：${arg}（支持 --wxr <path> 与 --out <dir>）`)
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 单域 unresolved nicename 明细固化上限（防 post_tag 这类大域使 smoke.json 失去上界） */
|
|||
|
|
const UNRESOLVED_LIST_CAP = 50
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* nicename 锚点核对（Task 4 冻结点：sourceTermId 以 item 内 @nicename 为锚）。
|
|||
|
|
* 单遍流式（与 probe 独立的第二遍读，采样器已停）：全量收集 wp:term 的
|
|||
|
|
* (taxonomy → Set(slug)) 与 item 引用 (domain → Map(nicename → 计数))、
|
|||
|
|
* 文档序前 10 个含 category 域引用的 item 明细——流结束后统一比对
|
|||
|
|
* （不依赖 wp:term 与 item 的文档先后顺序）。
|
|||
|
|
*
|
|||
|
|
* 证据分两层（阶段 3 taxonomyRefs 映射策略的决策依据）：
|
|||
|
|
* - 任务要求的 10 条抽检：item 的 category 域 nicename ∈ wp:term(category) 的 slug 集；
|
|||
|
|
* - 全量分层核对（每个 domain）：区分「taxonomy 未出现在 wp:term 表」（导出缺该
|
|||
|
|
* taxonomy，锚点无从对拍）与「taxonomy 在表但 nicename 无对应 slug」（真正的
|
|||
|
|
* nicename≠slug 不一致）。前者是本 WXR 的实际形态（category/post_tag 未导出
|
|||
|
|
* term 表），如实记录而非记为锚点失败。
|
|||
|
|
*/
|
|||
|
|
async function nicenameAudit(wxrPath) {
|
|||
|
|
const termSlugsByTaxonomy = new Map()
|
|||
|
|
const domainCounts = new Map()
|
|||
|
|
const nicenameCountsByDomain = new Map()
|
|||
|
|
const sampledItems = []
|
|||
|
|
let itemsSeen = 0
|
|||
|
|
let itemsWithCategoryRef = 0
|
|||
|
|
|
|||
|
|
for await (const ev of parseWxrStream(createReadStream(wxrPath))) {
|
|||
|
|
if (ev.kind === 'term') {
|
|||
|
|
const { taxonomy, slug } = ev.term
|
|||
|
|
let set = termSlugsByTaxonomy.get(taxonomy)
|
|||
|
|
if (set === undefined) {
|
|||
|
|
set = new Set()
|
|||
|
|
termSlugsByTaxonomy.set(taxonomy, set)
|
|||
|
|
}
|
|||
|
|
set.add(slug)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
itemsSeen++
|
|||
|
|
const categoryRefs = ev.item.categories.filter((c) => c.domain === 'category')
|
|||
|
|
for (const ref of ev.item.categories) {
|
|||
|
|
domainCounts.set(ref.domain, (domainCounts.get(ref.domain) ?? 0) + 1)
|
|||
|
|
let byNicename = nicenameCountsByDomain.get(ref.domain)
|
|||
|
|
if (byNicename === undefined) {
|
|||
|
|
byNicename = new Map()
|
|||
|
|
nicenameCountsByDomain.set(ref.domain, byNicename)
|
|||
|
|
}
|
|||
|
|
byNicename.set(ref.nicename, (byNicename.get(ref.nicename) ?? 0) + 1)
|
|||
|
|
}
|
|||
|
|
if (categoryRefs.length === 0) continue
|
|||
|
|
itemsWithCategoryRef++
|
|||
|
|
if (sampledItems.length < NICENAME_SAMPLE_ITEMS) {
|
|||
|
|
sampledItems.push({
|
|||
|
|
rawLocation: ev.item.rawLocation,
|
|||
|
|
postId: ev.item.postId,
|
|||
|
|
postType: ev.item.postType,
|
|||
|
|
refs: categoryRefs.map((ref) => ({ nicename: ref.nicename, value: ref.value })),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const categorySlugs = termSlugsByTaxonomy.get('category') ?? new Set()
|
|||
|
|
|
|||
|
|
// 抽检 10 条：明细级比对（nicename ∈ category 域 wp:term 的 term_slug 集）
|
|||
|
|
let sampleMatched = 0
|
|||
|
|
let sampleTotal = 0
|
|||
|
|
const sampleMismatches = []
|
|||
|
|
for (const item of sampledItems) {
|
|||
|
|
for (const ref of item.refs) {
|
|||
|
|
sampleTotal++
|
|||
|
|
const resolved = categorySlugs.has(ref.nicename)
|
|||
|
|
if (resolved) sampleMatched++
|
|||
|
|
else sampleMismatches.push({ rawLocation: item.rawLocation, postId: item.postId, ...ref })
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 全量分层核对（每个 domain 一行）：taxonomy 在表与否 × nicename 可解析与否
|
|||
|
|
const perDomain = [...nicenameCountsByDomain.entries()]
|
|||
|
|
.sort(([a], [b]) => a.localeCompare(b))
|
|||
|
|
.map(([domain, byNicename]) => {
|
|||
|
|
const slugs = termSlugsByTaxonomy.get(domain)
|
|||
|
|
let refs = 0
|
|||
|
|
let matched = 0
|
|||
|
|
let distinctUnresolved = 0
|
|||
|
|
const unresolved = []
|
|||
|
|
for (const [nicename, count] of byNicename) {
|
|||
|
|
refs += count
|
|||
|
|
if (slugs !== undefined && slugs.has(nicename)) {
|
|||
|
|
matched += count
|
|||
|
|
} else {
|
|||
|
|
distinctUnresolved++
|
|||
|
|
if (unresolved.length < UNRESOLVED_LIST_CAP) unresolved.push({ nicename, refCount: count })
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return {
|
|||
|
|
domain,
|
|||
|
|
refs,
|
|||
|
|
matched,
|
|||
|
|
rate: refs === 0 ? null : matched / refs,
|
|||
|
|
taxonomyInTermTable: slugs !== undefined,
|
|||
|
|
distinctUnresolved,
|
|||
|
|
unresolvedCapped: unresolved.length < distinctUnresolved,
|
|||
|
|
unresolved,
|
|||
|
|
}
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
const conclusion = {
|
|||
|
|
/** taxonomy 在 wp:term 表中的域的引用数（这些引用可严格对拍 nicename vs slug） */
|
|||
|
|
refsAgainstExistingTaxonomy: 0,
|
|||
|
|
matchedAgainstExistingTaxonomy: 0,
|
|||
|
|
/** taxonomy 不在 wp:term 表的域的引用数（锚点无从对拍——导出缺表，非锚点不一致） */
|
|||
|
|
refsOfAbsentTaxonomy: 0,
|
|||
|
|
/** taxonomy 在表但 nicename 无对应 slug 的引用数（真正的 nicename≠slug 违例） */
|
|||
|
|
nicenameEqualsSlugViolations: 0,
|
|||
|
|
}
|
|||
|
|
for (const row of perDomain) {
|
|||
|
|
if (row.taxonomyInTermTable) {
|
|||
|
|
conclusion.refsAgainstExistingTaxonomy += row.refs
|
|||
|
|
conclusion.matchedAgainstExistingTaxonomy += row.matched
|
|||
|
|
conclusion.nicenameEqualsSlugViolations += row.refs - row.matched
|
|||
|
|
} else {
|
|||
|
|
conclusion.refsOfAbsentTaxonomy += row.refs
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
sampleRule: `文档序前 ${NICENAME_SAMPLE_ITEMS} 个含 domain="category" 引用的 item（共 ${itemsWithCategoryRef} 个 item 含 category 域引用，items 总数 ${itemsSeen}）`,
|
|||
|
|
sampledItems: sampledItems.map((item) => ({
|
|||
|
|
...item,
|
|||
|
|
refs: item.refs.map((ref) => ({ ...ref, resolvedToTermSlug: categorySlugs.has(ref.nicename) })),
|
|||
|
|
})),
|
|||
|
|
sampleMatched,
|
|||
|
|
sampleTotal,
|
|||
|
|
sampleRate: sampleTotal === 0 ? null : sampleMatched / sampleTotal,
|
|||
|
|
sampleMismatches,
|
|||
|
|
perDomain,
|
|||
|
|
conclusion,
|
|||
|
|
domainCounts: Object.fromEntries([...domainCounts.entries()].sort(([a], [b]) => a.localeCompare(b))),
|
|||
|
|
termTaxonomies: Object.fromEntries(
|
|||
|
|
[...termSlugsByTaxonomy.entries()]
|
|||
|
|
.sort(([a], [b]) => a.localeCompare(b))
|
|||
|
|
.map(([taxonomy, set]) => [taxonomy, set.size]),
|
|||
|
|
),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function main() {
|
|||
|
|
const args = parseArgs(process.argv.slice(2))
|
|||
|
|
const wxrPath = resolve(args.wxr)
|
|||
|
|
const { checks, check, checkEq } = makeChecks()
|
|||
|
|
|
|||
|
|
process.stderr.write(`[smoke] 输入：${wxrPath}\n`)
|
|||
|
|
const fileStat = await stat(wxrPath)
|
|||
|
|
const sha256 = await sha256File(wxrPath)
|
|||
|
|
checkEq('input.bytes', BASELINE.fileBytes, fileStat.size)
|
|||
|
|
checkEq('input.sha256', BASELINE.sha256, sha256)
|
|||
|
|
|
|||
|
|
// —— probe + 内存采样（计划钉死口径）——
|
|||
|
|
process.stderr.write('[smoke] 运行 probe（parse→map→sanitize 全链）+ setInterval(100ms) 内存采样…\n')
|
|||
|
|
let maxHeapUsed = 0
|
|||
|
|
let maxRss = 0
|
|||
|
|
let samples = 0
|
|||
|
|
const sample = () => {
|
|||
|
|
const mu = process.memoryUsage()
|
|||
|
|
if (mu.heapUsed > maxHeapUsed) maxHeapUsed = mu.heapUsed
|
|||
|
|
if (mu.rss > maxRss) maxRss = mu.rss
|
|||
|
|
samples++
|
|||
|
|
}
|
|||
|
|
const sampler = setInterval(sample, SAMPLE_INTERVAL_MS)
|
|||
|
|
const report = await runWxrAdapterReport(createReadStream(wxrPath))
|
|||
|
|
clearInterval(sampler)
|
|||
|
|
sample() // 终末补采一次，覆盖收尾段（报告对象组装等）
|
|||
|
|
|
|||
|
|
// —— 计数对照（基线逐项）——
|
|||
|
|
const t = report.totals
|
|||
|
|
checkEq('totals.rows', BASELINE.rowsTotal, t.rows)
|
|||
|
|
checkEq('totals.terms', BASELINE.terms, t.terms)
|
|||
|
|
checkEq('totals.unsupported', BASELINE.unsupportedTotal, t.unsupported)
|
|||
|
|
checkEq('totals.failed', BASELINE.failedTotal, t.failed)
|
|||
|
|
checkEq('items.total(rows+unsupported+failed)', BASELINE.itemsTotal, t.rows + t.unsupported + t.failed)
|
|||
|
|
|
|||
|
|
for (const [type, expected] of Object.entries(BASELINE.rowsByType)) {
|
|||
|
|
const c = report.counts[type]
|
|||
|
|
checkEq(`rows.${type}(created+skipped+failed)`, expected, c === undefined ? 0 : c.created + c.skipped + c.failed)
|
|||
|
|
}
|
|||
|
|
for (const [type, expected] of Object.entries(BASELINE.unsupportedByType)) {
|
|||
|
|
const c = report.counts[type]
|
|||
|
|
checkEq(`unsupported.${type}`, expected, c === undefined ? 0 : c.unsupported)
|
|||
|
|
}
|
|||
|
|
checkEq(
|
|||
|
|
'counts.keys(无多余/缺失桶)',
|
|||
|
|
EXPECTED_COUNTS_KEYS.join(','),
|
|||
|
|
Object.keys(report.counts).sort().join(','),
|
|||
|
|
)
|
|||
|
|
checkEq('probe.fatalError', null, report.fatalError)
|
|||
|
|
check('memory.heapUsedMax<100MB', `< ${HEAP_LIMIT_BYTES}`, maxHeapUsed, maxHeapUsed < HEAP_LIMIT_BYTES)
|
|||
|
|
|
|||
|
|
// —— nicename 锚点核对（证据采集，非 pass/fail 断言）——
|
|||
|
|
process.stderr.write('[smoke] nicename 锚点核对（第二遍流式）…\n')
|
|||
|
|
const nicename = await nicenameAudit(wxrPath)
|
|||
|
|
|
|||
|
|
const verdict = checks.every((c) => c.pass) ? 'PASS' : 'FAIL'
|
|||
|
|
|
|||
|
|
// —— 产物（与 wxr:probe CLI 同布局 + smoke.json）——
|
|||
|
|
const runId = makeRunId()
|
|||
|
|
const dir = join(resolve(args.outDir), runId)
|
|||
|
|
await mkdir(dir, { recursive: true })
|
|||
|
|
await writeFile(join(dir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`, 'utf8')
|
|||
|
|
await writeFile(join(dir, 'report.html'), `${renderReportHtml(report)}\n`, 'utf8')
|
|||
|
|
const smoke = {
|
|||
|
|
runId,
|
|||
|
|
generatedAt: new Date().toISOString(),
|
|||
|
|
nodeVersion: process.version,
|
|||
|
|
input: { path: wxrPath, bytes: fileStat.size, sha256 },
|
|||
|
|
probe: {
|
|||
|
|
totals: report.totals,
|
|||
|
|
counts: report.counts,
|
|||
|
|
sanitize: report.sanitize,
|
|||
|
|
timing: report.timing,
|
|||
|
|
rowErrors: { recorded: report.rowErrors.length, truncated: report.rowErrorsTruncated },
|
|||
|
|
fatalError: report.fatalError,
|
|||
|
|
},
|
|||
|
|
memory: {
|
|||
|
|
intervalMs: SAMPLE_INTERVAL_MS,
|
|||
|
|
samples,
|
|||
|
|
maxHeapUsedBytes: maxHeapUsed,
|
|||
|
|
maxRssBytes: maxRss,
|
|||
|
|
heapLimitBytes: HEAP_LIMIT_BYTES,
|
|||
|
|
heapUnderLimit: maxHeapUsed < HEAP_LIMIT_BYTES,
|
|||
|
|
},
|
|||
|
|
nicenameAudit: nicename,
|
|||
|
|
checks,
|
|||
|
|
verdict,
|
|||
|
|
artifacts: { dir, report: join(dir, 'report.json'), smoke: join(dir, 'smoke.json') },
|
|||
|
|
}
|
|||
|
|
await writeFile(join(dir, 'smoke.json'), `${JSON.stringify(smoke, null, 2)}\n`, 'utf8')
|
|||
|
|
|
|||
|
|
for (const c of checks) {
|
|||
|
|
process.stderr.write(` [${c.pass ? 'ok' : 'FAIL'}] ${c.name}: expected=${JSON.stringify(c.expected)} actual=${JSON.stringify(c.actual)}\n`)
|
|||
|
|
}
|
|||
|
|
const nc = nicename.conclusion
|
|||
|
|
process.stderr.write(
|
|||
|
|
`[smoke] verdict=${verdict}（nicename 抽检 ${nicename.sampleMatched}/${nicename.sampleTotal}；` +
|
|||
|
|
`在表 taxonomy 对拍 ${nc.matchedAgainstExistingTaxonomy}/${nc.refsAgainstExistingTaxonomy}，` +
|
|||
|
|
`nicename≠slug 违例 ${nc.nicenameEqualsSlugViolations}，缺表 taxonomy 引用 ${nc.refsOfAbsentTaxonomy}）\n`,
|
|||
|
|
)
|
|||
|
|
process.stderr.write(`[smoke] 产物：${join(dir, 'smoke.json')}\n`)
|
|||
|
|
|
|||
|
|
process.stdout.write(`${JSON.stringify(smoke, null, 2)}\n`)
|
|||
|
|
process.exitCode = verdict === 'PASS' ? 0 : 1
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
main().catch((err) => {
|
|||
|
|
process.stderr.write(`[smoke] 未预期异常：${err instanceof Error ? err.stack : String(err)}\n`)
|
|||
|
|
process.exitCode = 1
|
|||
|
|
})
|