feat(adapter-wp): saxes 流式 WXR 解析器
parseWxrStream(readable, {limits, now}) → AsyncGenerator<WxrEvent>:
逐 chunk 喂入 saxes(^6.0.0,ISC;计划所写 ^0.9 不存在于版本历史),
item/term 按文档序交错产出;任一时刻只持有当前聚合状态 + 单 chunk 内
待 yield 队列,绝不收集已产出 item 列表(峰值内存与文件大小解耦)。
- 字段:postId(数值解析)/postType/status/postName/link/title/
contentEncoded/excerptEncoded/postDate/postModified/attachmentUrl/
creator + metas{key,value}[] + categories{domain,nicename,value}[] +
rawLocation(wxr://item/<n>、wxr://term/<n>)
- 错误体系照 PhpError 先例:WxrError 基类;WxrParseError 带 saxes
行列位置(error 事件同步抛出时捕获);WxrLimitError 按 kind 区分
- 限额:bytes/items/terms/depth/cdata/fieldLength(先检查后追加,
超限内容永不入缓冲)/metaPerItem/time(可注入时钟)——超限即抛,
不静默截断
- 实体防线(测试钉死):saxes 实体表固定为五预定义 + 数字引用,
DOCTYPE 内部/SYSTEM 实体不注册——引用即 WxrParseError,替换文本
不进产出;纯字符串解析,全程无网络 I/O
- 跨 chunk 喂入(7 字节/块)与单 chunk 产出一致(StringDecoder 处理
多字节边界);畸形 XML(提前闭合/截断)→ WxrParseError 带行列
130 测试全绿(新增 22)。
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sf/contracts": "workspace:*"
|
||||
"@sf/contracts": "workspace:*",
|
||||
"saxes": "^6.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
export * from './limits.js'
|
||||
export * from './parse/php-unserialize.js'
|
||||
export * from './parse/wxr-stream.js'
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
// saxes 流式 WXR 解析器(Plan 2 Task 3,ADR-005):
|
||||
// Node Readable(WXR 文件字节流)→ AsyncGenerator<WxrEvent>,逐个产出 item/term 事件。
|
||||
//
|
||||
// 流式纪律:任一时刻只持有"当前 item/term 的聚合状态"与一个待 yield 队列(队列仅在
|
||||
// 单个 write() chunk 内暂存已完成事件、紧随其后排空,绝不跨 chunk 累积)——不存在
|
||||
// 已产出 item 的列表;峰值内存与文件大小解耦(Task 8 以真实 5.5MB 文件实测兜底)。
|
||||
//
|
||||
// 实体防线(前置事实 4,测试钉死):saxes 6 的实体表固定为 XML 五预定义实体
|
||||
// (amp/gt/lt/quot/apos)+ 数字字符引用,DOCTYPE 内部/SYSTEM 实体声明一律不注册——
|
||||
// 引用未定义实体即结构错误 "undefined entity."(WxrParseError)。saxes 是纯字符串
|
||||
// 解析器:外部 SYSTEM 标识符从不被抓取,解析全程无任何网络 I/O,替换文本不可能进入产出。
|
||||
//
|
||||
// 限额防线(WxrLimits,超限抛 WxrLimitError 而非静默截断):字节累计(bytes)、
|
||||
// item/term 计数(items/terms)、嵌套深度(depth)、单 CDATA 节长度(cdata)、
|
||||
// 单字段累计长度(fieldLength——先检查后追加,超限文本永不入缓冲,防单个 title 灌爆
|
||||
// 内存)、单 item postmeta 数(metaPerItem)、总时限(time,可注入时钟 now() 测试)。
|
||||
//
|
||||
// 错误体系照 PhpError 先例:WxrError 公共基类;WxrParseError 带 saxes 行列位置
|
||||
// (在 error 事件同步抛出时捕获 parser.line/column,即出错瞬间的准确位置);
|
||||
// WxrLimitError 以 kind 区分限额维度。
|
||||
|
||||
import { StringDecoder } from 'node:string_decoder'
|
||||
import type { Readable } from 'node:stream'
|
||||
import { SaxesParser, type SaxesTagPlain } from 'saxes'
|
||||
import { DEFAULT_WXR_LIMITS, type WxrLimits } from '../limits.js'
|
||||
|
||||
/** item 内 <wp:postmeta> 聚合产物:{key, value} 原样字符串(PHP 串解析是 Task 4 的职责) */
|
||||
export interface WxrPostmeta {
|
||||
key: string
|
||||
value: string
|
||||
}
|
||||
|
||||
/** item 内 <category domain nicename>text</category> 聚合产物 */
|
||||
export interface WxrCategoryRef {
|
||||
domain: string
|
||||
nicename: string
|
||||
value: string
|
||||
}
|
||||
|
||||
/** WXR <item> 的解析产物(字段即 WordPress 导出结构;缺省字段为 '' / null / 空数组) */
|
||||
export interface WxrItem {
|
||||
/** 来源定位:wxr://item/<文档内序号从1起>(按 item 出现顺序编号) */
|
||||
rawLocation: string
|
||||
/** <wp:post_id>(整数解析;缺失/非数字 → null) */
|
||||
postId: number | null
|
||||
/** <wp:post_type> */
|
||||
postType: string
|
||||
/** <wp:status> */
|
||||
status: string
|
||||
/** <wp:post_name> */
|
||||
postName: string
|
||||
/** <link> */
|
||||
link: string
|
||||
/** <title>(CDATA 已解包) */
|
||||
title: string
|
||||
/** <content:encoded>(CDATA 已解包,原文不清洗——清洗是 Task 5 的职责) */
|
||||
contentEncoded: string
|
||||
/** <excerpt:encoded>(CDATA 已解包) */
|
||||
excerptEncoded: string
|
||||
/** <wp:post_date>(原始字符串,格式化是下游职责) */
|
||||
postDate: string
|
||||
/** <wp:post_modified> */
|
||||
postModified: string
|
||||
/** <wp:attachment_url>(非 attachment 为 '') */
|
||||
attachmentUrl: string
|
||||
/** <dc:creator> */
|
||||
creator: string
|
||||
/** <wp:postmeta> 按文档序聚合 */
|
||||
metas: WxrPostmeta[]
|
||||
/** <category> 按文档序聚合 */
|
||||
categories: WxrCategoryRef[]
|
||||
}
|
||||
|
||||
/** WXR <wp:term> 的解析产物 */
|
||||
export interface WxrTerm {
|
||||
/** 来源定位:wxr://term/<文档内序号从1起> */
|
||||
rawLocation: string
|
||||
/** <wp:term_id>(整数解析;缺失/非数字 → null) */
|
||||
termId: number | null
|
||||
/** <wp:term_taxonomy> */
|
||||
taxonomy: string
|
||||
/** <wp:term_slug> */
|
||||
slug: string
|
||||
/** <wp:term_name> */
|
||||
name: string
|
||||
}
|
||||
|
||||
/** 解析器产出事件:kind 判别联合(item 与 term 按文档序交错产出) */
|
||||
export type WxrEvent = { kind: 'item'; item: WxrItem } | { kind: 'term'; term: WxrTerm }
|
||||
|
||||
/** 本模块错误公共基类:统一 name 约定(仅经子类实例化,照 PhpError 先例) */
|
||||
export abstract class WxrError extends Error {
|
||||
constructor(name: string, message: string) {
|
||||
super(message)
|
||||
this.name = name
|
||||
}
|
||||
}
|
||||
|
||||
/** 结构/语法错误(含 saxes 原始信息):line/column 为出错瞬间 saxes 报告的位置 */
|
||||
export class WxrParseError extends WxrError {
|
||||
readonly line: number
|
||||
readonly column: number
|
||||
|
||||
constructor(message: string, line: number, column: number) {
|
||||
super('WxrParseError', `${message}(line ${line}, column ${column})`)
|
||||
this.line = line
|
||||
this.column = column
|
||||
}
|
||||
}
|
||||
|
||||
/** 限额错误 kind:与 WxrLimits 的流式解析维度一一对应(任务清单六种 + bytes/metaPerItem 补全契约全维度) */
|
||||
export type WxrLimitKind =
|
||||
| 'items'
|
||||
| 'terms'
|
||||
| 'depth'
|
||||
| 'cdata'
|
||||
| 'fieldLength'
|
||||
| 'time'
|
||||
| 'bytes'
|
||||
| 'metaPerItem'
|
||||
|
||||
/** 限额错误(区别于 WxrParseError):kind 指明超限维度 */
|
||||
export class WxrLimitError extends WxrError {
|
||||
readonly kind: WxrLimitKind
|
||||
|
||||
constructor(kind: WxrLimitKind, message: string) {
|
||||
super('WxrLimitError', message)
|
||||
this.kind = kind
|
||||
}
|
||||
}
|
||||
|
||||
/** parseWxrStream 选项:limits 支持部分覆盖(缺省维度取 DEFAULT_WXR_LIMITS);now 可注入时钟(测试 time 预算) */
|
||||
export interface WxrStreamOptions {
|
||||
limits?: Partial<WxrLimits>
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
/** item 直接子字段:元素限定名 → WxrItem 字段(数值字段 postId 在提交时单独解析) */
|
||||
const ITEM_FIELDS: Record<string, keyof WxrItem> = {
|
||||
'wp:post_id': 'postId',
|
||||
title: 'title',
|
||||
link: 'link',
|
||||
'content:encoded': 'contentEncoded',
|
||||
'excerpt:encoded': 'excerptEncoded',
|
||||
'wp:post_date': 'postDate',
|
||||
'wp:post_modified': 'postModified',
|
||||
'wp:post_name': 'postName',
|
||||
'wp:post_type': 'postType',
|
||||
'wp:status': 'status',
|
||||
'wp:attachment_url': 'attachmentUrl',
|
||||
'dc:creator': 'creator',
|
||||
}
|
||||
|
||||
/** term 直接子字段:元素限定名 → WxrTerm 字段(数值字段 termId 在提交时单独解析) */
|
||||
const TERM_FIELDS: Record<string, keyof WxrTerm> = {
|
||||
'wp:term_id': 'termId',
|
||||
'wp:term_taxonomy': 'taxonomy',
|
||||
'wp:term_slug': 'slug',
|
||||
'wp:term_name': 'name',
|
||||
}
|
||||
|
||||
/** 当前文本捕获目标:一个元素一个目标,闭标签时提交并清空 */
|
||||
type Target =
|
||||
| { el: string; owner: 'item'; field: keyof WxrItem }
|
||||
| { el: string; owner: 'term'; field: keyof WxrTerm }
|
||||
| { el: string; owner: 'metaKey' }
|
||||
| { el: string; owner: 'metaValue' }
|
||||
| { el: string; owner: 'category'; domain: string; nicename: string }
|
||||
|
||||
function parseIntOrNull(text: string): number | null {
|
||||
const n = Number.parseInt(text.trim(), 10)
|
||||
return Number.isFinite(n) ? n : null
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式解析 WXR 文件:逐 chunk 喂入 saxes、逐事件产出,绝不整文件进内存、绝不访问网络。
|
||||
*
|
||||
* 产出契约:
|
||||
* - item/term 按文档序交错产出;每个事件在其闭合标签被处理后立即排队
|
||||
* - 限额/结构错误以 WxrLimitError / WxrParseError 拒绝整个流(已 yield 的事件不撤回)
|
||||
* - 消费方提前 break/throw 时销毁上游流(finally 兜底)
|
||||
*/
|
||||
export async function* parseWxrStream(
|
||||
readable: Readable,
|
||||
opts: WxrStreamOptions = {},
|
||||
): AsyncGenerator<WxrEvent, void, void> {
|
||||
const limits: WxrLimits = { ...DEFAULT_WXR_LIMITS, ...opts.limits }
|
||||
const now = opts.now ?? Date.now
|
||||
const startedAt = now()
|
||||
|
||||
const decoder = new StringDecoder('utf8')
|
||||
const parser = new SaxesParser({ position: true, xmlns: false, fragment: false })
|
||||
const queue: WxrEvent[] = []
|
||||
|
||||
let totalBytes = 0
|
||||
let depth = 0
|
||||
let itemIndex = 0
|
||||
let termIndex = 0
|
||||
let mode: 'outside' | 'item' | 'term' = 'outside'
|
||||
let item: WxrItem | null = null
|
||||
let term: WxrTerm | null = null
|
||||
let target: Target | null = null
|
||||
let buf = ''
|
||||
let meta: WxrPostmeta | null = null
|
||||
|
||||
const fail = (kind: WxrLimitKind, message: string): never => {
|
||||
throw new WxrLimitError(kind, `${message}(line ${parser.line}, column ${parser.column})`)
|
||||
}
|
||||
|
||||
/** 追加文本到当前字段缓冲:先限额检查后追加——超限内容永不入缓冲(内存上界保证) */
|
||||
const append = (text: string, isCdata: boolean): void => {
|
||||
if (isCdata && text.length > limits.maxCdataLength) {
|
||||
fail('cdata', `CDATA 长度 ${text.length} 超过上限 ${limits.maxCdataLength}`)
|
||||
}
|
||||
if (target === null) return // 字段元素之外的文本(标签间空白等)——丢弃
|
||||
if (buf.length + text.length > limits.maxFieldLength) {
|
||||
fail(
|
||||
'fieldLength',
|
||||
`字段 <${target.el}> 累计长度 ${buf.length + text.length} 超过上限 ${limits.maxFieldLength}`,
|
||||
)
|
||||
}
|
||||
buf += text
|
||||
}
|
||||
|
||||
parser.on('error', (err: Error) => {
|
||||
// 同步抛出即刻中止 write()/close();此刻 parser.line/column 即出错位置(测试钉死)
|
||||
throw new WxrParseError(`saxes: ${err.message}`, parser.line ?? 0, parser.column ?? 0)
|
||||
})
|
||||
|
||||
parser.on('opentag', (tag: SaxesTagPlain) => {
|
||||
const name = tag.name
|
||||
depth++
|
||||
if (depth > limits.maxDepth) {
|
||||
fail('depth', `嵌套深度 ${depth} 超过上限 ${limits.maxDepth}(<${name}>)`)
|
||||
}
|
||||
|
||||
if (mode === 'outside' && name === 'item') {
|
||||
itemIndex++
|
||||
if (itemIndex > limits.maxItems) {
|
||||
fail('items', `item 数 ${itemIndex} 超过上限 ${limits.maxItems}`)
|
||||
}
|
||||
mode = 'item'
|
||||
item = {
|
||||
rawLocation: `wxr://item/${itemIndex}`,
|
||||
postId: null,
|
||||
postType: '',
|
||||
status: '',
|
||||
postName: '',
|
||||
link: '',
|
||||
title: '',
|
||||
contentEncoded: '',
|
||||
excerptEncoded: '',
|
||||
postDate: '',
|
||||
postModified: '',
|
||||
attachmentUrl: '',
|
||||
creator: '',
|
||||
metas: [],
|
||||
categories: [],
|
||||
}
|
||||
return
|
||||
}
|
||||
if (mode === 'outside' && name === 'wp:term') {
|
||||
termIndex++
|
||||
if (termIndex > limits.maxTermCount) {
|
||||
fail('terms', `term 数 ${termIndex} 超过上限 ${limits.maxTermCount}`)
|
||||
}
|
||||
mode = 'term'
|
||||
term = {
|
||||
rawLocation: `wxr://term/${termIndex}`,
|
||||
termId: null,
|
||||
taxonomy: '',
|
||||
slug: '',
|
||||
name: '',
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (mode === 'item' && item !== null) {
|
||||
if (meta !== null) {
|
||||
// postmeta 内部:key/value 二选一
|
||||
if (name === 'wp:meta_key') target = { el: name, owner: 'metaKey' }
|
||||
else if (name === 'wp:meta_value') target = { el: name, owner: 'metaValue' }
|
||||
} else if (name === 'wp:postmeta') {
|
||||
if (item.metas.length + 1 > limits.maxMetaPerItem) {
|
||||
fail(
|
||||
'metaPerItem',
|
||||
`item ${item.rawLocation} 的 postmeta 数 ${item.metas.length + 1} 超过上限 ${limits.maxMetaPerItem}`,
|
||||
)
|
||||
}
|
||||
meta = { key: '', value: '' }
|
||||
} else if (name === 'category') {
|
||||
target = {
|
||||
el: name,
|
||||
owner: 'category',
|
||||
domain: tag.attributes['domain'] ?? '',
|
||||
nicename: tag.attributes['nicename'] ?? '',
|
||||
}
|
||||
} else if (target === null && ITEM_FIELDS[name] !== undefined) {
|
||||
target = { el: name, owner: 'item', field: ITEM_FIELDS[name] as keyof WxrItem }
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (mode === 'term' && term !== null && target === null && TERM_FIELDS[name] !== undefined) {
|
||||
target = { el: name, owner: 'term', field: TERM_FIELDS[name] as keyof WxrTerm }
|
||||
}
|
||||
})
|
||||
|
||||
parser.on('text', (text: string) => {
|
||||
append(text, false)
|
||||
})
|
||||
|
||||
parser.on('cdata', (text: string) => {
|
||||
append(text, true)
|
||||
})
|
||||
|
||||
parser.on('closetag', (tag: SaxesTagPlain) => {
|
||||
const name = tag.name
|
||||
depth--
|
||||
|
||||
// 先提交文本字段(字段闭标签紧邻结构闭标签之前)
|
||||
if (target !== null && target.el === name) {
|
||||
if (target.owner === 'item' && item !== null) {
|
||||
if (name === 'wp:post_id') item.postId = parseIntOrNull(buf)
|
||||
else (item[target.field] as string) = buf
|
||||
} else if (target.owner === 'term' && term !== null) {
|
||||
if (name === 'wp:term_id') term.termId = parseIntOrNull(buf)
|
||||
else (term[target.field] as string) = buf
|
||||
} else if (target.owner === 'metaKey' && meta !== null) {
|
||||
meta.key = buf
|
||||
} else if (target.owner === 'metaValue' && meta !== null) {
|
||||
meta.value = buf
|
||||
} else if (target.owner === 'category' && item !== null) {
|
||||
item.categories.push({ domain: target.domain, nicename: target.nicename, value: buf })
|
||||
}
|
||||
target = null
|
||||
buf = ''
|
||||
}
|
||||
|
||||
if (name === 'wp:postmeta' && mode === 'item' && item !== null && meta !== null) {
|
||||
item.metas.push(meta)
|
||||
meta = null
|
||||
return
|
||||
}
|
||||
|
||||
if (name === 'item' && mode === 'item' && item !== null) {
|
||||
queue.push({ kind: 'item', item })
|
||||
item = null
|
||||
mode = 'outside'
|
||||
return
|
||||
}
|
||||
|
||||
if (name === 'wp:term' && mode === 'term' && term !== null) {
|
||||
queue.push({ kind: 'term', term })
|
||||
term = null
|
||||
mode = 'outside'
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
for await (const chunk of readable as AsyncIterable<Buffer | string>) {
|
||||
totalBytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
|
||||
if (totalBytes > limits.maxWxrBytes) {
|
||||
fail('bytes', `累计输入 ${totalBytes} 字节超过上限 ${limits.maxWxrBytes}`)
|
||||
}
|
||||
const elapsed = now() - startedAt
|
||||
if (elapsed > limits.timeBudgetMs) {
|
||||
throw new WxrLimitError('time', `处理时长 ${elapsed}ms 超过预算 ${limits.timeBudgetMs}ms`)
|
||||
}
|
||||
parser.write(decoder.write(chunk))
|
||||
// 排空本次 chunk 内完成的事件(队列不跨 chunk 累积——结构即流式证明)
|
||||
while (queue.length > 0) {
|
||||
const ev = queue.shift()
|
||||
if (ev !== undefined) yield ev
|
||||
}
|
||||
}
|
||||
// StringDecoder 尾部(跨 chunk 的多字节残余)先写空再 close——saxes 类型签名的 close() 无参
|
||||
const tail = decoder.end()
|
||||
if (tail !== '') parser.write(tail)
|
||||
parser.close()
|
||||
while (queue.length > 0) {
|
||||
const ev = queue.shift()
|
||||
if (ev !== undefined) yield ev
|
||||
}
|
||||
} finally {
|
||||
// 消费方提前退出或解析失败:销毁上游流,避免泄漏(已完成/已失败流 destroy 为无害幂等)
|
||||
readable.destroy()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { PassThrough, Readable } from 'node:stream'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
parseWxrStream,
|
||||
WxrError,
|
||||
WxrLimitError,
|
||||
WxrParseError,
|
||||
type WxrEvent,
|
||||
type WxrItem,
|
||||
} from '../src/parse/wxr-stream.js'
|
||||
|
||||
// mini.xml(Plan 1 夹具):5 item(post 101 / product 111 / page 121 / attachment 201,202)
|
||||
// + 2 term(31 category / 32 product_cat);term 块在第三个 item 之后、attachment 之前。
|
||||
const FIXTURE = fileURLToPath(new URL('../../../tests/fixtures/wxr/mini.xml', import.meta.url))
|
||||
|
||||
/** 整串一次性喂入(单 chunk 路径) */
|
||||
function fromString(xml: string): Readable {
|
||||
const s = new PassThrough()
|
||||
s.end(xml)
|
||||
return s
|
||||
}
|
||||
|
||||
/** 按字节切块喂入(跨 chunk 的 UTF-8 多字节与 token 边界路径;7 字节必拆中文与 ]]>) */
|
||||
function fromStringChunked(xml: string, size = 7): Readable {
|
||||
return Readable.from(
|
||||
(function* () {
|
||||
const buf = Buffer.from(xml, 'utf8')
|
||||
for (let i = 0; i < buf.length; i += size) yield buf.subarray(i, i + size)
|
||||
})(),
|
||||
)
|
||||
}
|
||||
|
||||
async function collect(gen: AsyncGenerator<WxrEvent>): Promise<WxrEvent[]> {
|
||||
const out: WxrEvent[] = []
|
||||
for await (const ev of gen) out.push(ev)
|
||||
return out
|
||||
}
|
||||
|
||||
/** 捕获 reject 的错误实例(而非仅匹配),以便断言 kind / line / column 细节(照 php-unserialize 测试先例) */
|
||||
async function captureRejection(p: Promise<unknown>): Promise<unknown> {
|
||||
try {
|
||||
await p
|
||||
} catch (e) {
|
||||
return e
|
||||
}
|
||||
throw new Error('期望抛出但未抛出')
|
||||
}
|
||||
|
||||
function itemsOf(events: WxrEvent[]): WxrItem[] {
|
||||
return events
|
||||
.filter((ev): ev is Extract<WxrEvent, { kind: 'item' }> => ev.kind === 'item')
|
||||
.map((ev) => ev.item)
|
||||
}
|
||||
|
||||
/** 概要签名:item:<postId>:<postType> / term:<termId>:<taxonomy>(按文档序) */
|
||||
function signature(events: WxrEvent[]): string[] {
|
||||
return events.map((ev) =>
|
||||
ev.kind === 'item'
|
||||
? `item:${ev.item.postId}:${ev.item.postType}`
|
||||
: `term:${ev.term.termId}:${ev.term.taxonomy}`,
|
||||
)
|
||||
}
|
||||
|
||||
describe('parseWxrStream:mini.xml 流式产出', () => {
|
||||
it('产出顺序与文档一致:post → product → page → term → term → attachment → attachment', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
expect(signature(events)).toStrictEqual([
|
||||
'item:101:post',
|
||||
'item:111:product',
|
||||
'item:121:page',
|
||||
'term:31:category',
|
||||
'term:32:product_cat',
|
||||
'item:201:attachment',
|
||||
'item:202:attachment',
|
||||
])
|
||||
})
|
||||
|
||||
it('跨 chunk 切块喂入(7 字节/块)产出与单 chunk 完全一致', async () => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const xml = readFileSync(FIXTURE, 'utf8')
|
||||
const events = await collect(parseWxrStream(fromStringChunked(xml)))
|
||||
expect(signature(events)).toStrictEqual([
|
||||
'item:101:post',
|
||||
'item:111:product',
|
||||
'item:121:page',
|
||||
'term:31:category',
|
||||
'term:32:product_cat',
|
||||
'item:201:attachment',
|
||||
'item:202:attachment',
|
||||
])
|
||||
})
|
||||
|
||||
it('item 事件字段完整:post 101 逐一断言具体值', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
const item = itemsOf(events).find((it) => it.postId === 101)
|
||||
expect(item).toBeDefined()
|
||||
expect(item).toMatchObject({
|
||||
postId: 101,
|
||||
postType: 'post',
|
||||
status: 'publish',
|
||||
postName: 'fixture-news-a',
|
||||
link: 'https://fixture.example/news/fixture-news-a.html',
|
||||
title: 'Fixture News A',
|
||||
contentEncoded:
|
||||
'<p>Body with <a href="/products/fixture-product/">link</a> and <img src="https://fixture.example/wp-content/uploads/2026/10/fixture-a.png"/></p>',
|
||||
excerptEncoded: 'Fixture excerpt A',
|
||||
postDate: '2026-10-05 09:00:00',
|
||||
postModified: '2026-10-06 10:00:00',
|
||||
attachmentUrl: '',
|
||||
creator: 'editor',
|
||||
})
|
||||
})
|
||||
|
||||
it('attachment item:attachmentUrl 与 inherit 状态', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
const a201 = itemsOf(events).find((it) => it.postId === 201)
|
||||
expect(a201).toMatchObject({
|
||||
postType: 'attachment',
|
||||
status: 'inherit',
|
||||
attachmentUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png',
|
||||
})
|
||||
})
|
||||
|
||||
it('term 事件字段:term_id/taxonomy/slug/name(term 31)', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
const term = events.find((ev) => ev.kind === 'term' && ev.term.termId === 31)
|
||||
expect(term).toBeDefined()
|
||||
expect(term?.kind === 'term' && term.term).toMatchObject({
|
||||
termId: 31,
|
||||
taxonomy: 'category',
|
||||
slug: 'company',
|
||||
name: 'Company',
|
||||
})
|
||||
})
|
||||
|
||||
it('postmeta 聚合:post 101 恰 3 条(_thumbnail_id + Yoast title/metadesc)', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
const item = itemsOf(events).find((it) => it.postId === 101)
|
||||
expect(item?.metas).toStrictEqual([
|
||||
{ key: '_thumbnail_id', value: '201' },
|
||||
{ key: '_yoast_wpseo_title', value: 'Fixture SEO Title A' },
|
||||
{ key: '_yoast_wpseo_metadesc', value: 'Fixture SEO description A.' },
|
||||
])
|
||||
})
|
||||
|
||||
it('postmeta 聚合:product 111 恰 3 条(gallery 与 wppop_sections 原样字符串)', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
const item = itemsOf(events).find((it) => it.postId === 111)
|
||||
expect(item?.metas).toStrictEqual([
|
||||
{ key: '_thumbnail_id', value: '202' },
|
||||
{ key: '_product_image_gallery', value: '203,204' },
|
||||
{
|
||||
key: 'wppop_sections',
|
||||
value: 'a:1:{i:0;a:2:{s:5:"title";s:8:"SpecsTab";s:7:"content";s:10:"<p>Tab</p>";}}',
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('category 元素聚合进 item 事件:post 101 → category/company/Company;product 111 无', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
const post101 = itemsOf(events).find((it) => it.postId === 101)
|
||||
expect(post101?.categories).toStrictEqual([{ domain: 'category', nicename: 'company', value: 'Company' }])
|
||||
const product111 = itemsOf(events).find((it) => it.postId === 111)
|
||||
expect(product111?.categories).toStrictEqual([])
|
||||
})
|
||||
|
||||
it('rawLocation:wxr://item/<文档内序号从1起>,term 同理', async () => {
|
||||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||||
expect(itemsOf(events).map((it) => it.rawLocation)).toStrictEqual([
|
||||
'wxr://item/1',
|
||||
'wxr://item/2',
|
||||
'wxr://item/3',
|
||||
'wxr://item/4',
|
||||
'wxr://item/5',
|
||||
])
|
||||
expect(
|
||||
events
|
||||
.filter((ev) => ev.kind === 'term')
|
||||
.map((ev) => (ev.kind === 'term' ? ev.term.rawLocation : '')),
|
||||
).toStrictEqual(['wxr://term/1', 'wxr://term/2'])
|
||||
})
|
||||
})
|
||||
|
||||
// 标题 CDATA 长 16 字符:供 maxCdataLength / maxFieldLength(=10)触发超限
|
||||
const SMALL_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:wp="http://wordpress.org/export/1.2/">
|
||||
<channel>
|
||||
<item><title><![CDATA[AAAAAAAAAAAAAAAA]]></title><wp:post_id>1</wp:post_id><wp:post_type>post</wp:post_type><wp:status>publish</wp:status></item>
|
||||
<item><title><![CDATA[BBBBBBBBBBBBBBBB]]></title><wp:post_id>2</wp:post_id><wp:post_type>post</wp:post_type><wp:status>publish</wp:status></item>
|
||||
<item><title><![CDATA[CCCCCCCCCCCCCCCC]]></title><wp:post_id>3</wp:post_id><wp:post_type>post</wp:post_type><wp:status>publish</wp:status></item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
describe('parseWxrStream:限额防线(WxrLimitError,kind 区分)', () => {
|
||||
it('maxItems=2:第 3 个 item 开始前抛 kind items', async () => {
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxItems: 2 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err).toBeInstanceOf(WxrError)
|
||||
expect(err.kind).toBe('items')
|
||||
})
|
||||
|
||||
it('maxCdataLength=10:超长 CDATA 抛 kind cdata', async () => {
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxCdataLength: 10 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('cdata')
|
||||
})
|
||||
|
||||
it('maxFieldLength=10:单字段累计超限抛 kind fieldLength(防单个 title 灌爆内存)', async () => {
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxFieldLength: 10 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('fieldLength')
|
||||
})
|
||||
|
||||
it('maxDepth=2:rss>channel>item 嵌套超限抛 kind depth', async () => {
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxDepth: 2 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('depth')
|
||||
})
|
||||
|
||||
it('timeBudgetMs:可注入时钟超预算 → kind time', async () => {
|
||||
let calls = 0
|
||||
const now = () => (calls++ === 0 ? 0 : 1_000_000) // 第 1 次取起点,其后均"超预算"
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { timeBudgetMs: 120_000 }, now })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('time')
|
||||
})
|
||||
|
||||
it('maxTermCount=1:第 2 个 term 抛 kind terms', async () => {
|
||||
const xml = SMALL_XML.replace(
|
||||
'</channel>',
|
||||
'<wp:term><wp:term_id>1</wp:term_id><wp:term_taxonomy>category</wp:term_taxonomy><wp:term_slug>a</wp:term_slug><wp:term_name>A</wp:term_name></wp:term>' +
|
||||
'<wp:term><wp:term_id>2</wp:term_id><wp:term_taxonomy>category</wp:term_taxonomy><wp:term_slug>b</wp:term_slug><wp:term_name>B</wp:term_name></wp:term></channel>',
|
||||
)
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(xml), { limits: { maxTermCount: 1 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('terms')
|
||||
})
|
||||
|
||||
it('maxWxrBytes:累计字节数超限抛 kind bytes', async () => {
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxWxrBytes: 10 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('bytes')
|
||||
})
|
||||
|
||||
it('maxMetaPerItem:单 item postmeta 数超限抛 kind metaPerItem', async () => {
|
||||
const xml = SMALL_XML.replace(
|
||||
'</item>',
|
||||
'<wp:postmeta><wp:meta_key><![CDATA[k1]]></wp:meta_key><wp:meta_value><![CDATA[v1]]></wp:meta_value></wp:postmeta>' +
|
||||
'<wp:postmeta><wp:meta_key><![CDATA[k2]]></wp:meta_key><wp:meta_value><![CDATA[v2]]></wp:meta_value></wp:postmeta></item>',
|
||||
)
|
||||
const err = (await captureRejection(
|
||||
collect(parseWxrStream(fromString(xml), { limits: { maxMetaPerItem: 1 } })),
|
||||
)) as WxrLimitError
|
||||
expect(err).toBeInstanceOf(WxrLimitError)
|
||||
expect(err.kind).toBe('metaPerItem')
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseWxrStream:外部实体防线(不解析实体、不注入替换文本、不访问网络)', () => {
|
||||
// 选定行为(钉死):saxes 6 实体表固定为 XML 五预定义 + 数字字符引用,DOCTYPE
|
||||
// 内部/SYSTEM 实体声明一律不注册——引用即 "undefined entity." 结构错误(WxrParseError),
|
||||
// 替换文本绝不进入任何产出字段;saxes 为纯字符串解析器,外部 SYSTEM 标识符从不被抓取。
|
||||
const SECRET = 'SECRET-REPLACEMENT-TEXT'
|
||||
|
||||
async function withTempEntityXml(
|
||||
doctype: string,
|
||||
body: string,
|
||||
run: (path: string) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'wxr-entity-'))
|
||||
const path = join(dir, 'entity.xml')
|
||||
await writeFile(
|
||||
path,
|
||||
`<?xml version="1.0" encoding="UTF-8"?>\n${doctype}\n<rss version="2.0" xmlns:wp="http://wordpress.org/export/1.2/"><channel>${body}</channel></rss>`,
|
||||
'utf8',
|
||||
)
|
||||
try {
|
||||
await run(path)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
it('DOCTYPE 内部 ENTITY:引用处抛 WxrParseError,替换文本不进产出(临时文件 + createReadStream)', async () => {
|
||||
await withTempEntityXml(
|
||||
`<!DOCTYPE rss [<!ENTITY xxe "${SECRET}">]>`,
|
||||
'<item><title>&xxe;</title><wp:post_id>1</wp:post_id><wp:post_type>post</wp:post_type></item>',
|
||||
async (path) => {
|
||||
const events: WxrEvent[] = []
|
||||
const err = (await captureRejection(
|
||||
(async () => {
|
||||
for await (const ev of parseWxrStream(createReadStream(path))) events.push(ev)
|
||||
})(),
|
||||
)) as WxrParseError
|
||||
expect(err).toBeInstanceOf(WxrParseError)
|
||||
expect(err.message).toContain('undefined entity')
|
||||
// 无论解析走到哪一步:替换文本绝不出现于任何已产出事件
|
||||
expect(JSON.stringify(events)).not.toContain(SECRET)
|
||||
},
|
||||
)
|
||||
})
|
||||
|
||||
it('SYSTEM 外部实体:同样拒绝(不解析、不抓取——无任何网络 I/O)', async () => {
|
||||
await withTempEntityXml(
|
||||
'<!DOCTYPE rss [<!ENTITY ext SYSTEM "file:///etc/passwd">]>',
|
||||
'<item><title>&ext;</title><wp:post_id>1</wp:post_id><wp:post_type>post</wp:post_type></item>',
|
||||
async (path) => {
|
||||
const err = (await captureRejection(collect(parseWxrStream(createReadStream(path))))) as WxrParseError
|
||||
expect(err).toBeInstanceOf(WxrParseError)
|
||||
expect(err.message).toContain('undefined entity')
|
||||
},
|
||||
)
|
||||
})
|
||||
|
||||
it('预定义实体不受影响:& / 数字字符引用正常解析', async () => {
|
||||
const xml =
|
||||
'<?xml version="1.0"?><rss version="2.0" xmlns:wp="http://wordpress.org/export/1.2/"><channel>' +
|
||||
'<item><title>plain</title><wp:post_name>a&bA</wp:post_name><wp:post_id>7</wp:post_id><wp:post_type>post</wp:post_type></item>' +
|
||||
'</channel></rss>'
|
||||
const events = await collect(parseWxrStream(fromString(xml)))
|
||||
expect(itemsOf(events)[0]?.postName).toBe('a&bA')
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseWxrStream:畸形 XML → WxrParseError(带 saxes 行列位置)', () => {
|
||||
it('提前闭合:精确 line/column(4:17)与原始信息', async () => {
|
||||
const xml =
|
||||
'<?xml version="1.0"?>\n<rss><channel>\n<item><title>a</title></item>\n</channel></item></rss>'
|
||||
const err = (await captureRejection(collect(parseWxrStream(fromString(xml))))) as WxrParseError
|
||||
expect(err).toBeInstanceOf(WxrParseError)
|
||||
expect(err).toBeInstanceOf(WxrError)
|
||||
expect(err.line).toBe(4)
|
||||
expect(err.column).toBe(17)
|
||||
expect(err.message).toContain('unexpected close tag')
|
||||
})
|
||||
|
||||
it('未闭合(截断)文件:close 阶段报错,行列为有效正数', async () => {
|
||||
const xml = '<?xml version="1.0"?>\n<rss><channel>\n<item><title>a</title></item>\n</channel>'
|
||||
const err = (await captureRejection(collect(parseWxrStream(fromString(xml))))) as WxrParseError
|
||||
expect(err).toBeInstanceOf(WxrParseError)
|
||||
expect(err.line).toBeGreaterThanOrEqual(1)
|
||||
expect(err.column).toBeGreaterThanOrEqual(0)
|
||||
expect(err.message).toContain('unclosed')
|
||||
})
|
||||
})
|
||||
Generated
+16
@@ -66,6 +66,9 @@ importers:
|
||||
'@sf/contracts':
|
||||
specifier: workspace:*
|
||||
version: link:../contracts
|
||||
saxes:
|
||||
specifier: ^6.0.0
|
||||
version: 6.0.0
|
||||
|
||||
packages/contracts: {}
|
||||
|
||||
@@ -1513,6 +1516,10 @@ packages:
|
||||
rxjs@7.8.2:
|
||||
resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==}
|
||||
|
||||
saxes@6.0.0:
|
||||
resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==}
|
||||
engines: {node: '>=v12.22.7'}
|
||||
|
||||
semver@7.8.5:
|
||||
resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==}
|
||||
engines: {node: '>=10'}
|
||||
@@ -1712,6 +1719,9 @@ packages:
|
||||
resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
xmlchars@2.2.0:
|
||||
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
|
||||
|
||||
yocto-queue@0.1.0:
|
||||
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
|
||||
engines: {node: '>=10'}
|
||||
@@ -2843,6 +2853,10 @@ snapshots:
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
saxes@6.0.0:
|
||||
dependencies:
|
||||
xmlchars: 2.2.0
|
||||
|
||||
semver@7.8.5: {}
|
||||
|
||||
shebang-command@2.0.0:
|
||||
@@ -3020,4 +3034,6 @@ snapshots:
|
||||
|
||||
word-wrap@1.2.5: {}
|
||||
|
||||
xmlchars@2.2.0: {}
|
||||
|
||||
yocto-queue@0.1.0: {}
|
||||
|
||||
Reference in New Issue
Block a user