浅念
25e8b9cb5c
parseWxrStream(readable, {limits, now}) → AsyncGenerator<WxrEvent>：
逐 chunk 喂入 saxes（^6.0.0，ISC；计划所写 ^0.9 不存在于版本历史），
item/term 按文档序交错产出；任一时刻只持有当前聚合状态 + 单 chunk 内
待 yield 队列，绝不收集已产出 item 列表（峰值内存与文件大小解耦）。
- 字段：postId(数值解析)/postType/status/postName/link/title/
contentEncoded/excerptEncoded/postDate/postModified/attachmentUrl/
creator + metas{key,value}[] + categories{domain,nicename,value}[] +
rawLocation(wxr://item/<n>、wxr://term/<n>)
- 错误体系照 PhpError 先例：WxrError 基类；WxrParseError 带 saxes
行列位置（error 事件同步抛出时捕获）；WxrLimitError 按 kind 区分
- 限额：bytes/items/terms/depth/cdata/fieldLength（先检查后追加，
超限内容永不入缓冲）/metaPerItem/time（可注入时钟）——超限即抛，
不静默截断
- 实体防线（测试钉死）：saxes 实体表固定为五预定义 + 数字引用，
DOCTYPE 内部/SYSTEM 实体不注册——引用即 WxrParseError，替换文本
不进产出；纯字符串解析，全程无网络 I/O
- 跨 chunk 喂入（7 字节/块）与单 chunk 产出一致（StringDecoder 处理
多字节边界）；畸形 XML（提前闭合/截断）→ WxrParseError 带行列
130 测试全绿（新增 22）。
Co-Authored-By: Claude Code <noreply@anthropic.com>
365 lines
15 KiB
TypeScript
365 lines
15 KiB
TypeScript
import { createReadStream } from 'node:fs'
|
||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||
import { tmpdir } from 'node:os'
|
||
import { join } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { PassThrough, Readable } from 'node:stream'
|
||
import { describe, expect, it } from 'vitest'
|
||
import {
|
||
parseWxrStream,
|
||
WxrError,
|
||
WxrLimitError,
|
||
WxrParseError,
|
||
type WxrEvent,
|
||
type WxrItem,
|
||
} from '../src/parse/wxr-stream.js'
|
||
|
||
// mini.xml（Plan 1 夹具）：5 item（post 101 / product 111 / page 121 / attachment 201,202）
|
||
// + 2 term（31 category / 32 product_cat）；term 块在第三个 item 之后、attachment 之前。
|
||
const FIXTURE = fileURLToPath(new URL('../../../tests/fixtures/wxr/mini.xml', import.meta.url))
|
||
|
||
/** 整串一次性喂入（单 chunk 路径） */
|
||
function fromString(xml: string): Readable {
|
||
const s = new PassThrough()
|
||
s.end(xml)
|
||
return s
|
||
}
|
||
|
||
/** 按字节切块喂入（跨 chunk 的 UTF-8 多字节与 token 边界路径；7 字节必拆中文与 ]]>） */
|
||
function fromStringChunked(xml: string, size = 7): Readable {
|
||
return Readable.from(
|
||
(function* () {
|
||
const buf = Buffer.from(xml, 'utf8')
|
||
for (let i = 0; i < buf.length; i += size) yield buf.subarray(i, i + size)
|
||
})(),
|
||
)
|
||
}
|
||
|
||
async function collect(gen: AsyncGenerator<WxrEvent>): Promise<WxrEvent[]> {
|
||
const out: WxrEvent[] = []
|
||
for await (const ev of gen) out.push(ev)
|
||
return out
|
||
}
|
||
|
||
/** 捕获 reject 的错误实例（而非仅匹配），以便断言 kind / line / column 细节（照 php-unserialize 测试先例） */
|
||
async function captureRejection(p: Promise<unknown>): Promise<unknown> {
|
||
try {
|
||
await p
|
||
} catch (e) {
|
||
return e
|
||
}
|
||
throw new Error('期望抛出但未抛出')
|
||
}
|
||
|
||
function itemsOf(events: WxrEvent[]): WxrItem[] {
|
||
return events
|
||
.filter((ev): ev is Extract<WxrEvent, { kind: 'item' }> => ev.kind === 'item')
|
||
.map((ev) => ev.item)
|
||
}
|
||
|
||
/** 概要签名：item:<postId>:<postType> / term:<termId>:<taxonomy>（按文档序） */
|
||
function signature(events: WxrEvent[]): string[] {
|
||
return events.map((ev) =>
|
||
ev.kind === 'item'
|
||
? `item:${ev.item.postId}:${ev.item.postType}`
|
||
: `term:${ev.term.termId}:${ev.term.taxonomy}`,
|
||
)
|
||
}
|
||
|
||
describe('parseWxrStream：mini.xml 流式产出', () => {
|
||
it('产出顺序与文档一致：post → product → page → term → term → attachment → attachment', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
expect(signature(events)).toStrictEqual([
|
||
'item:101:post',
|
||
'item:111:product',
|
||
'item:121:page',
|
||
'term:31:category',
|
||
'term:32:product_cat',
|
||
'item:201:attachment',
|
||
'item:202:attachment',
|
||
])
|
||
})
|
||
|
||
it('跨 chunk 切块喂入（7 字节/块）产出与单 chunk 完全一致', async () => {
|
||
const { readFileSync } = await import('node:fs')
|
||
const xml = readFileSync(FIXTURE, 'utf8')
|
||
const events = await collect(parseWxrStream(fromStringChunked(xml)))
|
||
expect(signature(events)).toStrictEqual([
|
||
'item:101:post',
|
||
'item:111:product',
|
||
'item:121:page',
|
||
'term:31:category',
|
||
'term:32:product_cat',
|
||
'item:201:attachment',
|
||
'item:202:attachment',
|
||
])
|
||
})
|
||
|
||
it('item 事件字段完整：post 101 逐一断言具体值', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
const item = itemsOf(events).find((it) => it.postId === 101)
|
||
expect(item).toBeDefined()
|
||
expect(item).toMatchObject({
|
||
postId: 101,
|
||
postType: 'post',
|
||
status: 'publish',
|
||
postName: 'fixture-news-a',
|
||
link: 'https://fixture.example/news/fixture-news-a.html',
|
||
title: 'Fixture News A',
|
||
contentEncoded:
|
||
'<p>Body with <a href="/products/fixture-product/">link</a> and <img src="https://fixture.example/wp-content/uploads/2026/10/fixture-a.png"/></p>',
|
||
excerptEncoded: 'Fixture excerpt A',
|
||
postDate: '2026-10-05 09:00:00',
|
||
postModified: '2026-10-06 10:00:00',
|
||
attachmentUrl: '',
|
||
creator: 'editor',
|
||
})
|
||
})
|
||
|
||
it('attachment item：attachmentUrl 与 inherit 状态', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
const a201 = itemsOf(events).find((it) => it.postId === 201)
|
||
expect(a201).toMatchObject({
|
||
postType: 'attachment',
|
||
status: 'inherit',
|
||
attachmentUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png',
|
||
})
|
||
})
|
||
|
||
it('term 事件字段：term_id/taxonomy/slug/name（term 31）', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
const term = events.find((ev) => ev.kind === 'term' && ev.term.termId === 31)
|
||
expect(term).toBeDefined()
|
||
expect(term?.kind === 'term' && term.term).toMatchObject({
|
||
termId: 31,
|
||
taxonomy: 'category',
|
||
slug: 'company',
|
||
name: 'Company',
|
||
})
|
||
})
|
||
|
||
it('postmeta 聚合：post 101 恰 3 条（_thumbnail_id + Yoast title/metadesc）', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
const item = itemsOf(events).find((it) => it.postId === 101)
|
||
expect(item?.metas).toStrictEqual([
|
||
{ key: '_thumbnail_id', value: '201' },
|
||
{ key: '_yoast_wpseo_title', value: 'Fixture SEO Title A' },
|
||
{ key: '_yoast_wpseo_metadesc', value: 'Fixture SEO description A.' },
|
||
])
|
||
})
|
||
|
||
it('postmeta 聚合：product 111 恰 3 条（gallery 与 wppop_sections 原样字符串）', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
const item = itemsOf(events).find((it) => it.postId === 111)
|
||
expect(item?.metas).toStrictEqual([
|
||
{ key: '_thumbnail_id', value: '202' },
|
||
{ key: '_product_image_gallery', value: '203,204' },
|
||
{
|
||
key: 'wppop_sections',
|
||
value: 'a:1:{i:0;a:2:{s:5:"title";s:8:"SpecsTab";s:7:"content";s:10:"<p>Tab</p>";}}',
|
||
},
|
||
])
|
||
})
|
||
|
||
it('category 元素聚合进 item 事件：post 101 → category/company/Company；product 111 无', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
const post101 = itemsOf(events).find((it) => it.postId === 101)
|
||
expect(post101?.categories).toStrictEqual([{ domain: 'category', nicename: 'company', value: 'Company' }])
|
||
const product111 = itemsOf(events).find((it) => it.postId === 111)
|
||
expect(product111?.categories).toStrictEqual([])
|
||
})
|
||
|
||
it('rawLocation：wxr://item/<文档内序号从1起>，term 同理', async () => {
|
||
const events = await collect(parseWxrStream(createReadStream(FIXTURE)))
|
||
expect(itemsOf(events).map((it) => it.rawLocation)).toStrictEqual([
|
||
'wxr://item/1',
|
||
'wxr://item/2',
|
||
'wxr://item/3',
|
||
'wxr://item/4',
|
||
'wxr://item/5',
|
||
])
|
||
expect(
|
||
events
|
||
.filter((ev) => ev.kind === 'term')
|
||
.map((ev) => (ev.kind === 'term' ? ev.term.rawLocation : '')),
|
||
).toStrictEqual(['wxr://term/1', 'wxr://term/2'])
|
||
})
|
||
})
|
||
|
||
// 标题 CDATA 长 16 字符：供 maxCdataLength / maxFieldLength（=10）触发超限
|
||
const SMALL_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||
<rss version="2.0" xmlns:wp="http://wordpress.org/export/1.2/">
|
||
<channel>
|
||
<item><title><![CDATA[AAAAAAAAAAAAAAAA]]></title><wp:post_id>1</wp:post_id><wp:post_type>post</wp:post_type><wp:status>publish</wp:status></item>
|
||
<item><title><![CDATA[BBBBBBBBBBBBBBBB]]></title><wp:post_id>2</wp:post_id><wp:post_type>post</wp:post_type><wp:status>publish</wp:status></item>
|
||
<item><title><![CDATA[CCCCCCCCCCCCCCCC]]></title><wp:post_id>3</wp:post_id><wp:post_type>post</wp:post_type><wp:status>publish</wp:status></item>
|
||
</channel>
|
||
</rss>`
|
||
|
||
describe('parseWxrStream：限额防线（WxrLimitError，kind 区分）', () => {
|
||
it('maxItems=2：第 3 个 item 开始前抛 kind items', async () => {
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxItems: 2 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err).toBeInstanceOf(WxrError)
|
||
expect(err.kind).toBe('items')
|
||
})
|
||
|
||
it('maxCdataLength=10：超长 CDATA 抛 kind cdata', async () => {
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxCdataLength: 10 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('cdata')
|
||
})
|
||
|
||
it('maxFieldLength=10：单字段累计超限抛 kind fieldLength（防单个 title 灌爆内存）', async () => {
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxFieldLength: 10 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('fieldLength')
|
||
})
|
||
|
||
it('maxDepth=2：rss>channel>item 嵌套超限抛 kind depth', async () => {
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxDepth: 2 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('depth')
|
||
})
|
||
|
||
it('timeBudgetMs：可注入时钟超预算 → kind time', async () => {
|
||
let calls = 0
|
||
const now = () => (calls++ === 0 ? 0 : 1_000_000) // 第 1 次取起点，其后均"超预算"
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { timeBudgetMs: 120_000 }, now })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('time')
|
||
})
|
||
|
||
it('maxTermCount=1：第 2 个 term 抛 kind terms', async () => {
|
||
const xml = SMALL_XML.replace(
|
||
'</channel>',
|
||
'<wp:term><wp:term_id>1</wp:term_id><wp:term_taxonomy>category</wp:term_taxonomy><wp:term_slug>a</wp:term_slug><wp:term_name>A</wp:term_name></wp:term>' +
|
||
'<wp:term><wp:term_id>2</wp:term_id><wp:term_taxonomy>category</wp:term_taxonomy><wp:term_slug>b</wp:term_slug><wp:term_name>B</wp:term_name></wp:term></channel>',
|
||
)
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(xml), { limits: { maxTermCount: 1 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('terms')
|
||
})
|
||
|
||
it('maxWxrBytes：累计字节数超限抛 kind bytes', async () => {
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(SMALL_XML), { limits: { maxWxrBytes: 10 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('bytes')
|
||
})
|
||
|
||
it('maxMetaPerItem：单 item postmeta 数超限抛 kind metaPerItem', async () => {
|
||
const xml = SMALL_XML.replace(
|
||
'</item>',
|
||
'<wp:postmeta><wp:meta_key><![CDATA[k1]]></wp:meta_key><wp:meta_value><![CDATA[v1]]></wp:meta_value></wp:postmeta>' +
|
||
'<wp:postmeta><wp:meta_key><![CDATA[k2]]></wp:meta_key><wp:meta_value><![CDATA[v2]]></wp:meta_value></wp:postmeta></item>',
|
||
)
|
||
const err = (await captureRejection(
|
||
collect(parseWxrStream(fromString(xml), { limits: { maxMetaPerItem: 1 } })),
|
||
)) as WxrLimitError
|
||
expect(err).toBeInstanceOf(WxrLimitError)
|
||
expect(err.kind).toBe('metaPerItem')
|
||
})
|
||
})
|
||
|
||
describe('parseWxrStream：外部实体防线（不解析实体、不注入替换文本、不访问网络）', () => {
|
||
// 选定行为（钉死）：saxes 6 实体表固定为 XML 五预定义 + 数字字符引用，DOCTYPE
|
||
// 内部/SYSTEM 实体声明一律不注册——引用即 "undefined entity." 结构错误（WxrParseError），
|
||
// 替换文本绝不进入任何产出字段；saxes 为纯字符串解析器，外部 SYSTEM 标识符从不被抓取。
|
||
const SECRET = 'SECRET-REPLACEMENT-TEXT'
|
||
|
||
async function withTempEntityXml(
|
||
doctype: string,
|
||
body: string,
|
||
run: (path: string) => Promise<void>,
|
||
): Promise<void> {
|
||
const dir = await mkdtemp(join(tmpdir(), 'wxr-entity-'))
|
||
const path = join(dir, 'entity.xml')
|
||
await writeFile(
|
||
path,
|
||
`<?xml version="1.0" encoding="UTF-8"?>\n${doctype}\n<rss version="2.0" xmlns:wp="http://wordpress.org/export/1.2/"><channel>${body}</channel></rss>`,
|
||
'utf8',
|
||
)
|
||
try {
|
||
await run(path)
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true })
|
||
}
|
||
}
|
||
|
||
it('DOCTYPE 内部 ENTITY：引用处抛 WxrParseError，替换文本不进产出（临时文件 + createReadStream）', async () => {
|
||
await withTempEntityXml(
|
||
`<!DOCTYPE rss [<!ENTITY xxe "${SECRET}">]>`,
|
||
'<item><title>&xxe;</title><wp:post_id>1</wp:post_id><wp:post_type>post</wp:post_type></item>',
|
||
async (path) => {
|
||
const events: WxrEvent[] = []
|
||
const err = (await captureRejection(
|
||
(async () => {
|
||
for await (const ev of parseWxrStream(createReadStream(path))) events.push(ev)
|
||
})(),
|
||
)) as WxrParseError
|
||
expect(err).toBeInstanceOf(WxrParseError)
|
||
expect(err.message).toContain('undefined entity')
|
||
// 无论解析走到哪一步：替换文本绝不出现于任何已产出事件
|
||
expect(JSON.stringify(events)).not.toContain(SECRET)
|
||
},
|
||
)
|
||
})
|
||
|
||
it('SYSTEM 外部实体：同样拒绝（不解析、不抓取——无任何网络 I/O）', async () => {
|
||
await withTempEntityXml(
|
||
'<!DOCTYPE rss [<!ENTITY ext SYSTEM "file:///etc/passwd">]>',
|
||
'<item><title>&ext;</title><wp:post_id>1</wp:post_id><wp:post_type>post</wp:post_type></item>',
|
||
async (path) => {
|
||
const err = (await captureRejection(collect(parseWxrStream(createReadStream(path))))) as WxrParseError
|
||
expect(err).toBeInstanceOf(WxrParseError)
|
||
expect(err.message).toContain('undefined entity')
|
||
},
|
||
)
|
||
})
|
||
|
||
it('预定义实体不受影响：& / 数字字符引用正常解析', async () => {
|
||
const xml =
|
||
'<?xml version="1.0"?><rss version="2.0" xmlns:wp="http://wordpress.org/export/1.2/"><channel>' +
|
||
'<item><title>plain</title><wp:post_name>a&bA</wp:post_name><wp:post_id>7</wp:post_id><wp:post_type>post</wp:post_type></item>' +
|
||
'</channel></rss>'
|
||
const events = await collect(parseWxrStream(fromString(xml)))
|
||
expect(itemsOf(events)[0]?.postName).toBe('a&bA')
|
||
})
|
||
})
|
||
|
||
describe('parseWxrStream：畸形 XML → WxrParseError（带 saxes 行列位置）', () => {
|
||
it('提前闭合：精确 line/column（4:17）与原始信息', async () => {
|
||
const xml =
|
||
'<?xml version="1.0"?>\n<rss><channel>\n<item><title>a</title></item>\n</channel></item></rss>'
|
||
const err = (await captureRejection(collect(parseWxrStream(fromString(xml))))) as WxrParseError
|
||
expect(err).toBeInstanceOf(WxrParseError)
|
||
expect(err).toBeInstanceOf(WxrError)
|
||
expect(err.line).toBe(4)
|
||
expect(err.column).toBe(17)
|
||
expect(err.message).toContain('unexpected close tag')
|
||
})
|
||
|
||
it('未闭合（截断）文件：close 阶段报错，行列为有效正数', async () => {
|
||
const xml = '<?xml version="1.0"?>\n<rss><channel>\n<item><title>a</title></item>\n</channel>'
|
||
const err = (await captureRejection(collect(parseWxrStream(fromString(xml))))) as WxrParseError
|
||
expect(err).toBeInstanceOf(WxrParseError)
|
||
expect(err.line).toBeGreaterThanOrEqual(1)
|
||
expect(err.column).toBeGreaterThanOrEqual(0)
|
||
expect(err.message).toContain('unclosed')
|
||
})
|
||
})
|