From bb2d38130886315db29708ffe54a18f975cf9d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Wed, 21 Oct 2026 19:40:35 +0800 Subject: [PATCH] =?UTF-8?q?fix(import):=20finish=20=E5=9F=9F=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E5=AE=89=E5=85=A8=E5=AE=88=E5=8D=AB=EF=BC=9Battachmen?= =?UTF-8?q?t=20=3Fp=3D=20=E6=8E=92=E9=99=A4=E5=AF=B9=E9=BD=90=EF=BC=9B?= =?UTF-8?q?=E6=AD=A7=E4=B9=89/=E9=99=8D=E7=BA=A7=E8=AE=A1=E6=95=B0?= =?UTF-8?q?=E7=B2=BE=E5=8C=96=EF=BC=9Bpermalink=20=E5=BD=A2=E6=80=81?= =?UTF-8?q?=E8=A6=86=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - isSafePath 扩至 finish 域②（分类页 public_base_path+slug）与域④（media_assets public_path）——WXR 原始 slug 与库表原样切片同携穿越向量，拒绝即 issue+不登记 - writeAttachmentRoute 与内容行同口径排除 ?p=/&page_id=（?attachment_id= 不在排除表，照常 query_key 建模），排除计数 routesExcludedQueries - 多资产歧义消息改"命中 ≥2 行"（LIMIT 2 探针只知下界） - 降级汇总去虚报：writeRoute 返回四态，downgraded 只计实际写入（created/updated）——重复 attachment 事件 no-op 不重复计数 - foldCollisionKey JSDoc 钉死：折叠仅 JS 侧计算落列，勿用 PG normalize 库侧重算（ICU 版本差异面） - 测试补 permalink 形态 attachment 301（纯路径行 query_key NULL）+ 重复事件降级计数"1 条"断言 + 双 finish 域安全守卫拒绝用例（317 全绿） Co-Authored-By: Claude Code --- apps/api/src/modules/import/route-writer.ts | 95 +++++++---- tests/integration/import-routes.test.ts | 169 +++++++++++++++++++- 2 files changed, 231 insertions(+), 33 deletions(-) diff --git a/apps/api/src/modules/import/route-writer.ts b/apps/api/src/modules/import/route-writer.ts index 50b62de..cbfdf2c 100644 --- a/apps/api/src/modules/import/route-writer.ts +++ b/apps/api/src/modules/import/route-writer.ts @@ -31,7 +31,9 @@ // route_kind 六值（html/asset/redirect/gone/dynamic/unresolved）**无 excluded 值**——查询 // 路由本身是 SP3 能力（§6.1 明示"登记不实现"）。故本写入器对含 ?p=/&page_id= 的来源 URL // **不插 route_registry 行**，以 counts.routesExcludedQueries + 逐条 issue 落报告 -// （excluded_with_reason 的报告语义），行级登记随 SP3 查询路由能力一起落地。 +// （excluded_with_reason 的报告语义），行级登记随 SP3 查询路由能力一起落地。内容行与 +// attachment link **同口径排除**（?attachment_id= 不在排除表——attachment 页处置是 §6.1 +// 明示的 301 行，照常 query_key 建模）。 // // exact/collision 双键（ADR-007）：exact_route_key 保留大小写/尾斜杠语义（/About 与 /about // 两行并存）；collision_key 折叠（前置事实 9 钉死：NFC + lowercase + 去尾斜杠）只作风险 @@ -79,7 +81,8 @@ export const DEFAULT_NEWS_PATH = '/news/' /** * collision_key 折叠函数（前置事实 9 钉死）：NFC + lowercase + 去尾斜杠（根路径 '/' 折为 ''）。 - * 只用于风险检测，永不参与唯一约束与请求匹配（ADR-007）。 + * 只用于风险检测，永不参与唯一约束与请求匹配（ADR-007）。折叠仅在本写入器 JS 侧计算落列 + * ——勿用 PG normalize 在库侧重算（ICU 版本差异会致折叠结果漂移、预警失真）。 */ export function foldCollisionKey(path: string): string { return path.normalize('NFC').toLowerCase().replace(/\/+$/, '') @@ -136,7 +139,7 @@ interface RouteWriterCounts { routesRedirects: number /** collision_key 折叠碰撞预警次数（不计失败——两行并存是 ADR-007 语义） */ routesCollisions: number - /** ?p=/&page_id= 查询形态排除数（不插行——excluded_with_reason 的报告落点） */ + /** ?p=/&page_id= 查询形态排除数（内容行与 attachment link 同口径，不插行——excluded_with_reason 的报告落点） */ routesExcludedQueries: number } @@ -229,8 +232,11 @@ export function createRouteEventHandler(identity: { } /** - * 路径安全防线（最小集）：NUL/反斜杠/控制字符/穿越段拒绝。完整校验规则（RFC 非保留字符 - * 解码、Windows 保留设备名、尾部空格点、超长路径——spec §5.6）是 SP3 验证器职责。 + * 路径安全防线（最小集）：NUL/反斜杠/控制字符/穿越段拒绝——**全部四个写入域统一经过** + * （内容 permalink / attachment link / finish 期分类页 / finish 期 asset public_path： + * 后两者输入是 media-writer 的原样切片与 WXR 原始 slug，同携真实穿越向量）。完整校验 + * 规则（RFC 非保留字符解码、Windows 保留设备名、尾部空格点、超长路径——spec §5.6） + * 是 SP3 验证器职责。 */ function isSafePath(path: string): boolean { if (path.includes('\0') || path.includes('\\')) return false @@ -267,11 +273,15 @@ export function createRouteEventHandler(identity: { return rows[0]?.contentEntryId ?? null } + /** writeRoute 的四态产出（降级计数只认实际写入——'created'/'updated'） */ + type WriteOutcome = 'created' | 'updated' | 'noop' | 'conflict' + /** * 单行写入（三分 upsert，见模块头）：碰撞预警 → exact 查 → miss INSERT / 同 owner 比对 - * UPDATE / 异 owner 冲突 issue 保留先到。 + * UPDATE / 异 owner 冲突 issue 保留先到。返回四态（调用方按需消费——如 attachment 降级 + * 计数只数实际写入，重复事件 no-op 不重复计数）。 */ - async function writeRoute(p: RoutePayload): Promise { + async function writeRoute(p: RoutePayload): Promise { const collisionKey = foldCollisionKey(p.routePath) // 碰撞预警（ADR-007：折叠同键而 exact 不同——两行并存，只预警不拦截） @@ -338,7 +348,7 @@ export function createRouteEventHandler(identity: { }) counts.routesCreated++ if (p.routeKind === 'redirect') counts.routesRedirects++ - return + return 'created' } // owner 判据三元组（见模块头 upsert 注记） @@ -352,7 +362,7 @@ export function createRouteEventHandler(identity: { issues.push( `路由 exact_key 冲突（${p.host}${p.routePath}${queryPart}）：保留先到（${existing.routeKind} → ${existingTarget}）——跳过后到（${p.label}，${p.routeKind}）——不静默覆盖`, ) - return + return 'conflict' } const set: Partial = {} if (existing.sourcePathRaw !== p.sourcePathRaw) set.sourcePathRaw = p.sourcePathRaw @@ -367,8 +377,10 @@ export function createRouteEventHandler(identity: { set.updatedAt = new Date() await db.update(routeRegistry).set(set).where(eq(routeRegistry.id, existing.id)) counts.routesUpdated++ + return 'updated' } // 全等：幂等 no-op（重跑零计数） + return 'noop' } /** @@ -429,8 +441,36 @@ export function createRouteEventHandler(identity: { }) } - /** finish 期单条 attachment 处理：资产反查 → 301 载荷（查询/permalink 双形态） */ + /** + * finish 期单条 attachment 处理：URL 层防线（link 在场 → 解析 → ?p=/&page_id= 排除 → 路径 + * 安全）先行，资产反查在后 → 301 载荷（查询/permalink 双形态）。返回 true 仅当实际写入 + * （created/updated——重复事件 no-op 不重复计入降级汇总）。 + */ async function writeAttachmentRoute(att: PendingAttachment, host: string): Promise { + if (att.sourceUrl === '') { + issues.push(`attachment 缺 link（wp:attachment/${att.sourceId}，${att.rawLocation}）：无页面 URL 证据——301 路由不登记`) + return false + } + let parsed: URL + try { + parsed = new URL(att.sourceUrl) + } catch (err) { + issues.push(`attachment link 解析失败（wp:attachment/${att.sourceId}，${att.sourceUrl}）：${err instanceof Error ? err.message : String(err)}——301 路由不登记`) + return false + } + // ?p=/&page_id= 排除口径与内容行一致（Fix 对齐：attachment link 同形态同排除—— + // excluded_with_reason 语义见模块头；?attachment_id= 不在排除表，照常 query_key 建模） + for (const param of EXCLUDED_QUERY_PARAMS) { + if (parsed.searchParams.has(param)) { + counts.routesExcludedQueries++ + issues.push(`查询 URL 排除（wp:attachment/${att.sourceId}，${att.sourceUrl}）：?${param}= 形态不登记 route_registry（route_kind 无 excluded 值——查询路由 SP3；spec §6.1/§6.2 excluded_with_reason 以计数+本 issue 落报告）`) + return false + } + } + if (!isSafePath(parsed.pathname)) { + issues.push(`attachment 路径安全校验拒绝（wp:attachment/${att.sourceId}，${parsed.pathname}）：含 NUL/反斜杠/控制字符/穿越段——301 路由不登记（完整校验规则 SP3 验证器）`) + return false + } const assetRows = await db .select({ publicPath: mediaAssets.publicPath }) .from(mediaAssets) @@ -444,31 +484,17 @@ export function createRouteEventHandler(identity: { .orderBy(mediaAssets.sourceUrl) .limit(2) if (assetRows.length === 0) { - issues.push(`attachment 301 目标缺席（wp:attachment/${att.sourceId}，${att.sourceUrl || att.rawLocation}）：media_assets 无对应资产（媒体三级对拍 miss？）——attachment 页路由未登记`) + issues.push(`attachment 301 目标缺席（wp:attachment/${att.sourceId}，${att.sourceUrl}）：media_assets 无对应资产（媒体三级对拍 miss？）——attachment 页路由未登记`) return false } if (assetRows.length > 1) { - issues.push(`attachment 多资产歧义（wp:attachment/${att.sourceId}）：同 source_id 命中 ${assetRows.length} 行——不自动选（spec §5.5），attachment 页路由未登记`) + // LIMIT 2 探针只知下界——歧义消息按 ≥2 表述（不自动选，spec §5.5） + issues.push(`attachment 多资产歧义（wp:attachment/${att.sourceId}）：同 source_id 命中 ≥2 行——不自动选（spec §5.5），attachment 页路由未登记`) return false } const publicPath = assetRows[0]!.publicPath - if (att.sourceUrl === '') { - issues.push(`attachment 缺 link（wp:attachment/${att.sourceId}，${att.rawLocation}）：无页面 URL 证据——301 路由不登记`) - return false - } - let parsed: URL - try { - parsed = new URL(att.sourceUrl) - } catch (err) { - issues.push(`attachment link 解析失败（wp:attachment/${att.sourceId}，${att.sourceUrl}）：${err instanceof Error ? err.message : String(err)}——301 路由不登记`) - return false - } - if (!isSafePath(parsed.pathname)) { - issues.push(`attachment 路径安全校验拒绝（wp:attachment/${att.sourceId}，${parsed.pathname}）——301 路由不登记`) - return false - } const queryKey = canonicalQueryKey(parsed) - await writeRoute({ + const outcome = await writeRoute({ label: `attachment ${att.sourceId}`, host, sourcePathRaw: att.sourceUrl, @@ -487,7 +513,7 @@ export function createRouteEventHandler(identity: { priority: 0, verificationStatus: 'verified_redirect', }) - return true + return outcome === 'created' || outcome === 'updated' } return { @@ -566,6 +592,11 @@ export function createRouteEventHandler(identity: { const disposition = TERM_ROUTE_DISPOSITIONS[t.taxonomyKey]! const withSlug = `${t.basePath}${t.slug}` const pagePath = withSlug.endsWith('/') ? withSlug : `${withSlug}/` + // 安全防线与内容 permalink 同口径：slug 是 WXR 原始串（穿越向量真实存在） + if (!isSafePath(pagePath)) { + issues.push(`分类页路径安全校验拒绝（${t.taxonomyKey}/${t.slug}，${pagePath}）：含 NUL/反斜杠/控制字符/穿越段——路由不登记（完整校验规则 SP3 验证器）`) + continue + } if (disposition.kind === 'list') { await writeRoute({ label: `分类页 ${t.taxonomyKey}/${t.slug}`, @@ -622,6 +653,12 @@ export function createRouteEventHandler(identity: { .where(eq(mediaAssets.siteId, siteId)) .orderBy(mediaAssets.publicPath) for (const a of assetRows) { + // 安全防线与内容 permalink 同口径：public_path 是 media-writer 对来源 URL 的原样切片 + //（相对路径保留 .. 段——穿越向量真实存在），不因来源是库表而豁免 + if (!isSafePath(a.publicPath)) { + issues.push(`媒体路径安全校验拒绝（media_assets public_path=${a.publicPath}）：含 NUL/反斜杠/控制字符/穿越段——asset 路由不登记（完整校验规则 SP3 验证器）`) + continue + } await writeRoute({ label: `媒体 asset ${a.publicPath}`, host, diff --git a/tests/integration/import-routes.test.ts b/tests/integration/import-routes.test.ts index cd1d6bb..42efdfd 100644 --- a/tests/integration/import-routes.test.ts +++ b/tests/integration/import-routes.test.ts @@ -80,6 +80,24 @@ function contentEvent(spec: { sourceId: string; url: string; sourceType?: string return { kind: 'row', row, issues: [], sanitizeDiffs: [], changed: false } } +/** 手构 attachment row 事件（sourceUrl = 原串：查询/permalink 形态的判别输入） */ +function attachmentEvent(spec: { sourceId: string; url: string }): AdapterEvent { + const row: UniversalRow = { + sourceType: 'wp:attachment', + sourceId: spec.sourceId, + status: 'inherit', + locale: 'en', + taxonomyRefs: [], + attachmentRefs: [], + seo: {}, + knownFields: {}, + extras: {}, + rawLocation: `wxr://item/${spec.sourceId}`, + } + if (spec.url !== '') row.sourceUrl = spec.url + return { kind: 'row', row, issues: [], sanitizeDiffs: [], changed: false } +} + /** 手构 wp:term 事件（分类页/301 在 finish 期由 terms 表生成——term-writer 先物化） */ function termEvent(spec: { termId: number; taxonomy: string; slug: string; name: string }): AdapterEvent { return { @@ -205,6 +223,25 @@ function termRoutePipeline(paths?: { newsPath?: string }): ImportPipeline { } } +/** route 单写入器（手构 attachment/直插 media_assets 场景——站点页+asset 行仍在 finish 生成） */ +function routeOnlyPipeline(): ImportPipeline { + return async (ctx) => { + const route = routeHandlerOf(ctx) + for await (const ev of ctx.events) { + await route.handleEvent(ev) + } + return route.finish() + } +} + +/** 直插 media_assets 行（route finish 域④ 的输入——不经 media-writer 三级对拍） */ +async function insertAssetRow(db: Db, siteId: string, connectionId: string, spec: { sourceId: string; sourceUrl: string; publicPath: string }): Promise { + await db.execute(`insert into media_assets (site_id, source_connection_id, source_id, source_url, + source_path, storage_path, public_path, sha256, detected_mime, byte_size, status) + values ('${siteId}', '${connectionId}', '${spec.sourceId}', '${spec.sourceUrl}', + 'fixture/source-path', 'fixture/storage-path', '${spec.publicPath}', 'fake-sha256', 'image/png', 1, 'ready')`) +} + /** 本文件写入的行清场：FK 反序（route_registry 先于 source_snapshots/sites；两张不可变表走触发器开关） */ async function cleanupSiteRows(db: Db, siteId: string): Promise { await db.execute(`delete from route_registry where site_id = '${siteId}'`) @@ -486,13 +523,15 @@ describe('路由注册表写入器（§6.1 处置全表 + ADR-007 双键）', () }) }) - it('?p=/&page_id= 查询形态：不插 route_registry 行（查询路由 SP3），routesExcludedQueries 计数 + issue 说明', async () => { + it('?p=/&page_id= 查询形态（内容行与 attachment link 同口径）：不插 route_registry 行、routesExcludedQueries 计数 + issue', async () => { await withTestDb(async (db) => { const work = await mkdtemp(join(tmpdir(), 'sf-routes-qexc-')) try { const events: AdapterEvent[] = [ contentEvent({ sourceId: '401', url: 'https://h.example/?p=999' }), contentEvent({ sourceId: '402', url: 'https://h.example/?page_id=888' }), + // attachment link 的 ?p= 形态同排除（?attachment_id= 不在排除表——见下用例） + attachmentEvent({ sourceId: '501', url: 'https://h.example/?p=123' }), ] const { key, reg, wxr } = await registerMini(db, work) try { @@ -513,18 +552,20 @@ describe('路由注册表写入器（§6.1 处置全表 + ADR-007 双键）', () expect(report.counts).toStrictEqual({ contentEntries: 2, revisionsCreated: 2, - sourceRecords: 2, + // attachment 行经 content-writer 落 record-only 足迹（前置事实 11） + sourceRecords: 3, skipped: 0, routesCreated: 3, routesUpdated: 0, routesRedirects: 0, routesCollisions: 0, - routesExcludedQueries: 2, + routesExcludedQueries: 3, }) const excluded = report.issues.filter((i) => i.includes('查询 URL 排除')) - expect(excluded).toHaveLength(2) + expect(excluded).toHaveLength(3) expect(excluded.some((i) => i.includes('p=999'))).toBe(true) expect(excluded.some((i) => i.includes('page_id=888'))).toBe(true) + expect(excluded.some((i) => i.includes('p=123'))).toBe(true) } finally { await cleanupSiteRows(db, reg.siteId) } @@ -673,4 +714,124 @@ describe('路由注册表写入器（§6.1 处置全表 + ADR-007 双键）', () } }) }) + + it('attachment permalink 形态 301（纯路径行）+ 同事件重复：降级汇总只计实际写入（1 条不虚报 2 条）', async () => { + await withTestDb(async (db) => { + const work = await mkdtemp(join(tmpdir(), 'sf-routes-perm-')) + try { + const { key, reg, wxr } = await registerMini(db, work) + const connRow = (await db.execute(`select source_connection_id from source_snapshots + where id = '${wxr.id}'`)) as any + // 直插资产行（finish 域④ 反查输入——不经 media-writer 三级对拍） + await insertAssetRow(db, reg.siteId, connRow[0].source_connection_id, { + sourceId: '502', + sourceUrl: 'https://fixture.example/wp-content/uploads/2026/10/extra.png', + publicPath: '/wp-content/uploads/2026/10/extra.png', + }) + // permalink 形态 attachment 页（无查询串）+ 同事件重复（WXR 文档内可重复出现） + const att = attachmentEvent({ sourceId: '502', url: 'https://fixture.example/news/fixture-att-page/' }) + try { + const result = await createImportJob(db, { + siteKey: key, + snapshotId: wxr.id, + reportRoot: join(work, 'artifacts', 'import'), + events: eventsOf([att, att]), + pipeline: routeOnlyPipeline(), + }) + expect(result.status).toBe('succeeded_with_issues') + const rows = await routesOf(db, reg.siteId) + // 3 站点页 + 1 permalink 形态 301（纯路径行 query_key NULL）+ 1 asset = 5 + expect(rows).toHaveLength(5) + expect(rows.find((r) => r.path === '/news/fixture-att-page/')).toStrictEqual({ + path: '/news/fixture-att-page/', + q: null, + kind: 'redirect', + tStatus: '301', + rStatus: 301, + loc: '/wp-content/uploads/2026/10/extra.png', + file: '/wp-content/uploads/2026/10/extra.png', + idx: false, + pri: 0, + ver: 'verified_redirect', + entry: null, + }) + expect(rows.find((r) => r.path === '/wp-content/uploads/2026/10/extra.png')!.kind).toBe('asset') + + const report = await readReport(work, result.jobId) + expect(report.counts).toStrictEqual({ + routesCreated: 5, + routesUpdated: 0, + routesRedirects: 1, + routesCollisions: 0, + routesExcludedQueries: 0, + }) + // 降级汇总只计实际写入：重复事件 no-op 不重复计数（1 条，不虚报 2 条） + const downgrade = report.issues.filter((i) => i.includes('attachment 页降级')) + expect(downgrade).toHaveLength(1) + expect(downgrade[0]).toContain('1 条') + expect(report.issues).toHaveLength(1) + } finally { + await cleanupSiteRows(db, reg.siteId) + } + } finally { + await rm(work, { recursive: true, force: true }) + } + }) + }) + + it('finish 域路径安全守卫：term slug 穿越段与 media public_path 穿越段拒绝登记 + issue 如实报告', async () => { + await withTestDb(async (db) => { + const work = await mkdtemp(join(tmpdir(), 'sf-routes-guard-')) + try { + const { key, reg, wxr } = await registerMini(db, work) + const connRow = (await db.execute(`select source_connection_id from source_snapshots + where id = '${wxr.id}'`)) as any + // 直插带穿越段的 public_path（media-writer 对来源 URL 原样切片的形态） + await insertAssetRow(db, reg.siteId, connRow[0].source_connection_id, { + sourceId: '503', + sourceUrl: 'https://fixture.example/wp-content/uploads/../secret.png', + publicPath: '/wp-content/uploads/../secret.png', + }) + const events: AdapterEvent[] = [ + termEvent({ termId: 51, taxonomy: 'category', slug: '../../../evil', name: 'Evil' }), + ] + try { + const result = await createImportJob(db, { + siteKey: key, + snapshotId: wxr.id, + reportRoot: join(work, 'artifacts', 'import'), + events: eventsOf(events), + pipeline: termRoutePipeline(), + }) + expect(result.status).toBe('succeeded_with_issues') + const rows = await routesOf(db, reg.siteId) + // 仅 3 条站点页：穿越段分类页与穿越段 asset 行均被守卫拒绝 + expect(rows).toHaveLength(3) + expect(rows.every((r) => r.path === '/' || r.path === '/products/' || r.path === '/news/')).toBe(true) + + const report = await readReport(work, result.jobId) + expect(report.counts).toStrictEqual({ + termsProcessed: 1, + autoCreatedTerms: 0, + contentTermsCreated: 0, + routesCreated: 3, + routesUpdated: 0, + routesRedirects: 0, + routesCollisions: 0, + routesExcludedQueries: 0, + }) + const termGuard = report.issues.filter((i) => i.includes('分类页路径安全校验拒绝')) + expect(termGuard).toHaveLength(1) + expect(termGuard[0]).toContain('/category/../../../evil') + const assetGuard = report.issues.filter((i) => i.includes('媒体路径安全校验拒绝')) + expect(assetGuard).toHaveLength(1) + expect(assetGuard[0]).toContain('..') + } finally { + await cleanupSiteRows(db, reg.siteId) + } + } finally { + await rm(work, { recursive: true, force: true }) + } + }) + }) })