feat(import): 媒体登记与安全校验

Plan 3 Task 4:media-writer(attachment 行三级对拍 + 内容行引用提取双事件消费)
+ magic-bytes 手写签名表(零新增依赖)。

- 三级对拍(前置事实 7):①uploads 相对路径清单精确命中 ②侧车 source_url→
  file_path→清单实体(精确 miss 按文件名候选,多候选歧义不自动选全列 issue)
  ③missing 只进报告不插行(sha256 NOT NULL 库层防线)
- 魔数校验:命中实体读存储副本头部实际字节——SVG 检出 status=quarantined 不进
  产物语义(引用不建 usage);未知签名 unsupported;扩展名声明 MIME 与检测不符
  → issue(按检测登记)
- usage 两阶段时序(WXR 文档序 items 先于 attachment):内容行提取引用 + 修订
  锚点(依赖 content-writer 先处理同事件——Task 6 编排顺序契约),finish 统一
  对 media_assets 反查(id→source_id / URL→source_url→public_path),顺序无关
- 悬空引用计数 mediaDanglingRefs 不静默(mini 203/204 钉死)
- counts 六键 media 前缀独占(勿复用 'skipped' 的键契约注记)
- 幂等:(site,source_url) 定向 onConflictDoNothing + usage 复合主键——重跑零增行

308 测试(296+6 单测+6 集成);typecheck/lint 三绿。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-10-21 18:38:10 +08:00
parent 99c07906cd
commit ed081ec477
4 changed files with 1375 additions and 0 deletions
@@ -0,0 +1,49 @@
// 手写魔数签名表(Plan 3 Task 4,spec §11.1、计划"Tech Stack:零新增依赖"):
// 常见六类媒体格式判别 + SVG 文本特征检出。刻意不引 file-type 依赖——导入管道的
// 安全决策只须回答两问:"是什么格式"与"是否 SVG(隔离)",一张可审计的小签名表
// 足够,且避免为一次头部读取引入整个 MIME 数据库。
//
// 判别语义(detectMediaType):
// - 命中二进制签名 → {type: MIME, svg: false}(PNG/JPEG/GIF87a/GIF89a/WEBP/PDF);
// - 无二进制命中但前 1KB 文本窗口含 "<svg"(大小写不敏感)→ {type: 'image/svg+xml',
// svg: true}——调用方据此 status='quarantined'(spec §11.1:SVG 可携带脚本与外部
// 引用,不进产物语义);
// - 均未命中 → null(未知签名——调用方 status='unsupported' + issue,不猜格式)。
//
// 顺序纪律:二进制签名先于 SVG 文本扫描——PNG/JPEG 正例即便尾部字节碰巧含 "<svg"
// 片段也按二进制格式判(文本窗口只对非二进制文件有意义)。窗口 1KB 是钉死的折中:
// 覆盖常规 "<?xml …?><svg" 前导(声明/注释/doctype),不给深藏脚本的巨型前缀放行。
//
// Buffer 读取口径:latin1 单字节解码后小写化做包含检测——不做 UTF-8 多字节解码,
// 避免截断的字节序列被解码成意外字符。
/** 二进制签名表(前缀匹配;WEBP 的 RIFF+WEBP@8 双段结构在表外单查) */
const SIGNATURES: ReadonlyArray<{ mime: string; bytes: Buffer }> = Object.freeze([
{ mime: 'image/png', bytes: Buffer.from([0x89, 0x50, 0x4e, 0x47]) },
{ mime: 'image/jpeg', bytes: Buffer.from([0xff, 0xd8, 0xff]) },
{ mime: 'image/gif', bytes: Buffer.from('GIF87a', 'latin1') },
{ mime: 'image/gif', bytes: Buffer.from('GIF89a', 'latin1') },
{ mime: 'application/pdf', bytes: Buffer.from('%PDF', 'latin1') },
])
const RIFF = Buffer.from('RIFF', 'latin1')
const WEBP = Buffer.from('WEBP', 'latin1')
const SVG_WINDOW_BYTES = 1024
/**
* 魔数检测:读文件头部字节判别媒体类型。返回 null = 未知签名(调用方按
* unsupported 处置);svg=true 仅出现在 type='image/svg+xml'(文本特征命中)。
*/
export function detectMediaType(buf: Buffer): { type: string; svg: boolean } | null {
for (const sig of SIGNATURES) {
if (buf.length >= sig.bytes.length && buf.subarray(0, sig.bytes.length).equals(sig.bytes)) {
return { type: sig.mime, svg: false }
}
}
// WEBP:RIFF 容器头(4B 长度)+ 'WEBP' Magic(@8)——RIFF 家族其他成员(WAV/AVI)不误报
if (buf.length >= 12 && buf.subarray(0, 4).equals(RIFF) && buf.subarray(8, 12).equals(WEBP)) {
return { type: 'image/webp', svg: false }
}
const head = buf.subarray(0, Math.min(buf.length, SVG_WINDOW_BYTES)).toString('latin1').toLowerCase()
if (head.includes('<svg')) return { type: 'image/svg+xml', svg: true }
return null
}
+617
View File
@@ -0,0 +1,617 @@
// 媒体登记写入器(Plan 3 Task 4,spec §5.5/§11.1、前置事实 7/11、评审 Issue 1/3):
// 消费适配器事件流的**双事件**面——attachment 行建资产(media_assets)、内容行提取引用
// (media_usages)——媒体域的领域 owner(Task 6 分派规则:content-writer 对 attachment 行
// 只留 source_records 足迹,media_assets 的唯一写入方在此)。
//
// 三级对拍(前置事实 7 钉死,真实数据实证:媒体目录扁平 GUID 命名、与 WXR
// attachment_url 零精确命中,唯一映射在旧 cms 库侧车):
// ① uploads 相对路径清单精确命中:attachment_url 掐掉
// '…/wp-content/uploads/' 前缀 → manifest[path](清单 = media_bundle 快照 metadata.
// manifest,{path, bytes, sha256});
// ② 侧车命中:manifest miss → sidecarMap[source_url] → filePath → 清单实体(精确路径
// miss 时按**文件名**候选——真实侧车 file_path 是扁平文件名而清单可含子目录;候选
// 多条 = 歧义,不自动选、全部列 issue、不插行,spec §5.5);
// ③ missing:两级皆 miss → 只进报告(**不插 media_assets 行**——sha256 NOT NULL 无清单
// 实体可取,行插不进是库层防线不是巧合)。
// 命中行取值:public_path = 旧 uploads 路径(保旧 URL 语义)、sha256/byte_size 从清单、
// storage_path = bundle 副本相对路径 + 清单 path(绝对副本 = metadata.storageRoot + 此列)。
//
// 魔数安全校验(spec §11.1):命中实体读存储副本头部实际字节(detectMediaType)——
// - SVG 检出(文本特征)→ status='quarantined':行存在(登记可审计)但不进产物语义,
// 其引用不建 media_usages(usage 是发布面消费的引用关系,隔离资产不得经此进产物);
// - 未知签名 → status='unsupported' + issue(不猜格式);
// - 声明 MIME(清单/WXR 均无 MIME——声明侧由 URL 扩展名推断)与检测不符 → issue
// (仍登记 ready:内容真实、扩展名说谎是数据质量问题不是安全风险)。
//
// usage 时序(设计决定,WXR 文档序钉死):items 先于 attachment——引用先行、资产后到,
// 故 usage 建立分两阶段:内容行到达时提取引用并落修订锚点(source_records 六列身份 →
// entry_locales.current_revision_id——**依赖 content-writer 先处理同事件**,Task 6 编排
// 顺序契约),资产解析统一推迟到 finish() 对 media_assets 反查(id 反查 source_id /
// URL 反查 source_url→public_path)——两种文档序终态一致。引用无对应资产且属本站媒体域
// (id 引用或 uploads 路径 URL)→ mediaDanglingRefs 计数 + issue(§7.6 禁静默);外域
// 热链 URL 不属本域对拍范围,不计悬空(无清单可比对,静默越过)。
//
// 幂等:media_assets 唯一 (site, source_url) onConflictDoNothing(重跑零增行);
// media_usages 复合主键 (asset, revision, field_path) onConflictDoNothing——field_path
// 由提取序确定(featured/gallery 序号/正文 img 序号),重跑同流同值。事务边界:全部
// 单语句原子操作(ctx.db 直连约定,无跨语句不变量——与 term-writer 同纪律)。
import { open, readFile } from 'node:fs/promises'
import { join } from 'node:path'
import { and, eq } from 'drizzle-orm'
import type { AdapterEvent } from '@sf/adapter-wp'
import type { UniversalRow } from '@sf/contracts'
import { entryLocales, mediaAssets, mediaUsages, sourceRecords } from '../../db/schema.js'
import { isUniqueViolation, type MediaManifestEntry } from '../source/register.service.js'
import { detectMediaType } from './magic-bytes.js'
import type { ImportDb, ImportJobContext, ImportPipeline, ImportPipelineOutcome } from './import.service.js'
const UPLOADS_MARKER = '/wp-content/uploads/'
/** 未知签名的 detected_mime 落值(列 NOT NULL——不猜格式但要有确定值) */
const UNKNOWN_DETECTED_MIME = 'application/octet-stream'
/** 魔数读取的头部字节(SVG 文本窗口 1KB + 余量) */
const HEAD_BYTES = 4096
/** URL 扩展名 → 声明 MIME(声明侧推断表——与 detectMediaType 的检出表独立) */
const EXT_MIME: Readonly<Record<string, string>> = Object.freeze({
png: 'image/png',
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
gif: 'image/gif',
webp: 'image/webp',
pdf: 'application/pdf',
svg: 'image/svg+xml',
})
/**
* 可分派的媒体写入器句柄(Task 6 编排形状,与 term/content 同构——但 finish 是异步:
* usage 解析推迟到流终,见模块头"usage 时序")。句柄非可重入:一个句柄只服务一遍流。
*/
export interface MediaEventHandler {
/** 单事件分派(term/unsupported/error 为 no-op——无媒体信息) */
handleEvent(ev: AdapterEvent): Promise<void>
/** 流终:解析悬置引用 → media_usages / 悬空报告,取回累计产出(调用后句柄弃用) */
finish(): Promise<ImportPipelineOutcome>
}
/**
* 本写入器的 counts 键与归属(**counts 键是跨写入器的扁平全局命名空间**——import.service
* 按返回值覆盖累加器同名键合并,各写入器必须各持独占键名,见 ImportPipelineOutcome.counts
* 的键契约注记)。本写入器独占六键(media 前缀,**勿复用 content-writer 的 'skipped'**):
* - mediaRegistered:本运行**新插**的 media_assets 行数(含 quarantined/unsupported——
* 行已登记;幂等重跑为 0);
* - mediaFromSidecar:其中经级②侧车解析的行数(mediaRegistered 的子集);
* - mediaMissing:级③ missing 的 attachment 数(按事件计——重跑如实再现);
* - mediaQuarantined:本运行检出 SVG 的 attachment 数(按事件计——重跑如实再现,
* 与 mediaRegistered 在重跑时不再严格子集:行幂等而检出如实);
* - mediaDanglingRefs:内容引用未解析到资产的条数(按引用计——重跑如实再现);
* - mediaAmbiguous:多候选歧义的 attachment 数(按事件计,不插行)。
* issues:missing/歧义/隔离/未知签名/MIME 不符/悬空引用/副本读取失败/public_path 唯一键
* 冲突——全部显式报告(§7.6 禁静默)。
*/
interface MediaWriterCounts {
mediaRegistered: number
mediaFromSidecar: number
mediaMissing: number
mediaQuarantined: number
mediaDanglingRefs: number
mediaAmbiguous: number
}
/** 悬置引用(内容行提取、流终解析):锚点 + 引用键两半 */
interface PendingRef {
/** 引用方修订(content-writer 落的 current_revision_id——usage 的不可变锚) */
revisionId: string
/** 修订数据域内的字段位(同流重跑稳定——复合主键的 field_path 侧) */
fieldPath: string
kind: 'id' | 'url'
/** id 引用 = attachment post id;URL 引用 = 引文原串 */
value: string
/** URL 引用属本站 uploads 域时的公开路径形态(悬空裁决域);id 引用恒属本域 */
uploadsPublicPath: string | null
rawLocation: string
}
/** 侧车 JSON 条目形状(register --media-map 已在登记期校验,此处只做读取防御) */
interface SidecarEntry {
sourceUrl: string
filePath: string
}
export function createMediaEventHandler(identity: {
db: ImportDb
siteId: string
connectionId: string
snapshotId: string
/** 引用行 locale 校验基准(与 content-writer 同拒收口径——不双报) */
defaultLocale: string
/** 存储副本绝对根(media_bundle 快照 metadata.storageRoot——登记期事实) */
storageRoot: string
/** media_bundle 快照的 storagePath(相对 storageRoot;直传清单场景传 '') */
mediaBundleStoragePath: string
/** 媒体清单(media_bundle metadata.manifest;缺席 = 空表 → 全部走级③) */
mediaManifest: MediaManifestEntry[]
/** 侧车映射(manual_url_list:source_url → file_path) */
sidecarMap: Map<string, string>
}): MediaEventHandler {
const { db, siteId } = identity
const counts: MediaWriterCounts & Record<string, number> = {
mediaRegistered: 0,
mediaFromSidecar: 0,
mediaMissing: 0,
mediaQuarantined: 0,
mediaDanglingRefs: 0,
mediaAmbiguous: 0,
}
const issues: string[] = []
const pendingRefs: PendingRef[] = []
// 清单索引:精确路径 + 文件名候选(级②的侧车 file_path 是扁平名,清单可含子目录)
const manifestByPath = new Map<string, MediaManifestEntry>()
const manifestByBasename = new Map<string, MediaManifestEntry[]>()
for (const entry of identity.mediaManifest) {
manifestByPath.set(entry.path, entry)
const base = basenameOf(entry.path)
const bucket = manifestByBasename.get(base)
if (bucket === undefined) manifestByBasename.set(base, [entry])
else bucket.push(entry)
}
function basenameOf(path: string): string {
const idx = path.lastIndexOf('/')
return idx === -1 ? path : path.slice(idx + 1)
}
/** 扩展名推断声明 MIME(无扩展名/表外扩展 → null:列可空,无声明则无对拍) */
function declaredMimeOf(pathLike: string): string | null {
const m = /\.([a-z0-9]+)$/i.exec(pathLike)
if (m === null) return null
const hit = EXT_MIME[m[1]!.toLowerCase()]
return hit ?? null
}
/**
* attachment_url → 旧 uploads 相对路径。无 uploads 标记的 URL(外站迁移源)级①无从
* 掐前缀:relPath=null(级①跳过),source/public 路径以 URL pathname 兜底。
* 注:不做百分号解码——清单 path 是字面文件名,编码差异按 miss 如实进报告。
*/
function uploadsRelOf(fileUrl: string): { relPath: string | null; pathname: string } {
const idx = fileUrl.indexOf(UPLOADS_MARKER)
if (idx !== -1) {
const rel = fileUrl.slice(idx + UPLOADS_MARKER.length)
if (rel !== '') return { relPath: rel, pathname: fileUrl.slice(idx) }
}
try {
return { relPath: null, pathname: new URL(fileUrl).pathname }
} catch {
// 非 URL 形态的原始串: pathname 兜底保留原文(可审计),级①不可用
return { relPath: null, pathname: fileUrl.startsWith('/') ? fileUrl : `/${fileUrl}` }
}
}
/** 存储副本头部读取(魔数只需前若干字节——171MB 级文件不做全量读) */
async function readHead(abs: string): Promise<Buffer> {
const handle = await open(abs, 'r')
try {
const buf = Buffer.alloc(HEAD_BYTES)
const { bytesRead } = await handle.read(buf, 0, HEAD_BYTES, 0)
return buf.subarray(0, bytesRead)
} finally {
await handle.close()
}
}
/**
* attachment 行主路径:三级解析 → 魔数 → 插 media_assets。
* locale 拒收口径与 content-writer 一致(该行已被 content-writer 拒收并记 issue,
* 此处静默跳过不双报——attachment 行的媒体域处理同样以 locale 合法为前提)。
*/
async function handleAttachmentRow(row: UniversalRow): Promise<void> {
if (row.locale !== identity.defaultLocale) return
const fileUrl = typeof row.knownFields.attachmentUrl === 'string' ? row.knownFields.attachmentUrl : ''
if (fileUrl === '') {
counts.mediaMissing++
issues.push(
`attachment 缺 attachment_url(${row.sourceType}/${row.sourceId},${row.rawLocation})——三级对拍无从下手,按 missing 报告`,
)
return
}
const { relPath, pathname } = uploadsRelOf(fileUrl)
// 级①:uploads 相对路径清单精确命中
let entry: MediaManifestEntry | null = relPath === null ? null : manifestByPath.get(relPath) ?? null
let fromSidecar = false
if (entry === null) {
// 级②:侧车 source_url → file_path → 清单实体(精确路径 → 文件名候选)
const filePath = identity.sidecarMap.get(fileUrl)
if (filePath !== undefined) {
const exact = manifestByPath.get(filePath)
if (exact !== undefined) {
entry = exact
fromSidecar = true
} else {
const candidates = manifestByBasename.get(basenameOf(filePath)) ?? []
if (candidates.length > 1) {
// 歧义不自动选(spec §5.5):全部候选列报告,不插行
counts.mediaAmbiguous++
issues.push(
`媒体多候选歧义(${row.sourceType}/${row.sourceId},${fileUrl}):侧车 file_path=${filePath} 按文件名命中 ${candidates.length} 条清单实体——[${candidates.map((c) => c.path).join('、')}]——不自动选择,未登记`,
)
return
}
if (candidates.length === 1) {
entry = candidates[0]!
fromSidecar = true
}
}
}
}
if (entry === null) {
// 级③:missing——只进报告不插行(sha256 NOT NULL:无清单实体插不进,库层防线)
counts.mediaMissing++
issues.push(
`媒体 missing(${row.sourceType}/${row.sourceId},${fileUrl}):清单与侧车均未命中——不插 media_assets 行(sha256 无来源)`,
)
return
}
// 魔数:读存储副本实际字节(storageRoot + bundle 相对路径 + 清单 path)
const storagePath =
identity.mediaBundleStoragePath === ''
? entry.path
: `${identity.mediaBundleStoragePath}/${entry.path}`
let head: Buffer
try {
head = await readHead(join(identity.storageRoot, storagePath))
} catch (err) {
issues.push(
`媒体副本读取失败(${row.sourceType}/${row.sourceId},${fileUrl} → ${storagePath}):${err instanceof Error ? err.message : String(err)}——清单声称存在而副本缺席,未登记`,
)
return
}
const detected = detectMediaType(head)
const publicPath = relPath === null ? pathname : `${UPLOADS_MARKER}${relPath}`
const declared = declaredMimeOf(publicPath)
let status: 'ready' | 'quarantined' | 'unsupported'
let detectedMime: string
if (detected === null) {
status = 'unsupported'
detectedMime = UNKNOWN_DETECTED_MIME
issues.push(
`媒体未知签名(${row.sourceType}/${row.sourceId},${publicPath}):头部字节不匹配签名表——status=unsupported,不猜格式`,
)
} else if (detected.svg) {
status = 'quarantined'
detectedMime = detected.type
counts.mediaQuarantined++
issues.push(
`SVG 检出已隔离(${row.sourceType}/${row.sourceId},${publicPath}):文本 <svg 特征命中——status=quarantined,不进产物语义(spec §11.1)`,
)
} else {
status = 'ready'
detectedMime = detected.type
if (declared !== null && declared !== detected.type) {
issues.push(
`媒体声明与检测不符(${row.sourceType}/${row.sourceId},${publicPath}):扩展名推断 ${declared} ≠ 魔数检测 ${detected.type}——按检测登记(ready),数据质量问题已报告`,
)
}
}
try {
const inserted = await db
.insert(mediaAssets)
.values({
siteId,
sourceConnectionId: identity.connectionId,
sourceId: row.sourceId,
sourceUrl: fileUrl,
sourcePath: relPath ?? pathname.replace(/^\//, ''),
storagePath,
publicPath,
sha256: entry.sha256,
detectedMimeType: detectedMime,
declaredMimeType: declared,
byteSize: entry.bytes,
status,
})
// 唯一 (site, source_url) 定向幂等:命中零增行;public_path 唯一键冲突**不放行**
// (同公开路径双源是真实冲突,须显式报告而非静默吞掉)
.onConflictDoNothing({ target: [mediaAssets.siteId, mediaAssets.sourceUrl] })
.returning({ id: mediaAssets.id })
if (inserted.length > 0) {
counts.mediaRegistered++
if (fromSidecar) counts.mediaFromSidecar++
}
} catch (err) {
if (isUniqueViolation(err)) {
issues.push(
`媒体 public_path 冲突(${row.sourceType}/${row.sourceId},${publicPath}):同站公开路径已被他行占用——未登记(不静默覆盖)`,
)
return
}
throw err
}
}
/**
* 内容行引用提取(usage 两阶段的第一阶段):refs + bodyHtml img src → 悬置引用。
* featured/gallery 取 id 锚(WP meta 的 _thumbnail_id/_product_image_gallery);未知
* role 以 refs 原始下标落位(field_path 稳定优先于语义命名)。
*/
function extractRefs(row: UniversalRow): Array<Omit<PendingRef, 'revisionId'>> {
const out: Array<Omit<PendingRef, 'revisionId'>> = []
let galleryOrdinal = 0
row.attachmentRefs.forEach((ref, i) => {
let fieldPath: string
if (ref.role === 'featured') fieldPath = 'attachmentRefs.featured'
else if (ref.role === 'gallery') fieldPath = `attachmentRefs.gallery[${galleryOrdinal++}]`
else fieldPath = `attachmentRefs.raw[${i}]`
if (ref.sourceId !== undefined && ref.sourceId !== '') {
out.push({ fieldPath, kind: 'id', value: ref.sourceId, uploadsPublicPath: null, rawLocation: row.rawLocation })
} else if (ref.sourceUrl !== undefined && ref.sourceUrl !== '') {
out.push({
fieldPath,
kind: 'url',
value: ref.sourceUrl,
uploadsPublicPath: uploadsPublicPathOf(ref.sourceUrl),
rawLocation: row.rawLocation,
})
}
// 双锚皆空:adapter 层已按非法 refs 记 issue(事件 issues 面),此处不重复报
})
const body = row.bodyHtml ?? ''
if (body !== '') {
const imgRe = /<img\b[^>]*?\bsrc\s*=\s*["']([^"']+)["']/gi
let m: RegExpExecArray | null
let imgOrdinal = 0
while ((m = imgRe.exec(body)) !== null) {
const src = m[1]!
if (!src.startsWith('data:')) {
// data: URI 是内联数据非资产引用;其余(绝对/相对/协议相对)统一进解析
out.push({
fieldPath: `data.bodyHtml.img[${imgOrdinal++}]`,
kind: 'url',
value: src,
uploadsPublicPath: uploadsPublicPathOf(src),
rawLocation: row.rawLocation,
})
}
}
}
return out
}
/** URL → 本站 uploads 公开路径形态(无标记 → null:外域热链不属悬空裁决域) */
function uploadsPublicPathOf(url: string): string | null {
const idx = url.indexOf(UPLOADS_MARKER)
return idx === -1 ? null : url.slice(idx)
}
/** 修订锚点:source_records 六列身份 → entry_locales.current_revision_id(content-writer 接缝) */
async function resolveRevisionAnchor(row: UniversalRow): Promise<string | null> {
const recs = await db
.select({ contentEntryId: sourceRecords.contentEntryId })
.from(sourceRecords)
.where(
and(
eq(sourceRecords.siteId, siteId),
eq(sourceRecords.sourceConnectionId, identity.connectionId),
eq(sourceRecords.sourceType, row.sourceType),
eq(sourceRecords.sourceId, row.sourceId),
eq(sourceRecords.locale, row.locale),
eq(sourceRecords.sourceSnapshotId, identity.snapshotId),
),
)
.limit(1)
const entryId = recs[0]?.contentEntryId
if (entryId === undefined || entryId === null) return null
const els = await db
.select({ currentRevisionId: entryLocales.currentRevisionId })
.from(entryLocales)
.where(
and(
eq(entryLocales.siteId, siteId),
eq(entryLocales.contentEntryId, entryId),
eq(entryLocales.locale, row.locale),
),
)
.limit(1)
return els[0]?.currentRevisionId ?? null
}
/** 内容行(非 attachment):locale 口径对齐 content-writer(拒收行不双报),提取引用挂锚 */
async function handleContentRowRefs(row: UniversalRow): Promise<void> {
if (row.locale !== identity.defaultLocale) return
const refs = extractRefs(row)
if (refs.length === 0) return
const anchor = await resolveRevisionAnchor(row)
if (anchor === null) {
issues.push(
`媒体引用行无修订锚点(${row.sourceType}/${row.sourceId}/${row.locale},refs=${refs.length}):source_records/entry_locales 无 current 指针(content-writer 未处理该行?)——usage 跳过`,
)
return
}
for (const ref of refs) pendingRefs.push({ ...ref, revisionId: anchor })
}
/** 悬置引用 → 资产反查(id → source_id;URL → source_url 精确 → public_path 形态) */
async function lookupAsset(
ref: PendingRef,
): Promise<{ id: string; sourceUrl: string; status: string } | null> {
const columns = {
id: mediaAssets.id,
sourceUrl: mediaAssets.sourceUrl,
status: mediaAssets.status,
}
if (ref.kind === 'id') {
const rows = await db
.select(columns)
.from(mediaAssets)
.where(
and(
eq(mediaAssets.siteId, siteId),
eq(mediaAssets.sourceConnectionId, identity.connectionId),
eq(mediaAssets.sourceId, ref.value),
),
)
.limit(1)
return rows[0] ?? null
}
const byUrl = await db
.select(columns)
.from(mediaAssets)
.where(and(eq(mediaAssets.siteId, siteId), eq(mediaAssets.sourceUrl, ref.value)))
.limit(1)
if (byUrl.length > 0) return byUrl[0]!
if (ref.uploadsPublicPath !== null) {
const byPath = await db
.select(columns)
.from(mediaAssets)
.where(and(eq(mediaAssets.siteId, siteId), eq(mediaAssets.publicPath, ref.uploadsPublicPath)))
.limit(1)
return byPath[0] ?? null
}
return null
}
return {
async handleEvent(ev) {
if (ev.kind !== 'row') return // term/unsupported/error:无媒体信息,本写入器不消费
if (ev.row.sourceType === 'wp:attachment') {
await handleAttachmentRow(ev.row)
return
}
await handleContentRowRefs(ev.row)
},
/** 流终解析:悬置引用逐条对资产反查——命中且 ready → usage;无资产且属本域 → 悬空报告 */
async finish() {
for (const ref of pendingRefs) {
const asset = await lookupAsset(ref)
if (asset === null) {
if (ref.kind === 'id' || ref.uploadsPublicPath !== null) {
counts.mediaDanglingRefs++
const target = ref.kind === 'id' ? `attachment ${ref.value}` : ref.value
issues.push(
`媒体引用悬空(${ref.rawLocation},${ref.fieldPath} → ${target}):无对应 attachment/清单记录——引用保留于报告,不静默丢弃`,
)
}
// 外域 URL(无 uploads 标记):站外热链不属本域对拍范围,静默越过(JSDoc 模块头)
continue
}
if (asset.status !== 'ready') {
// quarantined/unsupported 资产:不进产物语义——引用关系不建 usage(资产侧
// issue 已在登记时报告;引用命中非悬空)
continue
}
await db
.insert(mediaUsages)
.values({
siteId,
mediaAssetId: asset.id,
contentRevisionId: ref.revisionId,
fieldPath: ref.fieldPath,
// id 引用无引文 URL——落资产登记的 source_url(审计可追);URL 引用落引文原串
sourceUrl: ref.kind === 'url' ? ref.value : asset.sourceUrl,
})
.onConflictDoNothing()
}
return { counts, issues }
},
}
}
/**
* 从管道上下文构造媒体写入器句柄(Task 6 编排注入位):manifest/sidecar 从快照 metadata
* 与存储副本解析(media_bundle.metadata.manifest / manual_url_list 副本 JSON)。
* bundle 缺席 = 空清单(三级全走 missing——诚实报告而非跳过);侧车缺席 = 空映射。
*/
export async function createMediaEventHandlerFromContext(
ctx: Pick<
ImportJobContext,
| 'db'
| 'siteId'
| 'connectionId'
| 'defaultLocale'
| 'wxrSnapshot'
| 'mediaBundleSnapshot'
| 'manualUrlListSnapshot'
>,
): Promise<MediaEventHandler> {
const bundle = ctx.mediaBundleSnapshot
const manifestRaw = bundle?.metadata.manifest
const manifest: MediaManifestEntry[] = Array.isArray(manifestRaw) ? (manifestRaw as MediaManifestEntry[]) : []
let storageRoot = ''
if (manifest.length > 0) {
const raw = bundle?.metadata.storageRoot
if (typeof raw !== 'string' || raw === '') {
throw new Error('media_bundle 快照 metadata 缺 storageRoot(登记期事实缺席——无法读取存储副本做魔数校验)')
}
storageRoot = raw
}
return createMediaEventHandler({
db: ctx.db,
siteId: ctx.siteId,
connectionId: ctx.connectionId,
snapshotId: ctx.wxrSnapshot.id,
defaultLocale: ctx.defaultLocale,
storageRoot,
mediaBundleStoragePath: bundle?.storagePath ?? '',
mediaManifest: manifest,
sidecarMap: await loadSidecarMap(ctx),
})
}
/** 侧车加载:manual_url_list 副本 JSON → Map(source_url → file_path)。形状在登记期已校验 */
async function loadSidecarMap(
ctx: Pick<ImportJobContext, 'manualUrlListSnapshot'>,
): Promise<Map<string, string>> {
const snap = ctx.manualUrlListSnapshot
if (snap === null) return new Map()
const root = snap.metadata.storageRoot
if (typeof root !== 'string' || root === '') {
throw new Error('manual_url_list 快照 metadata 缺 storageRoot(登记期事实缺席——无法读取侧车副本)')
}
let parsed: unknown
try {
parsed = JSON.parse(await readFile(join(root, snap.storagePath), 'utf8'))
} catch (err) {
throw new Error(
`侧车快照副本解析失败(${snap.storagePath}):${err instanceof Error ? err.message : String(err)}`,
)
}
if (!Array.isArray(parsed)) {
throw new Error(`侧车快照副本形状异常(${snap.storagePath}):顶层不是数组`)
}
const map = new Map<string, string>()
for (const item of parsed) {
const entry = item as Partial<SidecarEntry>
if (
typeof entry.sourceUrl !== 'string' ||
entry.sourceUrl === '' ||
typeof entry.filePath !== 'string' ||
entry.filePath === ''
) {
throw new Error(`侧车快照副本形状异常(${snap.storagePath}):存在非 {sourceUrl, filePath} 条目`)
}
map.set(entry.sourceUrl, entry.filePath)
}
return map
}
/**
* 媒体登记管道写入器({@link createMediaEventHandler} 的自消费整流便捷壳):单遍拉完
* ctx.events、逐事件分派、流终返回 outcome。与 content-writer 组合消费同流时须用
* {@link createMediaEventHandlerFromContext} 句柄形状(单消费方拉流分派,无 tee)。
*/
export function createMediaWriter(): ImportPipeline {
return async (ctx) => {
const handler = await createMediaEventHandlerFromContext(ctx)
for await (const ev of ctx.events) {
await handler.handleEvent(ev)
}
return handler.finish()
}
}
+643
View File
@@ -0,0 +1,643 @@
// Plan 3 Task 4 失败测试(先于实现):媒体登记写入器(三级对拍 + 魔数/隔离 + 悬空引用)。
// 覆盖契约(对照计划 Task 4 与前置事实 7/11、spec §5.5/§11.1/§7.6):
// 1. mini 全链三级对拍:①uploads 相对路径清单精确命中(fixture-a)②侧车 source_url→
// file_path→清单实体(fixture-b——清单故意 miss 该路径逼走侧车级)③missing 不插行
// (media_assets sha256 NOT NULL——另测无 bundle 场景);命中行 public_path=旧 uploads
// 路径、sha256/byte_size/storage_path 从清单取;media_usages 三行(featured/gallery/
// 正文 img——从 row.attachmentRefs + bodyHtml 提取,revision 锚点经 source_records→
// entry_locales.current_revision_id);
// 2. 魔数(集成面):命中文件读实际字节——PNG 过;扩展名声明 MIME 与检测不符 → issue;
// SVG 检出(.png 伪装与显式 .svg 双例)→ status='quarantined' + issue(不进产物语义);
// 未知签名 → status='unsupported' + issue;quarantined 资产的引用不建 usage;
// 3. 悬空引用(mini 203/204):gallery 引用无对应 attachment/清单记录 → mediaDanglingRefs
// 计数 + issue(不静默丢弃,§7.6);
// 4. 多候选:侧车 file_path 按文件名命中两条清单(同 filename 不同目录)→ 歧义不自动选、
// 不插行、全部候选列 issue(spec §5.5);
// 5. 幂等:重跑 media_assets/media_usages 零增行(mediaRegistered=0;悬空引用按流如实再现);
// 6. counts 键(media 前缀全局命名空间):mediaRegistered/mediaFromSidecar/mediaMissing/
// mediaQuarantined/mediaDanglingRefs/mediaAmbiguous——与 content-writer 键并存无碰撞。
// 双事件消费设计(实现侧 JSDoc 同步):attachment 行建资产、内容行提取引用;WXR 文档序
// items 先于 attachment——引用先行资产后到,usage 解析在 finish() 统一对 media_assets
// 反查(顺序无关)。运行于 globalSetup 临时库(ADR-015);报告根 os.tmpdir,finally 清理。
import { createHash } from 'node:crypto'
import { createReadStream } from 'node:fs'
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import { runWxrAdapter, type AdapterEvent } from '@sf/adapter-wp'
import type { UniversalRow } from '@sf/contracts'
import type { MediaManifestEntry } from '../../apps/api/src/modules/source/register.service.js'
import type { ImportPipeline } from '../../apps/api/src/modules/import/import.service.js'
import { createImportJob } from '../../apps/api/src/modules/import/import.service.js'
import { createContentEventHandler } from '../../apps/api/src/modules/import/content-writer.js'
import { createMediaEventHandler, createMediaEventHandlerFromContext } from '../../apps/api/src/modules/import/media-writer.js'
import { registerSource } from '../../apps/api/src/modules/source/register.service.js'
import { rand, withImmutableCleanup, withTestDb, type Db } from './helpers.js'
const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url))
const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml')
const PNG_SIG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
const JPEG_SIG = Buffer.from([0xff, 0xd8, 0xff, 0xe0])
/** 伪 PNG/JPEG 体(签名 + 唯一种子载荷——魔数只看头,内容唯一避免跨测试清单撞哈希) */
function pngBytes(seed: string): Buffer {
return Buffer.concat([PNG_SIG, Buffer.from(`png-body:${seed}`, 'latin1')])
}
function jpegBytes(seed: string): Buffer {
return Buffer.concat([JPEG_SIG, Buffer.from(`jpeg-body:${seed}`, 'latin1')])
}
function sha256of(buf: Buffer): string {
return createHash('sha256').update(buf).digest('hex')
}
function manifestEntryOf(path: string, buf: Buffer): MediaManifestEntry {
return { path, bytes: buf.length, sha256: sha256of(buf) }
}
/** 手构事件流的 AsyncIterable 包装(每次调用产新迭代器——重跑各流独立) */
async function* eventsOf(events: AdapterEvent[]): AsyncGenerator<AdapterEvent, void, void> {
yield* events
}
/** db.execute 行对象规整(postgres.js 原型差异会让 toStrictEqual 报"无视觉差异"失败) */
function plain(rows: any[]): any[] {
return rows.map((r) => ({ ...r }))
}
/** 事件流收集(mini 全链一饮而尽) */
async function collectMiniEvents(): Promise<AdapterEvent[]> {
const out: AdapterEvent[] = []
for await (const ev of runWxrAdapter(createReadStream(WXR_FIXTURE), { mapConfig: { locale: 'en' } })) {
out.push(ev)
}
return out
}
/** 手构 attachment row 事件(knownFields.attachmentUrl 是三级对拍的输入锚点) */
function attachmentEvent(spec: { sourceId: string; url: string }): AdapterEvent {
const row: UniversalRow = {
sourceType: 'wp:attachment',
sourceId: spec.sourceId,
status: 'inherit',
locale: 'en',
taxonomyRefs: [],
attachmentRefs: [],
seo: {},
knownFields: { attachmentUrl: spec.url },
extras: {},
rawLocation: `wxr://item/${spec.sourceId}`,
}
return { kind: 'row', row, issues: [], sanitizeDiffs: [], changed: false }
}
/** Task 6 编排雏形:单消费方拉流,逐事件先 content 后 media 分派(content 先行是
* media-writer usage 锚点的顺序契约——source_records/revision 须已落);两态经 finish 取回 */
function contentPlusMediaPipeline(): ImportPipeline {
return contentPlusDispatch(async (ctx) => createMediaEventHandlerFromContext(ctx))
}
/** content + 直传 media 组合(手构场景:不经 register 的 media_bundle)——分派形状同上 */
function contentPlusDirectMediaPipeline(identity: {
storageRoot: string
manifest: MediaManifestEntry[]
sidecar: Array<[string, string]>
}): ImportPipeline {
return contentPlusDispatch(async (ctx) =>
createMediaEventHandler({
db: ctx.db,
siteId: ctx.siteId,
connectionId: ctx.connectionId,
snapshotId: ctx.wxrSnapshot.id,
defaultLocale: ctx.defaultLocale,
storageRoot: identity.storageRoot,
mediaBundleStoragePath: '',
mediaManifest: identity.manifest,
sidecarMap: new Map(identity.sidecar),
}),
)
}
/** 分派骨架:先 content 后 media(顺序契约),counts 展开合并(media 前缀无碰撞键) */
function contentPlusDispatch(
mediaOf: (ctx: Parameters<ImportPipeline>[0]) => Promise<ReturnType<typeof createMediaEventHandler>>,
): ImportPipeline {
return async (ctx) => {
const content = createContentEventHandler({
db: ctx.db,
siteId: ctx.siteId,
connectionId: ctx.connectionId,
snapshotId: ctx.wxrSnapshot.id,
jobId: ctx.jobId,
defaultLocale: ctx.defaultLocale,
versions: ctx.versions,
})
const media = await mediaOf(ctx)
for await (const ev of ctx.events) {
await content.handleEvent(ev)
await media.handleEvent(ev)
}
const c = content.finish()
const m = await media.finish()
return { counts: { ...c.counts, ...m.counts }, issues: [...c.issues, ...m.issues] }
}
}
/** 清单/侧车直传管道(手构场景:无内容行——纯 attachment 事件,media 单写入器) */
function directMediaPipeline(identity: {
storageRoot: string
manifest: MediaManifestEntry[]
sidecar: Array<[string, string]>
}): ImportPipeline {
return async (ctx) => {
const handler = createMediaEventHandler({
db: ctx.db,
siteId: ctx.siteId,
connectionId: ctx.connectionId,
snapshotId: ctx.wxrSnapshot.id,
defaultLocale: ctx.defaultLocale,
storageRoot: identity.storageRoot,
mediaBundleStoragePath: '',
mediaManifest: identity.manifest,
sidecarMap: new Map(identity.sidecar),
})
for await (const ev of ctx.events) {
await handler.handleEvent(ev)
}
return handler.finish()
}
}
/** 本文件写入的行清场:FK 反序(media_usages → media_assets 先于内容域;两张不可变表走触发器开关) */
async function cleanupSiteRows(db: Db, siteId: string): Promise<void> {
await db.execute(`delete from media_usages where site_id = '${siteId}'`)
await db.execute(`delete from media_assets where site_id = '${siteId}'`)
await db.execute(`delete from content_terms where site_id = '${siteId}'`)
await db.execute(`delete from terms where site_id = '${siteId}'`)
await db.execute(`delete from taxonomies where site_id = '${siteId}'`)
await db.execute(`delete from source_records where site_id = '${siteId}'`)
await db.execute(`delete from entry_locales where site_id = '${siteId}'`)
await withImmutableCleanup(db, 'content_revisions', 'content_revisions_immutable', `site_id = '${siteId}'`)
await db.execute(`delete from content_entries where site_id = '${siteId}'`)
await db.execute(`delete from import_jobs where site_id = '${siteId}'`)
await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`)
await db.execute(`delete from source_connections where site_id = '${siteId}'`)
await db.execute(`delete from sites where id = '${siteId}'`)
}
/** mini 登记(各用例公共前置):register → {siteId, connectionId, wxr 快照} */
async function registerMini(db: Db, work: string) {
const key = `sfmedia-${rand()}`
const reg = await registerSource(db, {
siteKey: key,
wxrPath: WXR_FIXTURE,
allowedRoots: [FIXTURES_ROOT, work],
storageRoot: join(work, 'storage'),
reportRoot: join(work, 'artifacts', 'register'),
})
return { key, reg, wxr: reg.snapshots.find((s) => s.kind === 'wxr')! }
}
/** 导入报告读取 */
async function readReport(work: string, jobId: string): Promise<{
status: string
counts: Record<string, number>
issues: string[]
}> {
return JSON.parse(await readFile(join(work, 'artifacts', 'import', jobId, 'reports', 'import.json'), 'utf8'))
}
describe('媒体登记写入器(三级对拍 + 魔数/隔离)', () => {
it('mini 全链三级对拍:①清单精确命中 ②侧车命中 → media_assets(public_path=旧 uploads 路径、sha256/storage_path 从清单取)+ usage 三行 + 悬空 203/204 + counts 全键', async () => {
await withTestDb(async (db) => {
const work = await mkdtemp(join(tmpdir(), 'sf-media-full-'))
try {
const events = await collectMiniEvents()
// 媒体目录:fixture-a 按旧 uploads 层级放(级①精确命中);fixture-b 用扁平
// GUID 名(清单故意不含 '2026/10/fixture-b.png'——逼走级②侧车)
const mediaDir = join(work, 'media')
await mkdir(join(mediaDir, '2026/10'), { recursive: true })
const pngA = pngBytes(`fixture-a-${rand()}`)
await writeFile(join(mediaDir, '2026/10/fixture-a.png'), pngA)
const guidB = `b-${rand()}.png`
const pngB = pngBytes(`fixture-b-${rand()}`)
await writeFile(join(mediaDir, guidB), pngB)
const sidecarPath = join(work, 'media-map.json')
await writeFile(
sidecarPath,
JSON.stringify([
{ sourceUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-b.png', filePath: guidB },
]),
)
const key = `sfmedia-${rand()}`
const reg = await registerSource(db, {
siteKey: key,
wxrPath: WXR_FIXTURE,
mediaDir,
mediaMapPath: sidecarPath,
allowedRoots: [FIXTURES_ROOT, work],
storageRoot: join(work, 'storage'),
reportRoot: join(work, 'artifacts', 'register'),
})
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
const bundle = reg.snapshots.find((s) => s.kind === 'media_bundle')!
try {
const result = await createImportJob(db, {
siteKey: key,
snapshotId: wxr.id,
reportRoot: join(work, 'artifacts', 'import'),
events: eventsOf(events),
pipeline: contentPlusMediaPipeline(),
})
// 悬空 203/204 → issues 非空 → succeeded_with_issues(§7.6 禁静默)
expect(result.status).toBe('succeeded_with_issues')
// media_assets 两行:fixture-a 级① / fixture-b 级②;身份列逐项对拍
const assets = (await db.execute(`select source_id, source_url, source_path, storage_path,
public_path, sha256, detected_mime, declared_mime, byte_size::int as byte_size, status
from media_assets where site_id = '${reg.siteId}' order by source_id`)) as any
expect(assets).toHaveLength(2)
const a = assets.find((x: any) => x.source_id === '201')!
expect(a.source_url).toBe('https://fixture.example/wp-content/uploads/2026/10/fixture-a.png')
expect(a.source_path).toBe('2026/10/fixture-a.png')
expect(a.public_path).toBe('/wp-content/uploads/2026/10/fixture-a.png')
expect(a.sha256).toBe(sha256of(pngA))
expect(a.byte_size).toBe(pngA.length)
expect(a.detected_mime).toBe('image/png')
expect(a.declared_mime).toBe('image/png')
expect(a.status).toBe('ready')
// storage_path = 清单实体定位(bundle 副本相对路径 + 清单 path——绝对副本 = storageRoot + 此列)
expect(a.storage_path).toBe(`${bundle.storagePath}/2026/10/fixture-a.png`)
const b = assets.find((x: any) => x.source_id === '202')!
expect(b.source_url).toBe('https://fixture.example/wp-content/uploads/2026/10/fixture-b.png')
expect(b.public_path).toBe('/wp-content/uploads/2026/10/fixture-b.png')
expect(b.sha256).toBe(sha256of(pngB))
expect(b.storage_path).toBe(`${bundle.storagePath}/${guidB}`)
expect(b.status).toBe('ready')
// media_usages 三行:101 featured(id 反查)+ 101 正文 img(URL 反查)+ 111 featured
const usages = (await db.execute(`select a.source_id as asset, u.field_path, u.source_url
from media_usages u join media_assets a on a.id = u.media_asset_id
where u.site_id = '${reg.siteId}' order by a.source_id, u.field_path`)) as any
expect(plain(usages)).toStrictEqual([
{
asset: '201',
field_path: 'attachmentRefs.featured',
source_url: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png',
},
{
asset: '201',
field_path: 'data.bodyHtml.img[0]',
source_url: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png',
},
{
asset: '202',
field_path: 'attachmentRefs.featured',
source_url: 'https://fixture.example/wp-content/uploads/2026/10/fixture-b.png',
},
])
// usage 挂在 content-writer 落的当前修订上(current_revision_id——导入不发布,published NULL)
const revCheck = (await db.execute(`select count(*)::int as n from media_usages u
join content_revisions cr on cr.id = u.content_revision_id
join entry_locales el on el.current_revision_id = cr.id and el.published_revision_id is null
where u.site_id = '${reg.siteId}'`)) as any
expect(revCheck[0].n).toBe(3)
// counts 全键:content 四键 + media 六键(全局命名空间并存无碰撞)
const report = await readReport(work, result.jobId)
expect(report.status).toBe('succeeded_with_issues')
expect(report.counts).toStrictEqual({
contentEntries: 3,
revisionsCreated: 3,
sourceRecords: 5,
skipped: 0,
mediaRegistered: 2,
mediaFromSidecar: 1,
mediaMissing: 0,
mediaQuarantined: 0,
// mini 全部引用 5 条:featured 201/202 + 正文 img 各解析命中,gallery 203/204 悬空
mediaDanglingRefs: 2,
mediaAmbiguous: 0,
})
// 悬空引用显式报告:203/204 各一条(gallery 引用无 attachment/清单记录)
const dangling = report.issues.filter((i) => i.includes('悬空'))
expect(dangling).toHaveLength(2)
expect(dangling.some((i) => i.includes('203'))).toBe(true)
expect(dangling.some((i) => i.includes('204'))).toBe(true)
} finally {
await cleanupSiteRows(db, reg.siteId)
}
} finally {
await rm(work, { recursive: true, force: true })
}
})
})
it('级③ missing(无 media_bundle 登记):不插 media_assets 行(sha256 NOT NULL)+ mediaMissing + 全部引用悬空如实报告', async () => {
await withTestDb(async (db) => {
const work = await mkdtemp(join(tmpdir(), 'sf-media-missing-'))
try {
const events = await collectMiniEvents()
const { key, reg, wxr } = await registerMini(db, work)
try {
const result = await createImportJob(db, {
siteKey: key,
snapshotId: wxr.id,
reportRoot: join(work, 'artifacts', 'import'),
events: eventsOf(events),
pipeline: contentPlusMediaPipeline(),
})
expect(result.status).toBe('succeeded_with_issues')
// 零资产行:清单缺席 → 级①②全 miss → 级③ missing(不插行——sha256 无来源)
expect(((await db.execute(`select count(*)::int as n from media_assets
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
expect(((await db.execute(`select count(*)::int as n from media_usages
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
const report = await readReport(work, result.jobId)
expect(report.counts).toStrictEqual({
contentEntries: 3,
revisionsCreated: 3,
sourceRecords: 5,
skipped: 0,
mediaRegistered: 0,
mediaFromSidecar: 0,
mediaMissing: 2,
mediaQuarantined: 0,
// 引用全悬空:101 featured+img、111 featured+gallery×2 = 5
mediaDanglingRefs: 5,
mediaAmbiguous: 0,
})
const missing = report.issues.filter((i) => i.includes('missing'))
expect(missing).toHaveLength(2)
expect(missing.some((i) => i.includes('fixture-a.png'))).toBe(true)
expect(missing.some((i) => i.includes('fixture-b.png'))).toBe(true)
expect(report.issues.filter((i) => i.includes('悬空'))).toHaveLength(5)
} finally {
await cleanupSiteRows(db, reg.siteId)
}
} finally {
await rm(work, { recursive: true, force: true })
}
})
})
it('魔数校验:命中文件读实际字节——PNG 过;声明(扩展名推断)与检测不符 → issue;SVG 检出(伪装 .png 与显式 .svg)→ quarantined 不进产物;未知签名 → unsupported', async () => {
await withTestDb(async (db) => {
const work = await mkdtemp(join(tmpdir(), 'sf-media-magic-'))
try {
// 五个手构文件(放旧 uploads 层级——级① 相对路径精确命中):正例 PNG / 扩展名谎言
// (.png 实为 JPEG)/ SVG 伪装 .png / 显式 .svg / 无签名 .txt
const files: Array<{ name: string; buf: Buffer }> = [
{ name: 'ok.png', buf: pngBytes(`ok-${rand()}`) },
{ name: 'lie.png', buf: jpegBytes(`lie-${rand()}`) },
{ name: 'evil.png', buf: Buffer.from(`<?xml version="1.0"?><SVG onload="alert(1)" xmlns="…">`) },
{ name: 'draw.svg', buf: Buffer.from('<svg width="10" height="10"></svg>') },
{ name: 'note.txt', buf: Buffer.from('plain text note, no signature at all') },
]
await mkdir(join(work, '2026/10'), { recursive: true })
for (const f of files) {
await writeFile(join(work, '2026/10', f.name), f.buf)
}
const manifest = files.map((f) => manifestEntryOf(`2026/10/${f.name}`, f.buf))
const events: AdapterEvent[] = files.map((f, i) =>
attachmentEvent({ sourceId: String(901 + i), url: `https://fixture.example/wp-content/uploads/2026/10/${f.name}` }),
)
const { key, reg, wxr } = await registerMini(db, work)
try {
const result = await createImportJob(db, {
siteKey: key,
snapshotId: wxr.id,
reportRoot: join(work, 'artifacts', 'import'),
events: eventsOf(events),
pipeline: directMediaPipeline({ storageRoot: work, manifest, sidecar: [] }),
})
expect(result.status).toBe('succeeded_with_issues')
const assets = (await db.execute(`select source_id, detected_mime, declared_mime, status
from media_assets where site_id = '${reg.siteId}' order by source_id`)) as any
expect(plain(assets)).toStrictEqual([
{ source_id: '901', detected_mime: 'image/png', declared_mime: 'image/png', status: 'ready' },
{ source_id: '902', detected_mime: 'image/jpeg', declared_mime: 'image/png', status: 'ready' },
{ source_id: '903', detected_mime: 'image/svg+xml', declared_mime: 'image/png', status: 'quarantined' },
{ source_id: '904', detected_mime: 'image/svg+xml', declared_mime: 'image/svg+xml', status: 'quarantined' },
{ source_id: '905', detected_mime: 'application/octet-stream', declared_mime: null, status: 'unsupported' },
])
const report = await readReport(work, result.jobId)
expect(report.counts).toStrictEqual({
mediaRegistered: 5,
mediaFromSidecar: 0,
mediaMissing: 0,
mediaQuarantined: 2,
mediaDanglingRefs: 0,
mediaAmbiguous: 0,
})
// 三类问题各就位:MIME 不符 1(lie.png)+ SVG 隔离 2(evil/draw)+ 未知签名 1(note.txt)
const mismatch = report.issues.filter((i) => i.includes('不符'))
expect(mismatch).toHaveLength(1)
expect(mismatch[0]).toContain('lie.png')
const quarantined = report.issues.filter((i) => i.includes('隔离'))
expect(quarantined).toHaveLength(2)
expect(report.issues.filter((i) => i.includes('未知签名'))).toHaveLength(1)
// quarantined/unsupported 资产不进产物语义:即便有引用也不建 usage(本例无内容行,零 usage)
expect(((await db.execute(`select count(*)::int as n from media_usages
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
} finally {
await cleanupSiteRows(db, reg.siteId)
}
} finally {
await rm(work, { recursive: true, force: true })
}
})
})
it('quarantined 资产的引用不建 usage(不进产物语义):SVG 资产被正文引用 → 引用关系不落 media_usages', async () => {
await withTestDb(async (db) => {
const work = await mkdtemp(join(tmpdir(), 'sf-media-quar-'))
try {
const evil = Buffer.from('<svg onload="x()"><script>bad()</script></svg>')
await mkdir(join(work, '2026/10'), { recursive: true })
await writeFile(join(work, '2026/10/evil.png'), evil)
const manifest = [manifestEntryOf('2026/10/evil.png', evil)]
const attachment = attachmentEvent({
sourceId: '951',
url: 'https://fixture.example/wp-content/uploads/2026/10/evil.png',
})
const citing: AdapterEvent = {
kind: 'row',
row: {
sourceType: 'wp:post',
sourceId: '950',
status: 'publish',
locale: 'en',
taxonomyRefs: [],
attachmentRefs: [{ role: 'featured', sourceId: '951' }],
seo: {},
knownFields: {},
extras: {},
rawLocation: 'wxr://item/950',
bodyHtml: '<p><img src="https://fixture.example/wp-content/uploads/2026/10/evil.png"/></p>',
},
issues: [],
sanitizeDiffs: [],
changed: false,
}
const { key, reg, wxr } = await registerMini(db, work)
try {
// 引用行先到、attachment 后到(WXR 文档序)——finish 统一解析
const result = await createImportJob(db, {
siteKey: key,
snapshotId: wxr.id,
reportRoot: join(work, 'artifacts', 'import'),
events: eventsOf([citing, attachment]),
pipeline: contentPlusDirectMediaPipeline({ storageRoot: work, manifest, sidecar: [] }),
})
expect(result.status).toBe('succeeded_with_issues')
// 资产行存在(登记可审计)但 quarantined;引用解析命中资产(非悬空)但不建 usage
const assets = (await db.execute(`select source_id, status from media_assets
where site_id = '${reg.siteId}'`)) as any
expect(plain(assets)).toStrictEqual([{ source_id: '951', status: 'quarantined' }])
expect(((await db.execute(`select count(*)::int as n from media_usages
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
const report = await readReport(work, result.jobId)
expect(report.counts.mediaRegistered).toBe(1)
expect(report.counts.mediaQuarantined).toBe(1)
expect(report.counts.mediaDanglingRefs).toBe(0)
expect(report.issues.filter((i) => i.includes('隔离'))).toHaveLength(1)
} finally {
await cleanupSiteRows(db, reg.siteId)
}
} finally {
await rm(work, { recursive: true, force: true })
}
})
})
it('多候选(同 filename 不同目录两条清单):歧义不自动选、不插行、全部候选列 issue', async () => {
await withTestDb(async (db) => {
const work = await mkdtemp(join(tmpdir(), 'sf-media-ambig-'))
try {
await mkdir(join(work, 'd1'), { recursive: true })
await mkdir(join(work, 'd2'), { recursive: true })
const f1 = pngBytes(`amb-1-${rand()}`)
const f2 = pngBytes(`amb-2-${rand()}`)
await writeFile(join(work, 'd1/thing.png'), f1)
await writeFile(join(work, 'd2/thing.png'), f2)
const manifest = [manifestEntryOf('d1/thing.png', f1), manifestEntryOf('d2/thing.png', f2)]
const url = 'https://fixture.example/wp-content/uploads/2024/01/thing.png'
const events: AdapterEvent[] = [attachmentEvent({ sourceId: '961', url })]
const { key, reg, wxr } = await registerMini(db, work)
try {
const result = await createImportJob(db, {
siteKey: key,
snapshotId: wxr.id,
reportRoot: join(work, 'artifacts', 'import'),
events: eventsOf(events),
// 侧车 file_path 只给文件名:精确路径 miss → 文件名候选 2 条 → 歧义
pipeline: directMediaPipeline({ storageRoot: work, manifest, sidecar: [[url, 'thing.png']] }),
})
expect(result.status).toBe('succeeded_with_issues')
expect(((await db.execute(`select count(*)::int as n from media_assets
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
const report = await readReport(work, result.jobId)
expect(report.counts).toStrictEqual({
mediaRegistered: 0,
mediaFromSidecar: 0,
mediaMissing: 0,
mediaQuarantined: 0,
mediaDanglingRefs: 0,
mediaAmbiguous: 1,
})
const amb = report.issues.filter((i) => i.includes('歧义'))
expect(amb).toHaveLength(1)
expect(amb[0]).toContain('d1/thing.png')
expect(amb[0]).toContain('d2/thing.png')
} finally {
await cleanupSiteRows(db, reg.siteId)
}
} finally {
await rm(work, { recursive: true, force: true })
}
})
})
it('幂等重跑:media_assets/media_usages 零增行、mediaRegistered=0;悬空引用按流如实再现', async () => {
await withTestDb(async (db) => {
const work = await mkdtemp(join(tmpdir(), 'sf-media-idem-'))
try {
const events = await collectMiniEvents()
const mediaDir = join(work, 'media')
await mkdir(join(mediaDir, '2026/10'), { recursive: true })
const pngA = pngBytes(`idem-a-${rand()}`)
await writeFile(join(mediaDir, '2026/10/fixture-a.png'), pngA)
const guidB = `b-${rand()}.png`
await writeFile(join(mediaDir, guidB), pngBytes(`idem-b-${rand()}`))
const sidecarPath = join(work, 'media-map.json')
await writeFile(
sidecarPath,
JSON.stringify([
{ sourceUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-b.png', filePath: guidB },
]),
)
const key = `sfmedia-${rand()}`
const reg = await registerSource(db, {
siteKey: key,
wxrPath: WXR_FIXTURE,
mediaDir,
mediaMapPath: sidecarPath,
allowedRoots: [FIXTURES_ROOT, work],
storageRoot: join(work, 'storage'),
reportRoot: join(work, 'artifacts', 'register'),
})
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
try {
const base = {
siteKey: key,
snapshotId: wxr.id,
reportRoot: join(work, 'artifacts', 'import'),
pipeline: contentPlusMediaPipeline(),
}
const first = await createImportJob(db, { ...base, events: eventsOf(events) })
expect(first.status).toBe('succeeded_with_issues')
// 同流重跑(换幂等键新 job——钉写入器幂等,不测 job 复用)
const again = await createImportJob(db, {
...base,
idempotencyKey: `sfmedia-rerun-${rand()}`,
events: eventsOf(events),
})
expect(again.reused).toBe(false)
expect(again.status).toBe('succeeded_with_issues')
const after = (await db.execute(`select
(select count(*)::int from media_assets where site_id = '${reg.siteId}') as assets,
(select count(*)::int from media_usages where site_id = '${reg.siteId}') as usages`)) as any
expect(plain(after)[0]).toStrictEqual({ assets: 2, usages: 3 })
// 行面零增;流面计数如实:registered/fromSidecar 归零,悬空引用再现(203/204)
const report = await readReport(work, again.jobId)
expect(report.counts).toStrictEqual({
contentEntries: 0,
revisionsCreated: 0,
sourceRecords: 0,
skipped: 3,
mediaRegistered: 0,
mediaFromSidecar: 0,
mediaMissing: 0,
mediaQuarantined: 0,
mediaDanglingRefs: 2,
mediaAmbiguous: 0,
})
expect(report.issues.filter((i) => i.includes('悬空'))).toHaveLength(2)
} finally {
await cleanupSiteRows(db, reg.siteId)
}
} finally {
await rm(work, { recursive: true, force: true })
}
})
})
})
+66
View File
@@ -0,0 +1,66 @@
// magic-bytes 单测(Plan 3 Task 4 Step 1):六类签名各正例 + 伪签名 null 例 +
// SVG 大小写不敏感 + 窗口边界(1KB 外不检出)+ 二进制签名优先于文本特征。
// 期望值全部手写字面量(与实现零共享——对拍而非引用)。
import { describe, expect, it } from 'vitest'
import { detectMediaType } from '../../apps/api/src/modules/import/magic-bytes.js'
const PNG_SIG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
describe('detectMediaType 手写签名表', () => {
it('六类二进制签名正例:PNG/JPEG/GIF87a/GIF89a/WEBP/PDF', () => {
expect(detectMediaType(Buffer.concat([PNG_SIG, Buffer.from('rest-of-png')]))).toStrictEqual({
type: 'image/png',
svg: false,
})
expect(detectMediaType(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]))).toStrictEqual({
type: 'image/jpeg',
svg: false,
})
expect(detectMediaType(Buffer.from('GIF87a-more-gif'))).toStrictEqual({ type: 'image/gif', svg: false })
expect(detectMediaType(Buffer.from('GIF89a-more-gif'))).toStrictEqual({ type: 'image/gif', svg: false })
expect(
detectMediaType(Buffer.concat([Buffer.from('RIFF'), Buffer.from([0x24, 0x00, 0x00, 0x00]), Buffer.from('WEBPVP8 ')])),
).toStrictEqual({ type: 'image/webp', svg: false })
expect(detectMediaType(Buffer.from('%PDF-1.7\n%fake-payload'))).toStrictEqual({
type: 'application/pdf',
svg: false,
})
})
it('SVG 文本特征:大小写不敏感(<svg / <SVG / <Svg,含 xml 声明前导),svg=true', () => {
expect(detectMediaType(Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"/>'))).toStrictEqual({
type: 'image/svg+xml',
svg: true,
})
expect(detectMediaType(Buffer.from('<?xml version="1.0"?><SVG onload="x()">'))).toStrictEqual({
type: 'image/svg+xml',
svg: true,
})
expect(detectMediaType(Buffer.from('<!-- c --><Svg width="1">'))).toStrictEqual({
type: 'image/svg+xml',
svg: true,
})
})
it('伪签名 / 纯文本 / 空缓冲 → null(未知签名不猜格式)', () => {
expect(detectMediaType(Buffer.from('hello plain text note'))).toBe(null)
expect(detectMediaType(Buffer.from([0x00, 0x01, 0x02, 0x03]))).toBe(null)
expect(detectMediaType(Buffer.alloc(0))).toBe(null)
})
it('RIFF 非 WEBP(WAV)不误报 webp → null', () => {
expect(
detectMediaType(Buffer.concat([Buffer.from('RIFF'), Buffer.from([0x24, 0x00, 0x00, 0x00]), Buffer.from('WAVEfmt ')])),
).toBe(null)
})
it('二进制签名优先:PNG 头 + 尾部 <svg 片段仍判 PNG(文本窗口只对非二进制生效)', () => {
const sneaky = Buffer.concat([PNG_SIG, Buffer.from('<svg onload="alert(1)"> trailing')])
expect(detectMediaType(sneaky)).toStrictEqual({ type: 'image/png', svg: false })
})
it('SVG 窗口 1KB 钉死:<svg 出现在 1KB 之后不检出 → null', () => {
const beyond = Buffer.concat([Buffer.alloc(1200, 0x20), Buffer.from('<svg>')])
expect(detectMediaType(beyond)).toBe(null)
})
})