feat(import): 媒体登记与安全校验
Plan 3 Task 4:media-writer(attachment 行三级对拍 + 内容行引用提取双事件消费) + magic-bytes 手写签名表(零新增依赖)。 - 三级对拍(前置事实 7):①uploads 相对路径清单精确命中 ②侧车 source_url→ file_path→清单实体(精确 miss 按文件名候选,多候选歧义不自动选全列 issue) ③missing 只进报告不插行(sha256 NOT NULL 库层防线) - 魔数校验:命中实体读存储副本头部实际字节——SVG 检出 status=quarantined 不进 产物语义(引用不建 usage);未知签名 unsupported;扩展名声明 MIME 与检测不符 → issue(按检测登记) - usage 两阶段时序(WXR 文档序 items 先于 attachment):内容行提取引用 + 修订 锚点(依赖 content-writer 先处理同事件——Task 6 编排顺序契约),finish 统一 对 media_assets 反查(id→source_id / URL→source_url→public_path),顺序无关 - 悬空引用计数 mediaDanglingRefs 不静默(mini 203/204 钉死) - counts 六键 media 前缀独占(勿复用 'skipped' 的键契约注记) - 幂等:(site,source_url) 定向 onConflictDoNothing + usage 复合主键——重跑零增行 308 测试(296+6 单测+6 集成);typecheck/lint 三绿。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
// 手写魔数签名表(Plan 3 Task 4,spec §11.1、计划"Tech Stack:零新增依赖"):
|
||||
// 常见六类媒体格式判别 + SVG 文本特征检出。刻意不引 file-type 依赖——导入管道的
|
||||
// 安全决策只须回答两问:"是什么格式"与"是否 SVG(隔离)",一张可审计的小签名表
|
||||
// 足够,且避免为一次头部读取引入整个 MIME 数据库。
|
||||
//
|
||||
// 判别语义(detectMediaType):
|
||||
// - 命中二进制签名 → {type: MIME, svg: false}(PNG/JPEG/GIF87a/GIF89a/WEBP/PDF);
|
||||
// - 无二进制命中但前 1KB 文本窗口含 "<svg"(大小写不敏感)→ {type: 'image/svg+xml',
|
||||
// svg: true}——调用方据此 status='quarantined'(spec §11.1:SVG 可携带脚本与外部
|
||||
// 引用,不进产物语义);
|
||||
// - 均未命中 → null(未知签名——调用方 status='unsupported' + issue,不猜格式)。
|
||||
//
|
||||
// 顺序纪律:二进制签名先于 SVG 文本扫描——PNG/JPEG 正例即便尾部字节碰巧含 "<svg"
|
||||
// 片段也按二进制格式判(文本窗口只对非二进制文件有意义)。窗口 1KB 是钉死的折中:
|
||||
// 覆盖常规 "<?xml …?><svg" 前导(声明/注释/doctype),不给深藏脚本的巨型前缀放行。
|
||||
//
|
||||
// Buffer 读取口径:latin1 单字节解码后小写化做包含检测——不做 UTF-8 多字节解码,
|
||||
// 避免截断的字节序列被解码成意外字符。
|
||||
/** 二进制签名表(前缀匹配;WEBP 的 RIFF+WEBP@8 双段结构在表外单查) */
|
||||
const SIGNATURES: ReadonlyArray<{ mime: string; bytes: Buffer }> = Object.freeze([
|
||||
{ mime: 'image/png', bytes: Buffer.from([0x89, 0x50, 0x4e, 0x47]) },
|
||||
{ mime: 'image/jpeg', bytes: Buffer.from([0xff, 0xd8, 0xff]) },
|
||||
{ mime: 'image/gif', bytes: Buffer.from('GIF87a', 'latin1') },
|
||||
{ mime: 'image/gif', bytes: Buffer.from('GIF89a', 'latin1') },
|
||||
{ mime: 'application/pdf', bytes: Buffer.from('%PDF', 'latin1') },
|
||||
])
|
||||
|
||||
const RIFF = Buffer.from('RIFF', 'latin1')
|
||||
const WEBP = Buffer.from('WEBP', 'latin1')
|
||||
const SVG_WINDOW_BYTES = 1024
|
||||
|
||||
/**
|
||||
* 魔数检测:读文件头部字节判别媒体类型。返回 null = 未知签名(调用方按
|
||||
* unsupported 处置);svg=true 仅出现在 type='image/svg+xml'(文本特征命中)。
|
||||
*/
|
||||
export function detectMediaType(buf: Buffer): { type: string; svg: boolean } | null {
|
||||
for (const sig of SIGNATURES) {
|
||||
if (buf.length >= sig.bytes.length && buf.subarray(0, sig.bytes.length).equals(sig.bytes)) {
|
||||
return { type: sig.mime, svg: false }
|
||||
}
|
||||
}
|
||||
// WEBP:RIFF 容器头(4B 长度)+ 'WEBP' Magic(@8)——RIFF 家族其他成员(WAV/AVI)不误报
|
||||
if (buf.length >= 12 && buf.subarray(0, 4).equals(RIFF) && buf.subarray(8, 12).equals(WEBP)) {
|
||||
return { type: 'image/webp', svg: false }
|
||||
}
|
||||
const head = buf.subarray(0, Math.min(buf.length, SVG_WINDOW_BYTES)).toString('latin1').toLowerCase()
|
||||
if (head.includes('<svg')) return { type: 'image/svg+xml', svg: true }
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1,617 @@
|
||||
// 媒体登记写入器(Plan 3 Task 4,spec §5.5/§11.1、前置事实 7/11、评审 Issue 1/3):
|
||||
// 消费适配器事件流的**双事件**面——attachment 行建资产(media_assets)、内容行提取引用
|
||||
// (media_usages)——媒体域的领域 owner(Task 6 分派规则:content-writer 对 attachment 行
|
||||
// 只留 source_records 足迹,media_assets 的唯一写入方在此)。
|
||||
//
|
||||
// 三级对拍(前置事实 7 钉死,真实数据实证:媒体目录扁平 GUID 命名、与 WXR
|
||||
// attachment_url 零精确命中,唯一映射在旧 cms 库侧车):
|
||||
// ① uploads 相对路径清单精确命中:attachment_url 掐掉
|
||||
// '…/wp-content/uploads/' 前缀 → manifest[path](清单 = media_bundle 快照 metadata.
|
||||
// manifest,{path, bytes, sha256});
|
||||
// ② 侧车命中:manifest miss → sidecarMap[source_url] → filePath → 清单实体(精确路径
|
||||
// miss 时按**文件名**候选——真实侧车 file_path 是扁平文件名而清单可含子目录;候选
|
||||
// 多条 = 歧义,不自动选、全部列 issue、不插行,spec §5.5);
|
||||
// ③ missing:两级皆 miss → 只进报告(**不插 media_assets 行**——sha256 NOT NULL 无清单
|
||||
// 实体可取,行插不进是库层防线不是巧合)。
|
||||
// 命中行取值:public_path = 旧 uploads 路径(保旧 URL 语义)、sha256/byte_size 从清单、
|
||||
// storage_path = bundle 副本相对路径 + 清单 path(绝对副本 = metadata.storageRoot + 此列)。
|
||||
//
|
||||
// 魔数安全校验(spec §11.1):命中实体读存储副本头部实际字节(detectMediaType)——
|
||||
// - SVG 检出(文本特征)→ status='quarantined':行存在(登记可审计)但不进产物语义,
|
||||
// 其引用不建 media_usages(usage 是发布面消费的引用关系,隔离资产不得经此进产物);
|
||||
// - 未知签名 → status='unsupported' + issue(不猜格式);
|
||||
// - 声明 MIME(清单/WXR 均无 MIME——声明侧由 URL 扩展名推断)与检测不符 → issue
|
||||
// (仍登记 ready:内容真实、扩展名说谎是数据质量问题不是安全风险)。
|
||||
//
|
||||
// usage 时序(设计决定,WXR 文档序钉死):items 先于 attachment——引用先行、资产后到,
|
||||
// 故 usage 建立分两阶段:内容行到达时提取引用并落修订锚点(source_records 六列身份 →
|
||||
// entry_locales.current_revision_id——**依赖 content-writer 先处理同事件**,Task 6 编排
|
||||
// 顺序契约),资产解析统一推迟到 finish() 对 media_assets 反查(id 反查 source_id /
|
||||
// URL 反查 source_url→public_path)——两种文档序终态一致。引用无对应资产且属本站媒体域
|
||||
// (id 引用或 uploads 路径 URL)→ mediaDanglingRefs 计数 + issue(§7.6 禁静默);外域
|
||||
// 热链 URL 不属本域对拍范围,不计悬空(无清单可比对,静默越过)。
|
||||
//
|
||||
// 幂等:media_assets 唯一 (site, source_url) onConflictDoNothing(重跑零增行);
|
||||
// media_usages 复合主键 (asset, revision, field_path) onConflictDoNothing——field_path
|
||||
// 由提取序确定(featured/gallery 序号/正文 img 序号),重跑同流同值。事务边界:全部
|
||||
// 单语句原子操作(ctx.db 直连约定,无跨语句不变量——与 term-writer 同纪律)。
|
||||
import { open, readFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { and, eq } from 'drizzle-orm'
|
||||
import type { AdapterEvent } from '@sf/adapter-wp'
|
||||
import type { UniversalRow } from '@sf/contracts'
|
||||
import { entryLocales, mediaAssets, mediaUsages, sourceRecords } from '../../db/schema.js'
|
||||
import { isUniqueViolation, type MediaManifestEntry } from '../source/register.service.js'
|
||||
import { detectMediaType } from './magic-bytes.js'
|
||||
import type { ImportDb, ImportJobContext, ImportPipeline, ImportPipelineOutcome } from './import.service.js'
|
||||
|
||||
const UPLOADS_MARKER = '/wp-content/uploads/'
|
||||
|
||||
/** 未知签名的 detected_mime 落值(列 NOT NULL——不猜格式但要有确定值) */
|
||||
const UNKNOWN_DETECTED_MIME = 'application/octet-stream'
|
||||
|
||||
/** 魔数读取的头部字节(SVG 文本窗口 1KB + 余量) */
|
||||
const HEAD_BYTES = 4096
|
||||
|
||||
/** URL 扩展名 → 声明 MIME(声明侧推断表——与 detectMediaType 的检出表独立) */
|
||||
const EXT_MIME: Readonly<Record<string, string>> = Object.freeze({
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
gif: 'image/gif',
|
||||
webp: 'image/webp',
|
||||
pdf: 'application/pdf',
|
||||
svg: 'image/svg+xml',
|
||||
})
|
||||
|
||||
/**
|
||||
* 可分派的媒体写入器句柄(Task 6 编排形状,与 term/content 同构——但 finish 是异步:
|
||||
* usage 解析推迟到流终,见模块头"usage 时序")。句柄非可重入:一个句柄只服务一遍流。
|
||||
*/
|
||||
export interface MediaEventHandler {
|
||||
/** 单事件分派(term/unsupported/error 为 no-op——无媒体信息) */
|
||||
handleEvent(ev: AdapterEvent): Promise<void>
|
||||
/** 流终:解析悬置引用 → media_usages / 悬空报告,取回累计产出(调用后句柄弃用) */
|
||||
finish(): Promise<ImportPipelineOutcome>
|
||||
}
|
||||
|
||||
/**
|
||||
* 本写入器的 counts 键与归属(**counts 键是跨写入器的扁平全局命名空间**——import.service
|
||||
* 按返回值覆盖累加器同名键合并,各写入器必须各持独占键名,见 ImportPipelineOutcome.counts
|
||||
* 的键契约注记)。本写入器独占六键(media 前缀,**勿复用 content-writer 的 'skipped'**):
|
||||
* - mediaRegistered:本运行**新插**的 media_assets 行数(含 quarantined/unsupported——
|
||||
* 行已登记;幂等重跑为 0);
|
||||
* - mediaFromSidecar:其中经级②侧车解析的行数(mediaRegistered 的子集);
|
||||
* - mediaMissing:级③ missing 的 attachment 数(按事件计——重跑如实再现);
|
||||
* - mediaQuarantined:本运行检出 SVG 的 attachment 数(按事件计——重跑如实再现,
|
||||
* 与 mediaRegistered 在重跑时不再严格子集:行幂等而检出如实);
|
||||
* - mediaDanglingRefs:内容引用未解析到资产的条数(按引用计——重跑如实再现);
|
||||
* - mediaAmbiguous:多候选歧义的 attachment 数(按事件计,不插行)。
|
||||
* issues:missing/歧义/隔离/未知签名/MIME 不符/悬空引用/副本读取失败/public_path 唯一键
|
||||
* 冲突——全部显式报告(§7.6 禁静默)。
|
||||
*/
|
||||
interface MediaWriterCounts {
|
||||
mediaRegistered: number
|
||||
mediaFromSidecar: number
|
||||
mediaMissing: number
|
||||
mediaQuarantined: number
|
||||
mediaDanglingRefs: number
|
||||
mediaAmbiguous: number
|
||||
}
|
||||
|
||||
/** 悬置引用(内容行提取、流终解析):锚点 + 引用键两半 */
|
||||
interface PendingRef {
|
||||
/** 引用方修订(content-writer 落的 current_revision_id——usage 的不可变锚) */
|
||||
revisionId: string
|
||||
/** 修订数据域内的字段位(同流重跑稳定——复合主键的 field_path 侧) */
|
||||
fieldPath: string
|
||||
kind: 'id' | 'url'
|
||||
/** id 引用 = attachment post id;URL 引用 = 引文原串 */
|
||||
value: string
|
||||
/** URL 引用属本站 uploads 域时的公开路径形态(悬空裁决域);id 引用恒属本域 */
|
||||
uploadsPublicPath: string | null
|
||||
rawLocation: string
|
||||
}
|
||||
|
||||
/** 侧车 JSON 条目形状(register --media-map 已在登记期校验,此处只做读取防御) */
|
||||
interface SidecarEntry {
|
||||
sourceUrl: string
|
||||
filePath: string
|
||||
}
|
||||
|
||||
export function createMediaEventHandler(identity: {
|
||||
db: ImportDb
|
||||
siteId: string
|
||||
connectionId: string
|
||||
snapshotId: string
|
||||
/** 引用行 locale 校验基准(与 content-writer 同拒收口径——不双报) */
|
||||
defaultLocale: string
|
||||
/** 存储副本绝对根(media_bundle 快照 metadata.storageRoot——登记期事实) */
|
||||
storageRoot: string
|
||||
/** media_bundle 快照的 storagePath(相对 storageRoot;直传清单场景传 '') */
|
||||
mediaBundleStoragePath: string
|
||||
/** 媒体清单(media_bundle metadata.manifest;缺席 = 空表 → 全部走级③) */
|
||||
mediaManifest: MediaManifestEntry[]
|
||||
/** 侧车映射(manual_url_list:source_url → file_path) */
|
||||
sidecarMap: Map<string, string>
|
||||
}): MediaEventHandler {
|
||||
const { db, siteId } = identity
|
||||
const counts: MediaWriterCounts & Record<string, number> = {
|
||||
mediaRegistered: 0,
|
||||
mediaFromSidecar: 0,
|
||||
mediaMissing: 0,
|
||||
mediaQuarantined: 0,
|
||||
mediaDanglingRefs: 0,
|
||||
mediaAmbiguous: 0,
|
||||
}
|
||||
const issues: string[] = []
|
||||
const pendingRefs: PendingRef[] = []
|
||||
|
||||
// 清单索引:精确路径 + 文件名候选(级②的侧车 file_path 是扁平名,清单可含子目录)
|
||||
const manifestByPath = new Map<string, MediaManifestEntry>()
|
||||
const manifestByBasename = new Map<string, MediaManifestEntry[]>()
|
||||
for (const entry of identity.mediaManifest) {
|
||||
manifestByPath.set(entry.path, entry)
|
||||
const base = basenameOf(entry.path)
|
||||
const bucket = manifestByBasename.get(base)
|
||||
if (bucket === undefined) manifestByBasename.set(base, [entry])
|
||||
else bucket.push(entry)
|
||||
}
|
||||
|
||||
function basenameOf(path: string): string {
|
||||
const idx = path.lastIndexOf('/')
|
||||
return idx === -1 ? path : path.slice(idx + 1)
|
||||
}
|
||||
|
||||
/** 扩展名推断声明 MIME(无扩展名/表外扩展 → null:列可空,无声明则无对拍) */
|
||||
function declaredMimeOf(pathLike: string): string | null {
|
||||
const m = /\.([a-z0-9]+)$/i.exec(pathLike)
|
||||
if (m === null) return null
|
||||
const hit = EXT_MIME[m[1]!.toLowerCase()]
|
||||
return hit ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* attachment_url → 旧 uploads 相对路径。无 uploads 标记的 URL(外站迁移源)级①无从
|
||||
* 掐前缀:relPath=null(级①跳过),source/public 路径以 URL pathname 兜底。
|
||||
* 注:不做百分号解码——清单 path 是字面文件名,编码差异按 miss 如实进报告。
|
||||
*/
|
||||
function uploadsRelOf(fileUrl: string): { relPath: string | null; pathname: string } {
|
||||
const idx = fileUrl.indexOf(UPLOADS_MARKER)
|
||||
if (idx !== -1) {
|
||||
const rel = fileUrl.slice(idx + UPLOADS_MARKER.length)
|
||||
if (rel !== '') return { relPath: rel, pathname: fileUrl.slice(idx) }
|
||||
}
|
||||
try {
|
||||
return { relPath: null, pathname: new URL(fileUrl).pathname }
|
||||
} catch {
|
||||
// 非 URL 形态的原始串: pathname 兜底保留原文(可审计),级①不可用
|
||||
return { relPath: null, pathname: fileUrl.startsWith('/') ? fileUrl : `/${fileUrl}` }
|
||||
}
|
||||
}
|
||||
|
||||
/** 存储副本头部读取(魔数只需前若干字节——171MB 级文件不做全量读) */
|
||||
async function readHead(abs: string): Promise<Buffer> {
|
||||
const handle = await open(abs, 'r')
|
||||
try {
|
||||
const buf = Buffer.alloc(HEAD_BYTES)
|
||||
const { bytesRead } = await handle.read(buf, 0, HEAD_BYTES, 0)
|
||||
return buf.subarray(0, bytesRead)
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* attachment 行主路径:三级解析 → 魔数 → 插 media_assets。
|
||||
* locale 拒收口径与 content-writer 一致(该行已被 content-writer 拒收并记 issue,
|
||||
* 此处静默跳过不双报——attachment 行的媒体域处理同样以 locale 合法为前提)。
|
||||
*/
|
||||
async function handleAttachmentRow(row: UniversalRow): Promise<void> {
|
||||
if (row.locale !== identity.defaultLocale) return
|
||||
const fileUrl = typeof row.knownFields.attachmentUrl === 'string' ? row.knownFields.attachmentUrl : ''
|
||||
if (fileUrl === '') {
|
||||
counts.mediaMissing++
|
||||
issues.push(
|
||||
`attachment 缺 attachment_url(${row.sourceType}/${row.sourceId},${row.rawLocation})——三级对拍无从下手,按 missing 报告`,
|
||||
)
|
||||
return
|
||||
}
|
||||
const { relPath, pathname } = uploadsRelOf(fileUrl)
|
||||
|
||||
// 级①:uploads 相对路径清单精确命中
|
||||
let entry: MediaManifestEntry | null = relPath === null ? null : manifestByPath.get(relPath) ?? null
|
||||
let fromSidecar = false
|
||||
if (entry === null) {
|
||||
// 级②:侧车 source_url → file_path → 清单实体(精确路径 → 文件名候选)
|
||||
const filePath = identity.sidecarMap.get(fileUrl)
|
||||
if (filePath !== undefined) {
|
||||
const exact = manifestByPath.get(filePath)
|
||||
if (exact !== undefined) {
|
||||
entry = exact
|
||||
fromSidecar = true
|
||||
} else {
|
||||
const candidates = manifestByBasename.get(basenameOf(filePath)) ?? []
|
||||
if (candidates.length > 1) {
|
||||
// 歧义不自动选(spec §5.5):全部候选列报告,不插行
|
||||
counts.mediaAmbiguous++
|
||||
issues.push(
|
||||
`媒体多候选歧义(${row.sourceType}/${row.sourceId},${fileUrl}):侧车 file_path=${filePath} 按文件名命中 ${candidates.length} 条清单实体——[${candidates.map((c) => c.path).join('、')}]——不自动选择,未登记`,
|
||||
)
|
||||
return
|
||||
}
|
||||
if (candidates.length === 1) {
|
||||
entry = candidates[0]!
|
||||
fromSidecar = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (entry === null) {
|
||||
// 级③:missing——只进报告不插行(sha256 NOT NULL:无清单实体插不进,库层防线)
|
||||
counts.mediaMissing++
|
||||
issues.push(
|
||||
`媒体 missing(${row.sourceType}/${row.sourceId},${fileUrl}):清单与侧车均未命中——不插 media_assets 行(sha256 无来源)`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// 魔数:读存储副本实际字节(storageRoot + bundle 相对路径 + 清单 path)
|
||||
const storagePath =
|
||||
identity.mediaBundleStoragePath === ''
|
||||
? entry.path
|
||||
: `${identity.mediaBundleStoragePath}/${entry.path}`
|
||||
let head: Buffer
|
||||
try {
|
||||
head = await readHead(join(identity.storageRoot, storagePath))
|
||||
} catch (err) {
|
||||
issues.push(
|
||||
`媒体副本读取失败(${row.sourceType}/${row.sourceId},${fileUrl} → ${storagePath}):${err instanceof Error ? err.message : String(err)}——清单声称存在而副本缺席,未登记`,
|
||||
)
|
||||
return
|
||||
}
|
||||
const detected = detectMediaType(head)
|
||||
const publicPath = relPath === null ? pathname : `${UPLOADS_MARKER}${relPath}`
|
||||
const declared = declaredMimeOf(publicPath)
|
||||
|
||||
let status: 'ready' | 'quarantined' | 'unsupported'
|
||||
let detectedMime: string
|
||||
if (detected === null) {
|
||||
status = 'unsupported'
|
||||
detectedMime = UNKNOWN_DETECTED_MIME
|
||||
issues.push(
|
||||
`媒体未知签名(${row.sourceType}/${row.sourceId},${publicPath}):头部字节不匹配签名表——status=unsupported,不猜格式`,
|
||||
)
|
||||
} else if (detected.svg) {
|
||||
status = 'quarantined'
|
||||
detectedMime = detected.type
|
||||
counts.mediaQuarantined++
|
||||
issues.push(
|
||||
`SVG 检出已隔离(${row.sourceType}/${row.sourceId},${publicPath}):文本 <svg 特征命中——status=quarantined,不进产物语义(spec §11.1)`,
|
||||
)
|
||||
} else {
|
||||
status = 'ready'
|
||||
detectedMime = detected.type
|
||||
if (declared !== null && declared !== detected.type) {
|
||||
issues.push(
|
||||
`媒体声明与检测不符(${row.sourceType}/${row.sourceId},${publicPath}):扩展名推断 ${declared} ≠ 魔数检测 ${detected.type}——按检测登记(ready),数据质量问题已报告`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const inserted = await db
|
||||
.insert(mediaAssets)
|
||||
.values({
|
||||
siteId,
|
||||
sourceConnectionId: identity.connectionId,
|
||||
sourceId: row.sourceId,
|
||||
sourceUrl: fileUrl,
|
||||
sourcePath: relPath ?? pathname.replace(/^\//, ''),
|
||||
storagePath,
|
||||
publicPath,
|
||||
sha256: entry.sha256,
|
||||
detectedMimeType: detectedMime,
|
||||
declaredMimeType: declared,
|
||||
byteSize: entry.bytes,
|
||||
status,
|
||||
})
|
||||
// 唯一 (site, source_url) 定向幂等:命中零增行;public_path 唯一键冲突**不放行**
|
||||
// (同公开路径双源是真实冲突,须显式报告而非静默吞掉)
|
||||
.onConflictDoNothing({ target: [mediaAssets.siteId, mediaAssets.sourceUrl] })
|
||||
.returning({ id: mediaAssets.id })
|
||||
if (inserted.length > 0) {
|
||||
counts.mediaRegistered++
|
||||
if (fromSidecar) counts.mediaFromSidecar++
|
||||
}
|
||||
} catch (err) {
|
||||
if (isUniqueViolation(err)) {
|
||||
issues.push(
|
||||
`媒体 public_path 冲突(${row.sourceType}/${row.sourceId},${publicPath}):同站公开路径已被他行占用——未登记(不静默覆盖)`,
|
||||
)
|
||||
return
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 内容行引用提取(usage 两阶段的第一阶段):refs + bodyHtml img src → 悬置引用。
|
||||
* featured/gallery 取 id 锚(WP meta 的 _thumbnail_id/_product_image_gallery);未知
|
||||
* role 以 refs 原始下标落位(field_path 稳定优先于语义命名)。
|
||||
*/
|
||||
function extractRefs(row: UniversalRow): Array<Omit<PendingRef, 'revisionId'>> {
|
||||
const out: Array<Omit<PendingRef, 'revisionId'>> = []
|
||||
let galleryOrdinal = 0
|
||||
row.attachmentRefs.forEach((ref, i) => {
|
||||
let fieldPath: string
|
||||
if (ref.role === 'featured') fieldPath = 'attachmentRefs.featured'
|
||||
else if (ref.role === 'gallery') fieldPath = `attachmentRefs.gallery[${galleryOrdinal++}]`
|
||||
else fieldPath = `attachmentRefs.raw[${i}]`
|
||||
if (ref.sourceId !== undefined && ref.sourceId !== '') {
|
||||
out.push({ fieldPath, kind: 'id', value: ref.sourceId, uploadsPublicPath: null, rawLocation: row.rawLocation })
|
||||
} else if (ref.sourceUrl !== undefined && ref.sourceUrl !== '') {
|
||||
out.push({
|
||||
fieldPath,
|
||||
kind: 'url',
|
||||
value: ref.sourceUrl,
|
||||
uploadsPublicPath: uploadsPublicPathOf(ref.sourceUrl),
|
||||
rawLocation: row.rawLocation,
|
||||
})
|
||||
}
|
||||
// 双锚皆空:adapter 层已按非法 refs 记 issue(事件 issues 面),此处不重复报
|
||||
})
|
||||
const body = row.bodyHtml ?? ''
|
||||
if (body !== '') {
|
||||
const imgRe = /<img\b[^>]*?\bsrc\s*=\s*["']([^"']+)["']/gi
|
||||
let m: RegExpExecArray | null
|
||||
let imgOrdinal = 0
|
||||
while ((m = imgRe.exec(body)) !== null) {
|
||||
const src = m[1]!
|
||||
if (!src.startsWith('data:')) {
|
||||
// data: URI 是内联数据非资产引用;其余(绝对/相对/协议相对)统一进解析
|
||||
out.push({
|
||||
fieldPath: `data.bodyHtml.img[${imgOrdinal++}]`,
|
||||
kind: 'url',
|
||||
value: src,
|
||||
uploadsPublicPath: uploadsPublicPathOf(src),
|
||||
rawLocation: row.rawLocation,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** URL → 本站 uploads 公开路径形态(无标记 → null:外域热链不属悬空裁决域) */
|
||||
function uploadsPublicPathOf(url: string): string | null {
|
||||
const idx = url.indexOf(UPLOADS_MARKER)
|
||||
return idx === -1 ? null : url.slice(idx)
|
||||
}
|
||||
|
||||
/** 修订锚点:source_records 六列身份 → entry_locales.current_revision_id(content-writer 接缝) */
|
||||
async function resolveRevisionAnchor(row: UniversalRow): Promise<string | null> {
|
||||
const recs = await db
|
||||
.select({ contentEntryId: sourceRecords.contentEntryId })
|
||||
.from(sourceRecords)
|
||||
.where(
|
||||
and(
|
||||
eq(sourceRecords.siteId, siteId),
|
||||
eq(sourceRecords.sourceConnectionId, identity.connectionId),
|
||||
eq(sourceRecords.sourceType, row.sourceType),
|
||||
eq(sourceRecords.sourceId, row.sourceId),
|
||||
eq(sourceRecords.locale, row.locale),
|
||||
eq(sourceRecords.sourceSnapshotId, identity.snapshotId),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
const entryId = recs[0]?.contentEntryId
|
||||
if (entryId === undefined || entryId === null) return null
|
||||
const els = await db
|
||||
.select({ currentRevisionId: entryLocales.currentRevisionId })
|
||||
.from(entryLocales)
|
||||
.where(
|
||||
and(
|
||||
eq(entryLocales.siteId, siteId),
|
||||
eq(entryLocales.contentEntryId, entryId),
|
||||
eq(entryLocales.locale, row.locale),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
return els[0]?.currentRevisionId ?? null
|
||||
}
|
||||
|
||||
/** 内容行(非 attachment):locale 口径对齐 content-writer(拒收行不双报),提取引用挂锚 */
|
||||
async function handleContentRowRefs(row: UniversalRow): Promise<void> {
|
||||
if (row.locale !== identity.defaultLocale) return
|
||||
const refs = extractRefs(row)
|
||||
if (refs.length === 0) return
|
||||
const anchor = await resolveRevisionAnchor(row)
|
||||
if (anchor === null) {
|
||||
issues.push(
|
||||
`媒体引用行无修订锚点(${row.sourceType}/${row.sourceId}/${row.locale},refs=${refs.length}):source_records/entry_locales 无 current 指针(content-writer 未处理该行?)——usage 跳过`,
|
||||
)
|
||||
return
|
||||
}
|
||||
for (const ref of refs) pendingRefs.push({ ...ref, revisionId: anchor })
|
||||
}
|
||||
|
||||
/** 悬置引用 → 资产反查(id → source_id;URL → source_url 精确 → public_path 形态) */
|
||||
async function lookupAsset(
|
||||
ref: PendingRef,
|
||||
): Promise<{ id: string; sourceUrl: string; status: string } | null> {
|
||||
const columns = {
|
||||
id: mediaAssets.id,
|
||||
sourceUrl: mediaAssets.sourceUrl,
|
||||
status: mediaAssets.status,
|
||||
}
|
||||
if (ref.kind === 'id') {
|
||||
const rows = await db
|
||||
.select(columns)
|
||||
.from(mediaAssets)
|
||||
.where(
|
||||
and(
|
||||
eq(mediaAssets.siteId, siteId),
|
||||
eq(mediaAssets.sourceConnectionId, identity.connectionId),
|
||||
eq(mediaAssets.sourceId, ref.value),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
return rows[0] ?? null
|
||||
}
|
||||
const byUrl = await db
|
||||
.select(columns)
|
||||
.from(mediaAssets)
|
||||
.where(and(eq(mediaAssets.siteId, siteId), eq(mediaAssets.sourceUrl, ref.value)))
|
||||
.limit(1)
|
||||
if (byUrl.length > 0) return byUrl[0]!
|
||||
if (ref.uploadsPublicPath !== null) {
|
||||
const byPath = await db
|
||||
.select(columns)
|
||||
.from(mediaAssets)
|
||||
.where(and(eq(mediaAssets.siteId, siteId), eq(mediaAssets.publicPath, ref.uploadsPublicPath)))
|
||||
.limit(1)
|
||||
return byPath[0] ?? null
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
async handleEvent(ev) {
|
||||
if (ev.kind !== 'row') return // term/unsupported/error:无媒体信息,本写入器不消费
|
||||
if (ev.row.sourceType === 'wp:attachment') {
|
||||
await handleAttachmentRow(ev.row)
|
||||
return
|
||||
}
|
||||
await handleContentRowRefs(ev.row)
|
||||
},
|
||||
|
||||
/** 流终解析:悬置引用逐条对资产反查——命中且 ready → usage;无资产且属本域 → 悬空报告 */
|
||||
async finish() {
|
||||
for (const ref of pendingRefs) {
|
||||
const asset = await lookupAsset(ref)
|
||||
if (asset === null) {
|
||||
if (ref.kind === 'id' || ref.uploadsPublicPath !== null) {
|
||||
counts.mediaDanglingRefs++
|
||||
const target = ref.kind === 'id' ? `attachment ${ref.value}` : ref.value
|
||||
issues.push(
|
||||
`媒体引用悬空(${ref.rawLocation},${ref.fieldPath} → ${target}):无对应 attachment/清单记录——引用保留于报告,不静默丢弃`,
|
||||
)
|
||||
}
|
||||
// 外域 URL(无 uploads 标记):站外热链不属本域对拍范围,静默越过(JSDoc 模块头)
|
||||
continue
|
||||
}
|
||||
if (asset.status !== 'ready') {
|
||||
// quarantined/unsupported 资产:不进产物语义——引用关系不建 usage(资产侧
|
||||
// issue 已在登记时报告;引用命中非悬空)
|
||||
continue
|
||||
}
|
||||
await db
|
||||
.insert(mediaUsages)
|
||||
.values({
|
||||
siteId,
|
||||
mediaAssetId: asset.id,
|
||||
contentRevisionId: ref.revisionId,
|
||||
fieldPath: ref.fieldPath,
|
||||
// id 引用无引文 URL——落资产登记的 source_url(审计可追);URL 引用落引文原串
|
||||
sourceUrl: ref.kind === 'url' ? ref.value : asset.sourceUrl,
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
}
|
||||
return { counts, issues }
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从管道上下文构造媒体写入器句柄(Task 6 编排注入位):manifest/sidecar 从快照 metadata
|
||||
* 与存储副本解析(media_bundle.metadata.manifest / manual_url_list 副本 JSON)。
|
||||
* bundle 缺席 = 空清单(三级全走 missing——诚实报告而非跳过);侧车缺席 = 空映射。
|
||||
*/
|
||||
export async function createMediaEventHandlerFromContext(
|
||||
ctx: Pick<
|
||||
ImportJobContext,
|
||||
| 'db'
|
||||
| 'siteId'
|
||||
| 'connectionId'
|
||||
| 'defaultLocale'
|
||||
| 'wxrSnapshot'
|
||||
| 'mediaBundleSnapshot'
|
||||
| 'manualUrlListSnapshot'
|
||||
>,
|
||||
): Promise<MediaEventHandler> {
|
||||
const bundle = ctx.mediaBundleSnapshot
|
||||
const manifestRaw = bundle?.metadata.manifest
|
||||
const manifest: MediaManifestEntry[] = Array.isArray(manifestRaw) ? (manifestRaw as MediaManifestEntry[]) : []
|
||||
let storageRoot = ''
|
||||
if (manifest.length > 0) {
|
||||
const raw = bundle?.metadata.storageRoot
|
||||
if (typeof raw !== 'string' || raw === '') {
|
||||
throw new Error('media_bundle 快照 metadata 缺 storageRoot(登记期事实缺席——无法读取存储副本做魔数校验)')
|
||||
}
|
||||
storageRoot = raw
|
||||
}
|
||||
return createMediaEventHandler({
|
||||
db: ctx.db,
|
||||
siteId: ctx.siteId,
|
||||
connectionId: ctx.connectionId,
|
||||
snapshotId: ctx.wxrSnapshot.id,
|
||||
defaultLocale: ctx.defaultLocale,
|
||||
storageRoot,
|
||||
mediaBundleStoragePath: bundle?.storagePath ?? '',
|
||||
mediaManifest: manifest,
|
||||
sidecarMap: await loadSidecarMap(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
/** 侧车加载:manual_url_list 副本 JSON → Map(source_url → file_path)。形状在登记期已校验 */
|
||||
async function loadSidecarMap(
|
||||
ctx: Pick<ImportJobContext, 'manualUrlListSnapshot'>,
|
||||
): Promise<Map<string, string>> {
|
||||
const snap = ctx.manualUrlListSnapshot
|
||||
if (snap === null) return new Map()
|
||||
const root = snap.metadata.storageRoot
|
||||
if (typeof root !== 'string' || root === '') {
|
||||
throw new Error('manual_url_list 快照 metadata 缺 storageRoot(登记期事实缺席——无法读取侧车副本)')
|
||||
}
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(join(root, snap.storagePath), 'utf8'))
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`侧车快照副本解析失败(${snap.storagePath}):${err instanceof Error ? err.message : String(err)}`,
|
||||
)
|
||||
}
|
||||
if (!Array.isArray(parsed)) {
|
||||
throw new Error(`侧车快照副本形状异常(${snap.storagePath}):顶层不是数组`)
|
||||
}
|
||||
const map = new Map<string, string>()
|
||||
for (const item of parsed) {
|
||||
const entry = item as Partial<SidecarEntry>
|
||||
if (
|
||||
typeof entry.sourceUrl !== 'string' ||
|
||||
entry.sourceUrl === '' ||
|
||||
typeof entry.filePath !== 'string' ||
|
||||
entry.filePath === ''
|
||||
) {
|
||||
throw new Error(`侧车快照副本形状异常(${snap.storagePath}):存在非 {sourceUrl, filePath} 条目`)
|
||||
}
|
||||
map.set(entry.sourceUrl, entry.filePath)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
/**
|
||||
* 媒体登记管道写入器({@link createMediaEventHandler} 的自消费整流便捷壳):单遍拉完
|
||||
* ctx.events、逐事件分派、流终返回 outcome。与 content-writer 组合消费同流时须用
|
||||
* {@link createMediaEventHandlerFromContext} 句柄形状(单消费方拉流分派,无 tee)。
|
||||
*/
|
||||
export function createMediaWriter(): ImportPipeline {
|
||||
return async (ctx) => {
|
||||
const handler = await createMediaEventHandlerFromContext(ctx)
|
||||
for await (const ev of ctx.events) {
|
||||
await handler.handleEvent(ev)
|
||||
}
|
||||
return handler.finish()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
// Plan 3 Task 4 失败测试(先于实现):媒体登记写入器(三级对拍 + 魔数/隔离 + 悬空引用)。
|
||||
// 覆盖契约(对照计划 Task 4 与前置事实 7/11、spec §5.5/§11.1/§7.6):
|
||||
// 1. mini 全链三级对拍:①uploads 相对路径清单精确命中(fixture-a)②侧车 source_url→
|
||||
// file_path→清单实体(fixture-b——清单故意 miss 该路径逼走侧车级)③missing 不插行
|
||||
// (media_assets sha256 NOT NULL——另测无 bundle 场景);命中行 public_path=旧 uploads
|
||||
// 路径、sha256/byte_size/storage_path 从清单取;media_usages 三行(featured/gallery/
|
||||
// 正文 img——从 row.attachmentRefs + bodyHtml 提取,revision 锚点经 source_records→
|
||||
// entry_locales.current_revision_id);
|
||||
// 2. 魔数(集成面):命中文件读实际字节——PNG 过;扩展名声明 MIME 与检测不符 → issue;
|
||||
// SVG 检出(.png 伪装与显式 .svg 双例)→ status='quarantined' + issue(不进产物语义);
|
||||
// 未知签名 → status='unsupported' + issue;quarantined 资产的引用不建 usage;
|
||||
// 3. 悬空引用(mini 203/204):gallery 引用无对应 attachment/清单记录 → mediaDanglingRefs
|
||||
// 计数 + issue(不静默丢弃,§7.6);
|
||||
// 4. 多候选:侧车 file_path 按文件名命中两条清单(同 filename 不同目录)→ 歧义不自动选、
|
||||
// 不插行、全部候选列 issue(spec §5.5);
|
||||
// 5. 幂等:重跑 media_assets/media_usages 零增行(mediaRegistered=0;悬空引用按流如实再现);
|
||||
// 6. counts 键(media 前缀全局命名空间):mediaRegistered/mediaFromSidecar/mediaMissing/
|
||||
// mediaQuarantined/mediaDanglingRefs/mediaAmbiguous——与 content-writer 键并存无碰撞。
|
||||
// 双事件消费设计(实现侧 JSDoc 同步):attachment 行建资产、内容行提取引用;WXR 文档序
|
||||
// items 先于 attachment——引用先行资产后到,usage 解析在 finish() 统一对 media_assets
|
||||
// 反查(顺序无关)。运行于 globalSetup 临时库(ADR-015);报告根 os.tmpdir,finally 清理。
|
||||
import { createHash } from 'node:crypto'
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { runWxrAdapter, type AdapterEvent } from '@sf/adapter-wp'
|
||||
import type { UniversalRow } from '@sf/contracts'
|
||||
import type { MediaManifestEntry } from '../../apps/api/src/modules/source/register.service.js'
|
||||
import type { ImportPipeline } from '../../apps/api/src/modules/import/import.service.js'
|
||||
import { createImportJob } from '../../apps/api/src/modules/import/import.service.js'
|
||||
import { createContentEventHandler } from '../../apps/api/src/modules/import/content-writer.js'
|
||||
import { createMediaEventHandler, createMediaEventHandlerFromContext } from '../../apps/api/src/modules/import/media-writer.js'
|
||||
import { registerSource } from '../../apps/api/src/modules/source/register.service.js'
|
||||
import { rand, withImmutableCleanup, withTestDb, type Db } from './helpers.js'
|
||||
|
||||
const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url))
|
||||
const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml')
|
||||
|
||||
const PNG_SIG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
||||
const JPEG_SIG = Buffer.from([0xff, 0xd8, 0xff, 0xe0])
|
||||
|
||||
/** 伪 PNG/JPEG 体(签名 + 唯一种子载荷——魔数只看头,内容唯一避免跨测试清单撞哈希) */
|
||||
function pngBytes(seed: string): Buffer {
|
||||
return Buffer.concat([PNG_SIG, Buffer.from(`png-body:${seed}`, 'latin1')])
|
||||
}
|
||||
function jpegBytes(seed: string): Buffer {
|
||||
return Buffer.concat([JPEG_SIG, Buffer.from(`jpeg-body:${seed}`, 'latin1')])
|
||||
}
|
||||
function sha256of(buf: Buffer): string {
|
||||
return createHash('sha256').update(buf).digest('hex')
|
||||
}
|
||||
function manifestEntryOf(path: string, buf: Buffer): MediaManifestEntry {
|
||||
return { path, bytes: buf.length, sha256: sha256of(buf) }
|
||||
}
|
||||
|
||||
/** 手构事件流的 AsyncIterable 包装(每次调用产新迭代器——重跑各流独立) */
|
||||
async function* eventsOf(events: AdapterEvent[]): AsyncGenerator<AdapterEvent, void, void> {
|
||||
yield* events
|
||||
}
|
||||
|
||||
/** db.execute 行对象规整(postgres.js 原型差异会让 toStrictEqual 报"无视觉差异"失败) */
|
||||
function plain(rows: any[]): any[] {
|
||||
return rows.map((r) => ({ ...r }))
|
||||
}
|
||||
|
||||
/** 事件流收集(mini 全链一饮而尽) */
|
||||
async function collectMiniEvents(): Promise<AdapterEvent[]> {
|
||||
const out: AdapterEvent[] = []
|
||||
for await (const ev of runWxrAdapter(createReadStream(WXR_FIXTURE), { mapConfig: { locale: 'en' } })) {
|
||||
out.push(ev)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** 手构 attachment row 事件(knownFields.attachmentUrl 是三级对拍的输入锚点) */
|
||||
function attachmentEvent(spec: { sourceId: string; url: string }): AdapterEvent {
|
||||
const row: UniversalRow = {
|
||||
sourceType: 'wp:attachment',
|
||||
sourceId: spec.sourceId,
|
||||
status: 'inherit',
|
||||
locale: 'en',
|
||||
taxonomyRefs: [],
|
||||
attachmentRefs: [],
|
||||
seo: {},
|
||||
knownFields: { attachmentUrl: spec.url },
|
||||
extras: {},
|
||||
rawLocation: `wxr://item/${spec.sourceId}`,
|
||||
}
|
||||
return { kind: 'row', row, issues: [], sanitizeDiffs: [], changed: false }
|
||||
}
|
||||
|
||||
/** Task 6 编排雏形:单消费方拉流,逐事件先 content 后 media 分派(content 先行是
|
||||
* media-writer usage 锚点的顺序契约——source_records/revision 须已落);两态经 finish 取回 */
|
||||
function contentPlusMediaPipeline(): ImportPipeline {
|
||||
return contentPlusDispatch(async (ctx) => createMediaEventHandlerFromContext(ctx))
|
||||
}
|
||||
|
||||
/** content + 直传 media 组合(手构场景:不经 register 的 media_bundle)——分派形状同上 */
|
||||
function contentPlusDirectMediaPipeline(identity: {
|
||||
storageRoot: string
|
||||
manifest: MediaManifestEntry[]
|
||||
sidecar: Array<[string, string]>
|
||||
}): ImportPipeline {
|
||||
return contentPlusDispatch(async (ctx) =>
|
||||
createMediaEventHandler({
|
||||
db: ctx.db,
|
||||
siteId: ctx.siteId,
|
||||
connectionId: ctx.connectionId,
|
||||
snapshotId: ctx.wxrSnapshot.id,
|
||||
defaultLocale: ctx.defaultLocale,
|
||||
storageRoot: identity.storageRoot,
|
||||
mediaBundleStoragePath: '',
|
||||
mediaManifest: identity.manifest,
|
||||
sidecarMap: new Map(identity.sidecar),
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
/** 分派骨架:先 content 后 media(顺序契约),counts 展开合并(media 前缀无碰撞键) */
|
||||
function contentPlusDispatch(
|
||||
mediaOf: (ctx: Parameters<ImportPipeline>[0]) => Promise<ReturnType<typeof createMediaEventHandler>>,
|
||||
): ImportPipeline {
|
||||
return async (ctx) => {
|
||||
const content = createContentEventHandler({
|
||||
db: ctx.db,
|
||||
siteId: ctx.siteId,
|
||||
connectionId: ctx.connectionId,
|
||||
snapshotId: ctx.wxrSnapshot.id,
|
||||
jobId: ctx.jobId,
|
||||
defaultLocale: ctx.defaultLocale,
|
||||
versions: ctx.versions,
|
||||
})
|
||||
const media = await mediaOf(ctx)
|
||||
for await (const ev of ctx.events) {
|
||||
await content.handleEvent(ev)
|
||||
await media.handleEvent(ev)
|
||||
}
|
||||
const c = content.finish()
|
||||
const m = await media.finish()
|
||||
return { counts: { ...c.counts, ...m.counts }, issues: [...c.issues, ...m.issues] }
|
||||
}
|
||||
}
|
||||
|
||||
/** 清单/侧车直传管道(手构场景:无内容行——纯 attachment 事件,media 单写入器) */
|
||||
function directMediaPipeline(identity: {
|
||||
storageRoot: string
|
||||
manifest: MediaManifestEntry[]
|
||||
sidecar: Array<[string, string]>
|
||||
}): ImportPipeline {
|
||||
return async (ctx) => {
|
||||
const handler = createMediaEventHandler({
|
||||
db: ctx.db,
|
||||
siteId: ctx.siteId,
|
||||
connectionId: ctx.connectionId,
|
||||
snapshotId: ctx.wxrSnapshot.id,
|
||||
defaultLocale: ctx.defaultLocale,
|
||||
storageRoot: identity.storageRoot,
|
||||
mediaBundleStoragePath: '',
|
||||
mediaManifest: identity.manifest,
|
||||
sidecarMap: new Map(identity.sidecar),
|
||||
})
|
||||
for await (const ev of ctx.events) {
|
||||
await handler.handleEvent(ev)
|
||||
}
|
||||
return handler.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/** 本文件写入的行清场:FK 反序(media_usages → media_assets 先于内容域;两张不可变表走触发器开关) */
|
||||
async function cleanupSiteRows(db: Db, siteId: string): Promise<void> {
|
||||
await db.execute(`delete from media_usages where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from media_assets where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from content_terms where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from terms where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from taxonomies where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from source_records where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from entry_locales where site_id = '${siteId}'`)
|
||||
await withImmutableCleanup(db, 'content_revisions', 'content_revisions_immutable', `site_id = '${siteId}'`)
|
||||
await db.execute(`delete from content_entries where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from import_jobs where site_id = '${siteId}'`)
|
||||
await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`)
|
||||
await db.execute(`delete from source_connections where site_id = '${siteId}'`)
|
||||
await db.execute(`delete from sites where id = '${siteId}'`)
|
||||
}
|
||||
|
||||
/** mini 登记(各用例公共前置):register → {siteId, connectionId, wxr 快照} */
|
||||
async function registerMini(db: Db, work: string) {
|
||||
const key = `sfmedia-${rand()}`
|
||||
const reg = await registerSource(db, {
|
||||
siteKey: key,
|
||||
wxrPath: WXR_FIXTURE,
|
||||
allowedRoots: [FIXTURES_ROOT, work],
|
||||
storageRoot: join(work, 'storage'),
|
||||
reportRoot: join(work, 'artifacts', 'register'),
|
||||
})
|
||||
return { key, reg, wxr: reg.snapshots.find((s) => s.kind === 'wxr')! }
|
||||
}
|
||||
|
||||
/** 导入报告读取 */
|
||||
async function readReport(work: string, jobId: string): Promise<{
|
||||
status: string
|
||||
counts: Record<string, number>
|
||||
issues: string[]
|
||||
}> {
|
||||
return JSON.parse(await readFile(join(work, 'artifacts', 'import', jobId, 'reports', 'import.json'), 'utf8'))
|
||||
}
|
||||
|
||||
describe('媒体登记写入器(三级对拍 + 魔数/隔离)', () => {
|
||||
it('mini 全链三级对拍:①清单精确命中 ②侧车命中 → media_assets(public_path=旧 uploads 路径、sha256/storage_path 从清单取)+ usage 三行 + 悬空 203/204 + counts 全键', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
const work = await mkdtemp(join(tmpdir(), 'sf-media-full-'))
|
||||
try {
|
||||
const events = await collectMiniEvents()
|
||||
// 媒体目录:fixture-a 按旧 uploads 层级放(级①精确命中);fixture-b 用扁平
|
||||
// GUID 名(清单故意不含 '2026/10/fixture-b.png'——逼走级②侧车)
|
||||
const mediaDir = join(work, 'media')
|
||||
await mkdir(join(mediaDir, '2026/10'), { recursive: true })
|
||||
const pngA = pngBytes(`fixture-a-${rand()}`)
|
||||
await writeFile(join(mediaDir, '2026/10/fixture-a.png'), pngA)
|
||||
const guidB = `b-${rand()}.png`
|
||||
const pngB = pngBytes(`fixture-b-${rand()}`)
|
||||
await writeFile(join(mediaDir, guidB), pngB)
|
||||
const sidecarPath = join(work, 'media-map.json')
|
||||
await writeFile(
|
||||
sidecarPath,
|
||||
JSON.stringify([
|
||||
{ sourceUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-b.png', filePath: guidB },
|
||||
]),
|
||||
)
|
||||
const key = `sfmedia-${rand()}`
|
||||
const reg = await registerSource(db, {
|
||||
siteKey: key,
|
||||
wxrPath: WXR_FIXTURE,
|
||||
mediaDir,
|
||||
mediaMapPath: sidecarPath,
|
||||
allowedRoots: [FIXTURES_ROOT, work],
|
||||
storageRoot: join(work, 'storage'),
|
||||
reportRoot: join(work, 'artifacts', 'register'),
|
||||
})
|
||||
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
|
||||
const bundle = reg.snapshots.find((s) => s.kind === 'media_bundle')!
|
||||
try {
|
||||
const result = await createImportJob(db, {
|
||||
siteKey: key,
|
||||
snapshotId: wxr.id,
|
||||
reportRoot: join(work, 'artifacts', 'import'),
|
||||
events: eventsOf(events),
|
||||
pipeline: contentPlusMediaPipeline(),
|
||||
})
|
||||
// 悬空 203/204 → issues 非空 → succeeded_with_issues(§7.6 禁静默)
|
||||
expect(result.status).toBe('succeeded_with_issues')
|
||||
|
||||
// media_assets 两行:fixture-a 级① / fixture-b 级②;身份列逐项对拍
|
||||
const assets = (await db.execute(`select source_id, source_url, source_path, storage_path,
|
||||
public_path, sha256, detected_mime, declared_mime, byte_size::int as byte_size, status
|
||||
from media_assets where site_id = '${reg.siteId}' order by source_id`)) as any
|
||||
expect(assets).toHaveLength(2)
|
||||
const a = assets.find((x: any) => x.source_id === '201')!
|
||||
expect(a.source_url).toBe('https://fixture.example/wp-content/uploads/2026/10/fixture-a.png')
|
||||
expect(a.source_path).toBe('2026/10/fixture-a.png')
|
||||
expect(a.public_path).toBe('/wp-content/uploads/2026/10/fixture-a.png')
|
||||
expect(a.sha256).toBe(sha256of(pngA))
|
||||
expect(a.byte_size).toBe(pngA.length)
|
||||
expect(a.detected_mime).toBe('image/png')
|
||||
expect(a.declared_mime).toBe('image/png')
|
||||
expect(a.status).toBe('ready')
|
||||
// storage_path = 清单实体定位(bundle 副本相对路径 + 清单 path——绝对副本 = storageRoot + 此列)
|
||||
expect(a.storage_path).toBe(`${bundle.storagePath}/2026/10/fixture-a.png`)
|
||||
const b = assets.find((x: any) => x.source_id === '202')!
|
||||
expect(b.source_url).toBe('https://fixture.example/wp-content/uploads/2026/10/fixture-b.png')
|
||||
expect(b.public_path).toBe('/wp-content/uploads/2026/10/fixture-b.png')
|
||||
expect(b.sha256).toBe(sha256of(pngB))
|
||||
expect(b.storage_path).toBe(`${bundle.storagePath}/${guidB}`)
|
||||
expect(b.status).toBe('ready')
|
||||
|
||||
// media_usages 三行:101 featured(id 反查)+ 101 正文 img(URL 反查)+ 111 featured
|
||||
const usages = (await db.execute(`select a.source_id as asset, u.field_path, u.source_url
|
||||
from media_usages u join media_assets a on a.id = u.media_asset_id
|
||||
where u.site_id = '${reg.siteId}' order by a.source_id, u.field_path`)) as any
|
||||
expect(plain(usages)).toStrictEqual([
|
||||
{
|
||||
asset: '201',
|
||||
field_path: 'attachmentRefs.featured',
|
||||
source_url: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png',
|
||||
},
|
||||
{
|
||||
asset: '201',
|
||||
field_path: 'data.bodyHtml.img[0]',
|
||||
source_url: 'https://fixture.example/wp-content/uploads/2026/10/fixture-a.png',
|
||||
},
|
||||
{
|
||||
asset: '202',
|
||||
field_path: 'attachmentRefs.featured',
|
||||
source_url: 'https://fixture.example/wp-content/uploads/2026/10/fixture-b.png',
|
||||
},
|
||||
])
|
||||
// usage 挂在 content-writer 落的当前修订上(current_revision_id——导入不发布,published NULL)
|
||||
const revCheck = (await db.execute(`select count(*)::int as n from media_usages u
|
||||
join content_revisions cr on cr.id = u.content_revision_id
|
||||
join entry_locales el on el.current_revision_id = cr.id and el.published_revision_id is null
|
||||
where u.site_id = '${reg.siteId}'`)) as any
|
||||
expect(revCheck[0].n).toBe(3)
|
||||
|
||||
// counts 全键:content 四键 + media 六键(全局命名空间并存无碰撞)
|
||||
const report = await readReport(work, result.jobId)
|
||||
expect(report.status).toBe('succeeded_with_issues')
|
||||
expect(report.counts).toStrictEqual({
|
||||
contentEntries: 3,
|
||||
revisionsCreated: 3,
|
||||
sourceRecords: 5,
|
||||
skipped: 0,
|
||||
mediaRegistered: 2,
|
||||
mediaFromSidecar: 1,
|
||||
mediaMissing: 0,
|
||||
mediaQuarantined: 0,
|
||||
// mini 全部引用 5 条:featured 201/202 + 正文 img 各解析命中,gallery 203/204 悬空
|
||||
mediaDanglingRefs: 2,
|
||||
mediaAmbiguous: 0,
|
||||
})
|
||||
// 悬空引用显式报告:203/204 各一条(gallery 引用无 attachment/清单记录)
|
||||
const dangling = report.issues.filter((i) => i.includes('悬空'))
|
||||
expect(dangling).toHaveLength(2)
|
||||
expect(dangling.some((i) => i.includes('203'))).toBe(true)
|
||||
expect(dangling.some((i) => i.includes('204'))).toBe(true)
|
||||
} finally {
|
||||
await cleanupSiteRows(db, reg.siteId)
|
||||
}
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('级③ missing(无 media_bundle 登记):不插 media_assets 行(sha256 NOT NULL)+ mediaMissing + 全部引用悬空如实报告', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
const work = await mkdtemp(join(tmpdir(), 'sf-media-missing-'))
|
||||
try {
|
||||
const events = await collectMiniEvents()
|
||||
const { key, reg, wxr } = await registerMini(db, work)
|
||||
try {
|
||||
const result = await createImportJob(db, {
|
||||
siteKey: key,
|
||||
snapshotId: wxr.id,
|
||||
reportRoot: join(work, 'artifacts', 'import'),
|
||||
events: eventsOf(events),
|
||||
pipeline: contentPlusMediaPipeline(),
|
||||
})
|
||||
expect(result.status).toBe('succeeded_with_issues')
|
||||
|
||||
// 零资产行:清单缺席 → 级①②全 miss → 级③ missing(不插行——sha256 无来源)
|
||||
expect(((await db.execute(`select count(*)::int as n from media_assets
|
||||
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
|
||||
expect(((await db.execute(`select count(*)::int as n from media_usages
|
||||
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
|
||||
|
||||
const report = await readReport(work, result.jobId)
|
||||
expect(report.counts).toStrictEqual({
|
||||
contentEntries: 3,
|
||||
revisionsCreated: 3,
|
||||
sourceRecords: 5,
|
||||
skipped: 0,
|
||||
mediaRegistered: 0,
|
||||
mediaFromSidecar: 0,
|
||||
mediaMissing: 2,
|
||||
mediaQuarantined: 0,
|
||||
// 引用全悬空:101 featured+img、111 featured+gallery×2 = 5
|
||||
mediaDanglingRefs: 5,
|
||||
mediaAmbiguous: 0,
|
||||
})
|
||||
const missing = report.issues.filter((i) => i.includes('missing'))
|
||||
expect(missing).toHaveLength(2)
|
||||
expect(missing.some((i) => i.includes('fixture-a.png'))).toBe(true)
|
||||
expect(missing.some((i) => i.includes('fixture-b.png'))).toBe(true)
|
||||
expect(report.issues.filter((i) => i.includes('悬空'))).toHaveLength(5)
|
||||
} finally {
|
||||
await cleanupSiteRows(db, reg.siteId)
|
||||
}
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('魔数校验:命中文件读实际字节——PNG 过;声明(扩展名推断)与检测不符 → issue;SVG 检出(伪装 .png 与显式 .svg)→ quarantined 不进产物;未知签名 → unsupported', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
const work = await mkdtemp(join(tmpdir(), 'sf-media-magic-'))
|
||||
try {
|
||||
// 五个手构文件(放旧 uploads 层级——级① 相对路径精确命中):正例 PNG / 扩展名谎言
|
||||
// (.png 实为 JPEG)/ SVG 伪装 .png / 显式 .svg / 无签名 .txt
|
||||
const files: Array<{ name: string; buf: Buffer }> = [
|
||||
{ name: 'ok.png', buf: pngBytes(`ok-${rand()}`) },
|
||||
{ name: 'lie.png', buf: jpegBytes(`lie-${rand()}`) },
|
||||
{ name: 'evil.png', buf: Buffer.from(`<?xml version="1.0"?><SVG onload="alert(1)" xmlns="…">`) },
|
||||
{ name: 'draw.svg', buf: Buffer.from('<svg width="10" height="10"></svg>') },
|
||||
{ name: 'note.txt', buf: Buffer.from('plain text note, no signature at all') },
|
||||
]
|
||||
await mkdir(join(work, '2026/10'), { recursive: true })
|
||||
for (const f of files) {
|
||||
await writeFile(join(work, '2026/10', f.name), f.buf)
|
||||
}
|
||||
const manifest = files.map((f) => manifestEntryOf(`2026/10/${f.name}`, f.buf))
|
||||
const events: AdapterEvent[] = files.map((f, i) =>
|
||||
attachmentEvent({ sourceId: String(901 + i), url: `https://fixture.example/wp-content/uploads/2026/10/${f.name}` }),
|
||||
)
|
||||
const { key, reg, wxr } = await registerMini(db, work)
|
||||
try {
|
||||
const result = await createImportJob(db, {
|
||||
siteKey: key,
|
||||
snapshotId: wxr.id,
|
||||
reportRoot: join(work, 'artifacts', 'import'),
|
||||
events: eventsOf(events),
|
||||
pipeline: directMediaPipeline({ storageRoot: work, manifest, sidecar: [] }),
|
||||
})
|
||||
expect(result.status).toBe('succeeded_with_issues')
|
||||
|
||||
const assets = (await db.execute(`select source_id, detected_mime, declared_mime, status
|
||||
from media_assets where site_id = '${reg.siteId}' order by source_id`)) as any
|
||||
expect(plain(assets)).toStrictEqual([
|
||||
{ source_id: '901', detected_mime: 'image/png', declared_mime: 'image/png', status: 'ready' },
|
||||
{ source_id: '902', detected_mime: 'image/jpeg', declared_mime: 'image/png', status: 'ready' },
|
||||
{ source_id: '903', detected_mime: 'image/svg+xml', declared_mime: 'image/png', status: 'quarantined' },
|
||||
{ source_id: '904', detected_mime: 'image/svg+xml', declared_mime: 'image/svg+xml', status: 'quarantined' },
|
||||
{ source_id: '905', detected_mime: 'application/octet-stream', declared_mime: null, status: 'unsupported' },
|
||||
])
|
||||
|
||||
const report = await readReport(work, result.jobId)
|
||||
expect(report.counts).toStrictEqual({
|
||||
mediaRegistered: 5,
|
||||
mediaFromSidecar: 0,
|
||||
mediaMissing: 0,
|
||||
mediaQuarantined: 2,
|
||||
mediaDanglingRefs: 0,
|
||||
mediaAmbiguous: 0,
|
||||
})
|
||||
// 三类问题各就位:MIME 不符 1(lie.png)+ SVG 隔离 2(evil/draw)+ 未知签名 1(note.txt)
|
||||
const mismatch = report.issues.filter((i) => i.includes('不符'))
|
||||
expect(mismatch).toHaveLength(1)
|
||||
expect(mismatch[0]).toContain('lie.png')
|
||||
const quarantined = report.issues.filter((i) => i.includes('隔离'))
|
||||
expect(quarantined).toHaveLength(2)
|
||||
expect(report.issues.filter((i) => i.includes('未知签名'))).toHaveLength(1)
|
||||
// quarantined/unsupported 资产不进产物语义:即便有引用也不建 usage(本例无内容行,零 usage)
|
||||
expect(((await db.execute(`select count(*)::int as n from media_usages
|
||||
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
|
||||
} finally {
|
||||
await cleanupSiteRows(db, reg.siteId)
|
||||
}
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('quarantined 资产的引用不建 usage(不进产物语义):SVG 资产被正文引用 → 引用关系不落 media_usages', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
const work = await mkdtemp(join(tmpdir(), 'sf-media-quar-'))
|
||||
try {
|
||||
const evil = Buffer.from('<svg onload="x()"><script>bad()</script></svg>')
|
||||
await mkdir(join(work, '2026/10'), { recursive: true })
|
||||
await writeFile(join(work, '2026/10/evil.png'), evil)
|
||||
const manifest = [manifestEntryOf('2026/10/evil.png', evil)]
|
||||
const attachment = attachmentEvent({
|
||||
sourceId: '951',
|
||||
url: 'https://fixture.example/wp-content/uploads/2026/10/evil.png',
|
||||
})
|
||||
const citing: AdapterEvent = {
|
||||
kind: 'row',
|
||||
row: {
|
||||
sourceType: 'wp:post',
|
||||
sourceId: '950',
|
||||
status: 'publish',
|
||||
locale: 'en',
|
||||
taxonomyRefs: [],
|
||||
attachmentRefs: [{ role: 'featured', sourceId: '951' }],
|
||||
seo: {},
|
||||
knownFields: {},
|
||||
extras: {},
|
||||
rawLocation: 'wxr://item/950',
|
||||
bodyHtml: '<p><img src="https://fixture.example/wp-content/uploads/2026/10/evil.png"/></p>',
|
||||
},
|
||||
issues: [],
|
||||
sanitizeDiffs: [],
|
||||
changed: false,
|
||||
}
|
||||
const { key, reg, wxr } = await registerMini(db, work)
|
||||
try {
|
||||
// 引用行先到、attachment 后到(WXR 文档序)——finish 统一解析
|
||||
const result = await createImportJob(db, {
|
||||
siteKey: key,
|
||||
snapshotId: wxr.id,
|
||||
reportRoot: join(work, 'artifacts', 'import'),
|
||||
events: eventsOf([citing, attachment]),
|
||||
pipeline: contentPlusDirectMediaPipeline({ storageRoot: work, manifest, sidecar: [] }),
|
||||
})
|
||||
expect(result.status).toBe('succeeded_with_issues')
|
||||
// 资产行存在(登记可审计)但 quarantined;引用解析命中资产(非悬空)但不建 usage
|
||||
const assets = (await db.execute(`select source_id, status from media_assets
|
||||
where site_id = '${reg.siteId}'`)) as any
|
||||
expect(plain(assets)).toStrictEqual([{ source_id: '951', status: 'quarantined' }])
|
||||
expect(((await db.execute(`select count(*)::int as n from media_usages
|
||||
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
|
||||
const report = await readReport(work, result.jobId)
|
||||
expect(report.counts.mediaRegistered).toBe(1)
|
||||
expect(report.counts.mediaQuarantined).toBe(1)
|
||||
expect(report.counts.mediaDanglingRefs).toBe(0)
|
||||
expect(report.issues.filter((i) => i.includes('隔离'))).toHaveLength(1)
|
||||
} finally {
|
||||
await cleanupSiteRows(db, reg.siteId)
|
||||
}
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('多候选(同 filename 不同目录两条清单):歧义不自动选、不插行、全部候选列 issue', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
const work = await mkdtemp(join(tmpdir(), 'sf-media-ambig-'))
|
||||
try {
|
||||
await mkdir(join(work, 'd1'), { recursive: true })
|
||||
await mkdir(join(work, 'd2'), { recursive: true })
|
||||
const f1 = pngBytes(`amb-1-${rand()}`)
|
||||
const f2 = pngBytes(`amb-2-${rand()}`)
|
||||
await writeFile(join(work, 'd1/thing.png'), f1)
|
||||
await writeFile(join(work, 'd2/thing.png'), f2)
|
||||
const manifest = [manifestEntryOf('d1/thing.png', f1), manifestEntryOf('d2/thing.png', f2)]
|
||||
const url = 'https://fixture.example/wp-content/uploads/2024/01/thing.png'
|
||||
const events: AdapterEvent[] = [attachmentEvent({ sourceId: '961', url })]
|
||||
const { key, reg, wxr } = await registerMini(db, work)
|
||||
try {
|
||||
const result = await createImportJob(db, {
|
||||
siteKey: key,
|
||||
snapshotId: wxr.id,
|
||||
reportRoot: join(work, 'artifacts', 'import'),
|
||||
events: eventsOf(events),
|
||||
// 侧车 file_path 只给文件名:精确路径 miss → 文件名候选 2 条 → 歧义
|
||||
pipeline: directMediaPipeline({ storageRoot: work, manifest, sidecar: [[url, 'thing.png']] }),
|
||||
})
|
||||
expect(result.status).toBe('succeeded_with_issues')
|
||||
expect(((await db.execute(`select count(*)::int as n from media_assets
|
||||
where site_id = '${reg.siteId}'`)) as any)[0].n).toBe(0)
|
||||
const report = await readReport(work, result.jobId)
|
||||
expect(report.counts).toStrictEqual({
|
||||
mediaRegistered: 0,
|
||||
mediaFromSidecar: 0,
|
||||
mediaMissing: 0,
|
||||
mediaQuarantined: 0,
|
||||
mediaDanglingRefs: 0,
|
||||
mediaAmbiguous: 1,
|
||||
})
|
||||
const amb = report.issues.filter((i) => i.includes('歧义'))
|
||||
expect(amb).toHaveLength(1)
|
||||
expect(amb[0]).toContain('d1/thing.png')
|
||||
expect(amb[0]).toContain('d2/thing.png')
|
||||
} finally {
|
||||
await cleanupSiteRows(db, reg.siteId)
|
||||
}
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('幂等重跑:media_assets/media_usages 零增行、mediaRegistered=0;悬空引用按流如实再现', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
const work = await mkdtemp(join(tmpdir(), 'sf-media-idem-'))
|
||||
try {
|
||||
const events = await collectMiniEvents()
|
||||
const mediaDir = join(work, 'media')
|
||||
await mkdir(join(mediaDir, '2026/10'), { recursive: true })
|
||||
const pngA = pngBytes(`idem-a-${rand()}`)
|
||||
await writeFile(join(mediaDir, '2026/10/fixture-a.png'), pngA)
|
||||
const guidB = `b-${rand()}.png`
|
||||
await writeFile(join(mediaDir, guidB), pngBytes(`idem-b-${rand()}`))
|
||||
const sidecarPath = join(work, 'media-map.json')
|
||||
await writeFile(
|
||||
sidecarPath,
|
||||
JSON.stringify([
|
||||
{ sourceUrl: 'https://fixture.example/wp-content/uploads/2026/10/fixture-b.png', filePath: guidB },
|
||||
]),
|
||||
)
|
||||
const key = `sfmedia-${rand()}`
|
||||
const reg = await registerSource(db, {
|
||||
siteKey: key,
|
||||
wxrPath: WXR_FIXTURE,
|
||||
mediaDir,
|
||||
mediaMapPath: sidecarPath,
|
||||
allowedRoots: [FIXTURES_ROOT, work],
|
||||
storageRoot: join(work, 'storage'),
|
||||
reportRoot: join(work, 'artifacts', 'register'),
|
||||
})
|
||||
const wxr = reg.snapshots.find((s) => s.kind === 'wxr')!
|
||||
try {
|
||||
const base = {
|
||||
siteKey: key,
|
||||
snapshotId: wxr.id,
|
||||
reportRoot: join(work, 'artifacts', 'import'),
|
||||
pipeline: contentPlusMediaPipeline(),
|
||||
}
|
||||
const first = await createImportJob(db, { ...base, events: eventsOf(events) })
|
||||
expect(first.status).toBe('succeeded_with_issues')
|
||||
|
||||
// 同流重跑(换幂等键新 job——钉写入器幂等,不测 job 复用)
|
||||
const again = await createImportJob(db, {
|
||||
...base,
|
||||
idempotencyKey: `sfmedia-rerun-${rand()}`,
|
||||
events: eventsOf(events),
|
||||
})
|
||||
expect(again.reused).toBe(false)
|
||||
expect(again.status).toBe('succeeded_with_issues')
|
||||
|
||||
const after = (await db.execute(`select
|
||||
(select count(*)::int from media_assets where site_id = '${reg.siteId}') as assets,
|
||||
(select count(*)::int from media_usages where site_id = '${reg.siteId}') as usages`)) as any
|
||||
expect(plain(after)[0]).toStrictEqual({ assets: 2, usages: 3 })
|
||||
|
||||
// 行面零增;流面计数如实:registered/fromSidecar 归零,悬空引用再现(203/204)
|
||||
const report = await readReport(work, again.jobId)
|
||||
expect(report.counts).toStrictEqual({
|
||||
contentEntries: 0,
|
||||
revisionsCreated: 0,
|
||||
sourceRecords: 0,
|
||||
skipped: 3,
|
||||
mediaRegistered: 0,
|
||||
mediaFromSidecar: 0,
|
||||
mediaMissing: 0,
|
||||
mediaQuarantined: 0,
|
||||
mediaDanglingRefs: 2,
|
||||
mediaAmbiguous: 0,
|
||||
})
|
||||
expect(report.issues.filter((i) => i.includes('悬空'))).toHaveLength(2)
|
||||
} finally {
|
||||
await cleanupSiteRows(db, reg.siteId)
|
||||
}
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,66 @@
|
||||
// magic-bytes 单测(Plan 3 Task 4 Step 1):六类签名各正例 + 伪签名 null 例 +
|
||||
// SVG 大小写不敏感 + 窗口边界(1KB 外不检出)+ 二进制签名优先于文本特征。
|
||||
// 期望值全部手写字面量(与实现零共享——对拍而非引用)。
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { detectMediaType } from '../../apps/api/src/modules/import/magic-bytes.js'
|
||||
|
||||
const PNG_SIG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
||||
|
||||
describe('detectMediaType 手写签名表', () => {
|
||||
it('六类二进制签名正例:PNG/JPEG/GIF87a/GIF89a/WEBP/PDF', () => {
|
||||
expect(detectMediaType(Buffer.concat([PNG_SIG, Buffer.from('rest-of-png')]))).toStrictEqual({
|
||||
type: 'image/png',
|
||||
svg: false,
|
||||
})
|
||||
expect(detectMediaType(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]))).toStrictEqual({
|
||||
type: 'image/jpeg',
|
||||
svg: false,
|
||||
})
|
||||
expect(detectMediaType(Buffer.from('GIF87a-more-gif'))).toStrictEqual({ type: 'image/gif', svg: false })
|
||||
expect(detectMediaType(Buffer.from('GIF89a-more-gif'))).toStrictEqual({ type: 'image/gif', svg: false })
|
||||
expect(
|
||||
detectMediaType(Buffer.concat([Buffer.from('RIFF'), Buffer.from([0x24, 0x00, 0x00, 0x00]), Buffer.from('WEBPVP8 ')])),
|
||||
).toStrictEqual({ type: 'image/webp', svg: false })
|
||||
expect(detectMediaType(Buffer.from('%PDF-1.7\n%fake-payload'))).toStrictEqual({
|
||||
type: 'application/pdf',
|
||||
svg: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('SVG 文本特征:大小写不敏感(<svg / <SVG / <Svg,含 xml 声明前导),svg=true', () => {
|
||||
expect(detectMediaType(Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"/>'))).toStrictEqual({
|
||||
type: 'image/svg+xml',
|
||||
svg: true,
|
||||
})
|
||||
expect(detectMediaType(Buffer.from('<?xml version="1.0"?><SVG onload="x()">'))).toStrictEqual({
|
||||
type: 'image/svg+xml',
|
||||
svg: true,
|
||||
})
|
||||
expect(detectMediaType(Buffer.from('<!-- c --><Svg width="1">'))).toStrictEqual({
|
||||
type: 'image/svg+xml',
|
||||
svg: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('伪签名 / 纯文本 / 空缓冲 → null(未知签名不猜格式)', () => {
|
||||
expect(detectMediaType(Buffer.from('hello plain text note'))).toBe(null)
|
||||
expect(detectMediaType(Buffer.from([0x00, 0x01, 0x02, 0x03]))).toBe(null)
|
||||
expect(detectMediaType(Buffer.alloc(0))).toBe(null)
|
||||
})
|
||||
|
||||
it('RIFF 非 WEBP(WAV)不误报 webp → null', () => {
|
||||
expect(
|
||||
detectMediaType(Buffer.concat([Buffer.from('RIFF'), Buffer.from([0x24, 0x00, 0x00, 0x00]), Buffer.from('WAVEfmt ')])),
|
||||
).toBe(null)
|
||||
})
|
||||
|
||||
it('二进制签名优先:PNG 头 + 尾部 <svg 片段仍判 PNG(文本窗口只对非二进制生效)', () => {
|
||||
const sneaky = Buffer.concat([PNG_SIG, Buffer.from('<svg onload="alert(1)"> trailing')])
|
||||
expect(detectMediaType(sneaky)).toStrictEqual({ type: 'image/png', svg: false })
|
||||
})
|
||||
|
||||
it('SVG 窗口 1KB 钉死:<svg 出现在 1KB 之后不检出 → null', () => {
|
||||
const beyond = Buffer.concat([Buffer.alloc(1200, 0x20), Buffer.from('<svg>')])
|
||||
expect(detectMediaType(beyond)).toBe(null)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user