feat(source): 媒体映射侧车导出与登记

Plan 3 Task 0(评审 Issue 1):旧 cms 库 media_assets 的 source_url→file_path
映射只读导出为侧车 JSON(859 条,覆盖率对照 WXR 855 attachment_url:精确命中
836 / 归一命中 0 / 未命中 19,97.8%),register 新增 --media-map 输入通道
(kind=manual_url_list 快照,形状校验 INVALID_MEDIA_MAP,metadata 含条目数,
幂等复用)。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-10-21 14:42:32 +08:00
parent ceb6106025
commit a227cfac49
5 changed files with 515 additions and 14 deletions
+85 -10
View File
@@ -1,5 +1,6 @@
// source:register 输入登记服务(Plan 2 Task 7,spec §7.1 / ADR-004/008):
// 把 WXR / sitemap / 媒体目录三类输入登记为 content-addressed 快照——
// 把 WXR / sitemap / 媒体目录 / 媒体映射侧车(manual_url_list,Plan 3 Task 0)四类输入
// 登记为 content-addressed 快照——
// 副本落 storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(媒体整树),
// 源输入文件从此不再被后续任务读取(导入只寻址副本)。
//
@@ -25,7 +26,7 @@
// 新 uuid 重写)但不再被本服务清理——手工清理策略:CLI 未运行时整目录删除 .tmp/ 即可。
import { createHash, randomUUID } from 'node:crypto'
import { createReadStream, createWriteStream, existsSync } from 'node:fs'
import { lstat, mkdir, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises'
import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises'
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { pipeline } from 'node:stream/promises'
@@ -49,13 +50,14 @@ export type SourceDb = NodePgDatabase<typeof schema>
*/
type SourceDbWriter = Pick<SourceDb, 'select' | 'insert' | 'execute'>
export type SourceSnapshotKind = 'wxr' | 'sitemap' | 'media_bundle'
export type SourceSnapshotKind = 'wxr' | 'sitemap' | 'media_bundle' | 'manual_url_list'
/** 各 kind 快照的 adapter_version:wxr 用适配器包单源版本;sitemap/media 尚无解析器,占位钉死 */
/** 各 kind 快照的 adapter_version:wxr 用适配器包单源版本;sitemap/media/media_map 尚无解析器,占位钉死 */
const SNAPSHOT_ADAPTER_VERSION: Record<SourceSnapshotKind, string> = {
wxr: `adapter-wp@${ADAPTER_VERSION}`,
sitemap: 'sitemap@0',
media_bundle: 'media-bundle@1',
manual_url_list: 'manual-url-list@1',
}
export type SourceRegisterErrorCode =
@@ -66,6 +68,7 @@ export type SourceRegisterErrorCode =
| 'INPUT_NOT_DIR'
| 'EMPTY_MEDIA_DIR'
| 'INPUT_READ_FAILED'
| 'INVALID_MEDIA_MAP'
| 'COPY_FAILED'
/** 登记失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */
@@ -89,6 +92,12 @@ export interface RegisterInput {
wxrPath?: string
sitemapPath?: string
mediaDir?: string
/**
* 媒体映射侧车 JSON(Plan 3 Task 0,评审 Issue 1):[{sourceUrl, filePath}] 数组——
* 旧 cms 库 media_assets 导出的 URL→扁平文件名映射;登记为 kind=manual_url_list 快照。
* 导入管道对拍三级中的第二级数据源(①清单 uploads 路径 → ②本侧车 → ③missing 报告)。
*/
mediaMapPath?: string
/** 输入路径白名单(realpath 后必须位于其中之一);缺省 [repoRoot/storage/incoming] */
allowedRoots?: string[]
/** 快照存储根(副本落 <storageRoot>/snapshots/...);缺省 repoRoot/storage */
@@ -256,9 +265,49 @@ interface Ingested {
durationMs: number
}
/** 文件输入(wxr/sitemap):temp 副本上单遍 copy+hash → 按 sha 定名落位(已存在则命中复用) */
/**
* 校验 media-map 侧车 JSON 形状并数条目:[{sourceUrl: string, filePath: string}] 数组,
* 每条两字段均为非空字符串。形状即契约(Plan 3 对拍二级的数据源)——脏输入在登记期
* 拒绝(快照不可变,落库后再发现无法补救)。返回条目数入 metadata。
*/
async function validateMediaMap(path: string): Promise<number> {
let parsed: unknown
try {
parsed = JSON.parse(await readFile(path, 'utf8'))
} catch (err) {
throw new SourceRegisterError(
'INVALID_MEDIA_MAP',
`media-map 不是合法 JSON:${path}(${err instanceof Error ? err.message : String(err)})`,
)
}
if (!Array.isArray(parsed)) {
throw new SourceRegisterError(
'INVALID_MEDIA_MAP',
`media-map 顶层必须是数组([{sourceUrl, filePath}]):${path},实际 ${parsed === null ? 'null' : typeof parsed}`,
)
}
for (let i = 0; i < parsed.length; i++) {
const entry = parsed[i] as unknown
const ok =
entry !== null &&
typeof entry === 'object' &&
typeof (entry as { sourceUrl?: unknown }).sourceUrl === 'string' &&
(entry as { sourceUrl: string }).sourceUrl !== '' &&
typeof (entry as { filePath?: unknown }).filePath === 'string' &&
(entry as { filePath: string }).filePath !== ''
if (!ok) {
throw new SourceRegisterError(
'INVALID_MEDIA_MAP',
`media-map 第 ${i} 条不满足形状 {sourceUrl: string, filePath: string}:${path}`,
)
}
}
return parsed.length
}
/** 文件输入(wxr/sitemap/media-map):temp 副本上单遍 copy+hash → 按 sha 定名落位(已存在则命中复用) */
async function ingestFile(
kind: 'wxr' | 'sitemap',
kind: 'wxr' | 'sitemap' | 'manual_url_list',
src: string,
storageRoot: string,
): Promise<Ingested> {
@@ -276,6 +325,17 @@ async function ingestFile(
`${kind} 输入读取失败:${src}(${err instanceof Error ? err.message : String(err)})`,
)
}
// media-map:形状校验在副本 rename 落位**之前**——脏输入不产 content-addressed 副本
//(校验读 temp 副本而非源,登记窗口内源被改写不会使校验结果与哈希脱钩)
let entryCount: number | undefined
if (kind === 'manual_url_list') {
try {
entryCount = await validateMediaMap(tmp)
} catch (err) {
await rm(tmp, { force: true })
throw err
}
}
const rel = `snapshots/${kind}/${sha256.slice(0, 2)}/${sha256}`
let copied: boolean
try {
@@ -286,6 +346,8 @@ async function ingestFile(
`${kind} 快照副本落位失败:${join(storageRoot, rel)}(${err instanceof Error ? err.message : String(err)})`,
)
}
const metadata: Record<string, unknown> = { sourceName: basename(src) }
if (entryCount !== undefined) metadata.entryCount = entryCount
return {
kind,
sourcePath: src,
@@ -293,7 +355,7 @@ async function ingestFile(
byteSize: bytes,
storagePath: rel,
copied,
metadata: { sourceName: basename(src) },
metadata,
durationMs: performance.now() - t0,
}
}
@@ -578,20 +640,33 @@ export async function registerSource(db: SourceDb, input: RegisterInput): Promis
let wxrReal: string | undefined
let sitemapReal: string | undefined
let mediaReal: string | undefined
let mediaMapReal: string | undefined
if (input.wxrPath !== undefined) wxrReal = await validateInputPath(input.wxrPath, 'file', allowedRoots, 'wxr')
if (input.sitemapPath !== undefined)
sitemapReal = await validateInputPath(input.sitemapPath, 'file', allowedRoots, 'sitemap')
if (input.mediaDir !== undefined)
mediaReal = await validateInputPath(input.mediaDir, 'directory', allowedRoots, 'media')
if (wxrReal === undefined && sitemapReal === undefined && mediaReal === undefined) {
throw new SourceRegisterError('NO_INPUTS', '未提供任何输入(wxrPath / sitemapPath / mediaDir 至少其一)')
if (input.mediaMapPath !== undefined)
mediaMapReal = await validateInputPath(input.mediaMapPath, 'file', allowedRoots, 'media-map')
if (
wxrReal === undefined &&
sitemapReal === undefined &&
mediaReal === undefined &&
mediaMapReal === undefined
) {
throw new SourceRegisterError(
'NO_INPUTS',
'未提供任何输入(wxrPath / sitemapPath / mediaDir / mediaMapPath 至少其一)',
)
}
// 阶段 1:摄取(hash + 副本落位)——顺序固定 wxr → sitemap → media(报告输入清单稳定序)
// 阶段 1:摄取(hash + 副本落位)——顺序固定 wxr → sitemap → media → media-map(报告输入清单稳定序)
const ingested: Ingested[] = []
if (wxrReal !== undefined) ingested.push(await ingestFile('wxr', wxrReal, storageRoot))
if (sitemapReal !== undefined) ingested.push(await ingestFile('sitemap', sitemapReal, storageRoot))
if (mediaReal !== undefined) ingested.push(await ingestMediaDir(mediaReal, storageRoot))
if (mediaMapReal !== undefined)
ingested.push(await ingestFile('manual_url_list', mediaMapReal, storageRoot))
// 阶段 2:DB 写入——单事务(site→connection→snapshots 全成或全回滚;"任何失败零落库"
// 由此成立)。connection 的 select-then-insert 在事务内靠 advisory xact lock 串行化。
+87
View File
@@ -0,0 +1,87 @@
# 媒体映射侧车导出证据(Plan 3 · Task 0,评审 Issue 1)
背景(评审实证):真实媒体目录 `D:/project/cms/storage/media/54f91c49-fffb-4c58-87c8-6c2cf6aaef79` 为扁平 GUID 命名(2625 文件,无 yyyy/mm 子目录),与 WXR attachment_url(`https://hesupipe.com/wp-content/uploads/<yyyy>/<mm>/<file>` 形态)**零精确命中**——唯一映射在旧 cms 库 `media_assets` 表(`source_url` → `file_path`)。本任务把该映射只读导出为侧车 JSON 并登记为 `manual_url_list` 快照,作为阶段 3 导入管道对拍三级中第二级的数据源。
## 导出
| 项 | 值 |
|---|---|
| 日期 | 2026-10-21 |
| 数据源 | 旧 cms PostgreSQL(容器 `infrastructure-postgres-1`,宿主 5433;凭证按 compose 只读环境 `cms`/`cms`) |
| 表/过滤 | `media_assets`,`site_id = '54f91c49-fffb-4c58-87c8-6c2cf6aaef79' AND source_url IS NOT NULL`(867 行中 8 行 source_url 为 NULL 不带出) |
| 导出行数 | **859**(与评审预期 859 一致;file_path 全部非空——另行只读核过 0 行空值) |
| 输出 | `storage/incoming/media-map-hesupipe.json`(143,945 B,SHA-256 `c935d444f52b6bb8d9fa5e511066cac5b0a684676986041882cfbc54a0da6a2e`;storage/ 已 gitignore) |
| 导出脚本 | `scripts/export-media-map.mjs`(幂等可重跑——重跑同 SELECT 覆写同路径,两次运行 SHA-256 一致已验证) |
实际执行的命令原文(仓库根 `D:\project\cms-one`):
```bash
pnpm exec node scripts/export-media-map.mjs --audit "D:/project/cms/storage/wxr/fd04d6ba8a1316b6.xml"
```
脚本内执行的唯一一条 cms 侧语句(经 `docker exec … psql --csv -t -c`,child_process execFile 参数数组传递、不经 shell):
```sql
SELECT source_url, file_path FROM media_assets
WHERE site_id = '54f91c49-fffb-4c58-87c8-6c2cf6aaef79' AND source_url IS NOT NULL
ORDER BY source_url
```
连接方式说明(任务给了二选一):选 `docker exec psql`——宿主无需装 pg 客户端、凭证由容器内环境承担;备选 node-postgres 直连 5433 需在脚本里钉死 compose 凭证,弃。
## 覆盖率预估(侧车 × WXR attachment_url 对照)
对照口径:WXR(`fd04d6ba8a1316b6.xml`,855 个 `<wp:attachment_url>`、去重后仍 855)逐条对侧车 `sourceUrl` 集合 join。归一化口径(估计用):去首尾空白 + 去尾斜杠 + 整串小写。
| 指标 | 值 |
|---|---|
| WXR attachment_url(去重) | 855 |
| 侧车条目 | 859 |
| **精确命中**(原文相等) | **836** |
| 仅归一后命中(大小写/尾斜杠救回) | **0**(本数据集无需归一化救援——未命中即真不在侧车) |
| **未命中** | **19** |
| 命中率(去重口径) | **97.8%**(836/855) |
| 侧车未被 WXR 引用的条目(悬空行) | 23(859 − 836;对拍三级用不上,仅记录) |
自洽核对:836 命中 + 19 未命中 = 855;859 侧车 − 836 被引用 = 23 悬空。
### 未命中清单(全部 19 条)
特征:**全部是 pdf/mp4/zip/txt/特殊字符文件名**,无常规 png/jpg——推测旧 cms 迁移时这批非图片资产未入 `media_assets`(或入在其他 site)。阶段 3 对拍时它们将落入第三级 missing 报告(按计划不插 media_assets 行,只进报告)。
```text
https://hesupipe.com/wp-content/uploads/2019/12/WPPOP-Introduction.pdf
https://hesupipe.com/wp-content/uploads/2019/12/WPPOP-Introduction.zip
https://hesupipe.com/wp-content/uploads/2019/12/WPPOP-Introduction.txt
https://hesupipe.com/wp-content/uploads/2019/12/wppop-ipad-pro-video.mp4
https://hesupipe.com/wp-content/uploads/2023/12/hesu-video.mp4
https://hesupipe.com/wp-content/uploads/2023/12/HESU-PIPE-ABOUNT.mp4
https://hesupipe.com/wp-content/uploads/2023/12/418311fef2a99d2b4bc018.mp4
https://hesupipe.com/wp-content/uploads/2023/12/hesuvideo.mp4
https://hesupipe.com/wp-content/uploads/2023/05/same-as-never_1_207_translate.pdf
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-1.jpg
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-1-1.jpg
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-2.jpg
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-3.jpg
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-6.jpg
https://hesupipe.com/wp-content/uploads/2023/12/HDPE-Double-Wall-Corrugated-Pipe.pdf
https://hesupipe.com/wp-content/uploads/2023/12/HESU-PPR-PIPES-FITTINGS.pdf
https://hesupipe.com/wp-content/uploads/2023/12/HESU-PVC-PIPE-FITTINGS.pdf
https://hesupipe.com/wp-content/uploads/2023/12/HESU-HDPE-WATER-PIPES-FITTINGS.pdf
https://hesupipe.com/wp-content/uploads/2024/10/HDPE-carat-pipe-(2).jpg
```
## 登记冒烟(临时库,验后已 DROP)
`--media-map` 输入通道对真实侧车文件的 CLI 冒烟(临时库 `sf_smoke_mediamap_t0`,本项目 PG 5434,非 cms 库;验后 DROP 已执行):
```bash
DATABASE_URL_TEST="postgres://siteforge:siteforge_dev@localhost:5434/sf_smoke_mediamap_t0" \
pnpm sf source:register --site hesupipe-smoke --media-map storage/incoming/media-map-hesupipe.json
```
结果:`manual_url_list` 快照 1 行——sha256 `c935d444f52b…`(= 导出文件哈希)、byte_size 143945、metadata `entryCount=859`、adapter_version `manual-url-list@1`、storage_path 指向 content-addressed 副本;重跑复用(snapshot 复用/副本已存在,行数仍 1)。
## 只读声明
本任务对 `D:\project\cms`(旧 cms 仓库、其 PostgreSQL 容器、WXR 与媒体文件)**只执行了读取类操作**:`docker exec … psql` 上列 SELECT(及行数核对的 SELECT)与 WXR 文件读取(grep/哈希/正则抽取)。未对 cms 数据库执行任何写语句(INSERT/UPDATE/DELETE/DDL 均无)、未写 cms 文件系统、未挂载任何卷。本项目(cms-one)侧写入仅限自身 storage/ 与 artifacts/。连接凭证信息仅从 compose 文件读得,未修改任何配置。
+223
View File
@@ -0,0 +1,223 @@
#!/usr/bin/env node
// 媒体映射侧车导出(Plan 3 · Task 0,评审 Issue 1——真实数据实证):
// 旧 cms 库(容器 infrastructure-postgres-1,宿主 5433)media_assets 表存有该站唯一的
// WXR attachment_url(yyyy/mm 形态)→ 媒体清单扁平 GUID 文件名 映射。本脚本只读导出为
// [{sourceUrl, filePath}] JSON 数组,供 `sf source:register --media-map` 登记
// kind=manual_url_list 快照(导入管道对拍三级中的第二级数据源)。
//
// 连接方式(二选一里选了 docker exec psql):经 child_process execFile 调
// `docker exec <container> psql --csv -t -c "<SELECT>"`——宿主无需装 pg 客户端,
// SQL 作为单参数传递不经 shell(无注入面),凭证由容器内环境承担(compose:cms/cms)。
// 备选 node-postgres 直连 5433 需在脚本里钉死 compose 凭证且依赖仓库外端口可达——弃。
//
// **只读纪律**:本脚本对 cms 只执行下方 EXPort_SQL 一条 SELECT(无任何写语句、
// 不挂载、不 --user 变更);对 cms 文件系统零接触。
//
// 幂等可重跑:输出文件整体覆写(重跑 = 重新 SELECT + 重写同一路径),无追加语义。
//
// 用法(仓库根):
// pnpm exec node scripts/export-media-map.mjs # 缺省站点/输出路径
// pnpm exec node scripts/export-media-map.mjs --audit <wxr路径> # 追加覆盖率对照(只读)
// 可选覆盖:--site-id <uuid> / --out <path> / --container <name> / --user <u> / --db <d>
import { createHash } from 'node:crypto'
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { execFile as execFileCallback } from 'node:child_process'
import { promisify } from 'node:util'
const execFile = promisify(execFileCallback)
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
/** 缺省导出对象:hesupipe 站(评审钉死);输出落 storage/incoming(常规登记通道,已 gitignore) */
const DEFAULT_SITE_ID = '54f91c49-fffb-4c58-87c8-6c2cf6aaef79'
const DEFAULT_OUT = join(REPO_ROOT, 'storage', 'incoming', 'media-map-hesupipe.json')
const DEFAULT_CONTAINER = 'infrastructure-postgres-1'
const DEFAULT_USER = 'cms'
const DEFAULT_DB = 'cms'
/** 唯一一条对 cms 执行的 SQL(只读 SELECT;source_url IS NOT NULL——867 中 8 行 NULL 不带出) */
function exportSql(siteId) {
return (
`SELECT source_url, file_path FROM media_assets ` +
`WHERE site_id = '${siteId}' AND source_url IS NOT NULL ORDER BY source_url`
)
}
/**
* 极简 CSV 行解析(psql --csv -t 输出):支持双引号包裹字段(内嵌 "" 转义与逗号)。
* 本数据域(URL/GUID 文件名)不出现字段内换行,按行切分安全。
*/
function parseCsvLine(line) {
const fields = []
let cur = ''
let inQuotes = false
for (let i = 0; i < line.length; i++) {
const ch = line[i]
if (inQuotes) {
if (ch === '"') {
if (line[i + 1] === '"') {
cur += '"'
i++
} else {
inQuotes = false
}
} else {
cur += ch
}
} else if (ch === '"') {
inQuotes = true
} else if (ch === ',') {
fields.push(cur)
cur = ''
} else {
cur += ch
}
}
fields.push(cur)
return fields
}
/** URL 归一(覆盖率对照用):去首尾空白 + 去尾斜杠(可多个)+ 整串小写——估计口径,非精确匹配口径 */
function normalizeUrl(url) {
return url.trim().replace(/\/+$/, '').toLowerCase()
}
/** 覆盖率对照(只读):侧车 source_url × WXR attachment_url 归一化 join——精确命中/未命中/未命中清单 */
async function auditAgainstWxr(mapPath, wxrPath) {
const map = JSON.parse(await readFile(mapPath, 'utf8'))
const sidecarByExact = new Set(map.map((e) => e.sourceUrl))
const sidecarByNormalized = new Set(map.map((e) => normalizeUrl(e.sourceUrl)))
const wxr = await readFile(wxrPath, 'utf8')
const attachmentUrls = [...wxr.matchAll(/<wp:attachment_url><!\[CDATA\[([^\]]*)\]\]><\/wp:attachment_url>/g)].map(
(m) => m[1],
)
const seen = new Set()
const hitsExact = [] // 归一前即完全相等
const hitsNormalized = [] // 仅归一后相等(大小写/尾斜杠救回)
const misses = []
for (const url of attachmentUrls) {
if (seen.has(url)) continue
seen.add(url)
if (sidecarByExact.has(url)) {
hitsExact.push(url)
} else if (sidecarByNormalized.has(normalizeUrl(url))) {
hitsNormalized.push(url)
} else {
misses.push(url)
}
}
const wxrNormalized = new Set([...seen].map(normalizeUrl))
const sidecarHit = hitsExact.length + hitsNormalized.length
return {
wxrAttachmentUrls: attachmentUrls.length,
wxrDistinct: seen.size,
sidecarEntries: map.length,
hitsExact: hitsExact.length,
hitsNormalizedOnly: hitsNormalized.length,
misses: misses.length,
hitRate: seen.size === 0 ? null : sidecarHit / seen.size,
missList: misses.slice(0, 20),
missListTruncated: misses.length > 20,
/** 反向:侧车中 WXR(含归一口径)从未引用的条目数——导入三级对拍用不上的悬空行 */
sidecarUnreferenced: map.filter((e) => !wxrNormalized.has(normalizeUrl(e.sourceUrl))).length,
}
}
/** --key <v> / --key=<v> 取值;未知参数报错(防静默拼错键名导出错误范围) */
function parseArgs(argv) {
const defaults = {
siteId: DEFAULT_SITE_ID,
out: DEFAULT_OUT,
container: DEFAULT_CONTAINER,
user: DEFAULT_USER,
db: DEFAULT_DB,
auditWxr: undefined,
}
const keys = { '--site-id': 'siteId', '--out': 'out', '--container': 'container', '--user': 'user', '--db': 'db', '--audit': 'auditWxr' }
for (let i = 0; i < argv.length; i++) {
const arg = argv[i]
let key = arg
let value = argv[i + 1]
if (arg.includes('=')) {
key = arg.slice(0, arg.indexOf('='))
value = arg.slice(arg.indexOf('=') + 1)
} else {
i++
}
const field = keys[key]
if (field === undefined) throw new Error(`未知参数:${arg}(支持 ${Object.keys(keys).join(' ')})`)
if (value === undefined || value === '') throw new Error(`${key} 缺少值参数`)
defaults[field] = value
}
return defaults
}
async function main() {
const args = parseArgs(process.argv.slice(2))
const sql = exportSql(args.siteId)
process.stderr.write(`[export-media-map] 只读查询:docker exec ${args.container} psql -U ${args.user} -d ${args.db}\n`)
process.stderr.write(`[export-media-map] SQL:${sql}\n`)
const { stdout } = await execFile('docker', [
'exec',
args.container,
'psql',
'-U',
args.user,
'-d',
args.db,
'--csv',
'-t',
'-c',
sql,
])
const entries = stdout
.split(/\r?\n/)
.filter((line) => line !== '')
.map((line) => {
const [sourceUrl, filePath] = parseCsvLine(line)
if (sourceUrl === undefined || filePath === undefined || filePath === '') {
throw new Error(`CSV 行字段不全(预期 2 列):${line}`)
}
return { sourceUrl, filePath }
})
// SQL ORDER BY source_url 已保序;此处复核非降序(防 psql 输出形态意外变化静默乱序)
for (let i = 1; i < entries.length; i++) {
if (entries[i - 1].sourceUrl > entries[i].sourceUrl) {
throw new Error('输出未按 source_url 排序——psql 输出形态与预期不符,中止(不写半坏文件)')
}
}
const json = `${JSON.stringify(entries, null, 2)}\n`
const outPath = resolve(args.out)
await mkdir(dirname(outPath), { recursive: true })
await writeFile(outPath, json, 'utf8')
const outStat = await stat(outPath)
const sha256 = createHash('sha256').update(json, 'utf8').digest('hex')
process.stderr.write(`[export-media-map] 导出 ${entries.length} 条 → ${outPath}\n`)
process.stderr.write(`[export-media-map] bytes=${outStat.size} sha256=${sha256}\n`)
// 可选覆盖率对照(--audit <wxr>):证据文档数字的复算通道
if (args.auditWxr !== undefined) {
const audit = await auditAgainstWxr(outPath, resolve(args.auditWxr))
process.stderr.write(`[export-media-map] 覆盖率对照(归一:去尾斜杠+小写):\n`)
process.stderr.write(` WXR attachment_url:${audit.wxrAttachmentUrls}(去重 ${audit.wxrDistinct})\n`)
process.stderr.write(` 侧车条目:${audit.sidecarEntries}\n`)
process.stderr.write(` 精确命中:${audit.hitsExact};归一命中:${audit.hitsNormalizedOnly};未命中:${audit.misses}\n`)
process.stderr.write(` 命中率(去重口径):${audit.hitRate === null ? 'n/a' : `${(audit.hitRate * 100).toFixed(1)}%`}\n`)
process.stderr.write(` 侧车未被 WXR 引用的条目:${audit.sidecarUnreferenced}\n`)
if (audit.misses > 0) {
process.stderr.write(` 未命中清单(前 ${audit.missList.length}${audit.missListTruncated ? `/${audit.misses}` : ''} 条):\n`)
for (const url of audit.missList) process.stderr.write(` - ${url}\n`)
}
}
}
main().catch((err) => {
process.stderr.write(`[export-media-map] 失败:${err instanceof Error ? err.message : String(err)}\n`)
process.exitCode = 1
})
+11 -3
View File
@@ -66,11 +66,12 @@ program
program
.command('source:register')
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录,可多选)')
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录 / 媒体映射侧车 JSON,可多选)')
.requiredOption('--site <key>', '站点 key(不存在则自动创建)')
.option('--wxr <path>', 'WXR 文件路径')
.option('--sitemap <path>', 'sitemap 文件路径')
.option('--media <dir>', '媒体目录路径(已解包目录,递归登记)')
.option('--media-map <path>', '媒体映射侧车 JSON 路径([{sourceUrl, filePath}] 数组,登记为 manual_url_list 快照)')
.option('--allow <dir...>', '额外输入白名单根目录(可多个;默认含 storage/incoming 与当前目录)', [])
.option('--name <name>', '新建站点时的显示名(缺省 = site key)')
.option('--locale <locale>', '新建站点时的默认 locale(缺省 en)')
@@ -80,13 +81,19 @@ program
wxr?: string
sitemap?: string
media?: string
mediaMap?: string
allow: string[]
name?: string
locale?: string
origin?: string
}) => {
if (opts.wxr === undefined && opts.sitemap === undefined && opts.media === undefined) {
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media 之一')
if (
opts.wxr === undefined &&
opts.sitemap === undefined &&
opts.media === undefined &&
opts.mediaMap === undefined
) {
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media / --media-map 之一')
}
// fail fast:建站默认列在服务端只在"站点新建"时生效——先做形状校验,避免跑完摄取才报错
if (opts.origin !== undefined) {
@@ -116,6 +123,7 @@ program
if (opts.wxr !== undefined) input.wxrPath = opts.wxr
if (opts.sitemap !== undefined) input.sitemapPath = opts.sitemap
if (opts.media !== undefined) input.mediaDir = opts.media
if (opts.mediaMap !== undefined) input.mediaMapPath = opts.mediaMap
const defaultsPassed = opts.name !== undefined || opts.locale !== undefined || opts.origin !== undefined
if (defaultsPassed) {
const defaults: NonNullable<RegisterInput['siteDefaults']> = {}
+109 -1
View File
@@ -12,7 +12,9 @@
// ENOENT / 类型不符均清晰报错且零落库;
// 6. 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机);
// 7. 并发登记同输入恰好一组行(DB 单事务 + 唯一约束/advisory lock 收敛)、双方同 id;
// 8. 空媒体目录拒绝(EMPTY_MEDIA_DIR)、metadata 固化 storageRoot、siteCreated 语义。
// 8. 空媒体目录拒绝(EMPTY_MEDIA_DIR)、metadata 固化 storageRoot、siteCreated 语义;
// 9. media-map 侧车输入(Plan 3 Task 0):kind=manual_url_list 快照、sha256=文件哈希、
// metadata 含条目数、幂等重跑复用;非法 JSON(非数组/条目字段缺失)→ INVALID_MEDIA_MAP 零落库。
// 运行于 globalSetup 临时库(ADR-015);storage/报告根用 os.tmpdir 临时目录,finally 清理。
// 注:符号链接跳过用目录 junction 验证(Windows 免特权;非 Windows 退化为普通符号链接);
// 创建失败(极端沙箱)时该断言静默放行——junction 创建不是本测试的被测契约。
@@ -481,6 +483,112 @@ describe('source:register 服务', () => {
})
})
it('media-map 输入:kind=manual_url_list 快照(sha256=文件哈希、metadata 含条目数),幂等重跑复用', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
// 侧车 JSON(Plan 3 Task 0 形态):[{sourceUrl, filePath}] 数组——真实导出脚本的输出同构
const entries = [
{
sourceUrl: 'https://hesupipe.com/wp-content/uploads/2019/12/Amazon.png',
filePath: 'f249747a-50d3-48ed-a28a-981cf1d5ee7f.png',
},
{
sourceUrl: 'https://hesupipe.com/wp-content/uploads/2019/12/map-location.jpg',
filePath: '0e6d0fd7-8d5f-4f57-a541-40a8f2cb9c02.jpg',
},
]
const mapContent = `${JSON.stringify(entries, null, 2)}\n`
const mapFile = join(ctx.work, 'media-map-hesupipe.json')
await writeFile(mapFile, mapContent, 'utf8')
const key = `sfreg-${rand()}`
const makeMapInput = (): RegisterInput => ({
siteKey: key,
mediaMapPath: mapFile,
allowedRoots: ctx.allowedRoots,
storageRoot: ctx.storageRoot,
reportRoot: ctx.reportRoot,
})
const r1 = await registerSource(db, makeMapInput())
try {
expect(r1.snapshots).toHaveLength(1)
const s = r1.snapshots[0]!
expect(s.kind).toBe('manual_url_list')
// sha256 = 文件内容哈希(与 wxr/sitemap 文件输入同口径)
expect(s.sha256).toBe(sha256(mapContent))
expect(s.byteSize).toBe(Buffer.byteLength(mapContent))
expect(s.storagePath).toBe(`snapshots/manual_url_list/${s.sha256.slice(0, 2)}/${s.sha256}`)
expect(existsSync(join(ctx.storageRoot, s.storagePath))).toBe(true)
expect(await readFile(join(ctx.storageRoot, s.storagePath), 'utf8')).toBe(mapContent)
// metadata 含条目数(Task 0 契约)与 sourceName;storageRoot 固化
expect(s.metadata).toMatchObject({ entryCount: 2, sourceName: 'media-map-hesupipe.json' })
expect((s.metadata as { storageRoot: string }).storageRoot).toBe(resolve(ctx.storageRoot))
// DB 行:kind 枚举值落库正确、metadata jsonb 含条目数
const rows = (await db.execute(
`select kind, sha256, byte_size, storage_path, metadata from source_snapshots where site_id = '${r1.siteId}'`,
)) as { kind: string; sha256: string; byte_size: string; storage_path: string; metadata: { entryCount: number } }[]
expect(rows).toHaveLength(1)
expect(rows[0]!.kind).toBe('manual_url_list')
expect(rows[0]!.sha256).toBe(sha256(mapContent))
expect(Number(rows[0]!.byte_size)).toBe(Buffer.byteLength(mapContent))
expect(rows[0]!.storage_path).toBe(s.storagePath)
expect(rows[0]!.metadata.entryCount).toBe(2)
// 幂等重跑:UNIQUE(site,kind,sha256) 复用既有行,不新增
const r2 = await registerSource(db, makeMapInput())
expect(r2.siteId).toBe(r1.siteId)
expect(r2.snapshots).toHaveLength(1)
expect(r2.snapshots[0]!.id).toBe(s.id)
expect(r2.snapshots[0]!.reused).toBe(true)
const counts = (await db.execute(
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns`,
)) as any
expect(counts[0]).toEqual({ snaps: '1', conns: '1' })
} finally {
await cleanupSiteRows(db, r1.siteId)
}
})
})
})
it('media-map 非法内容(非 JSON / 非数组 / 条目字段缺失或非串):INVALID_MEDIA_MAP 报错且零落库', async () => {
await withTestDb(async (db) => {
await withCtx(async (ctx) => {
const key = `sfreg-${rand()}`
const cases: { label: string; content: string }[] = [
{ label: '非 JSON 文本', content: 'not json at all\n' },
{ label: 'JSON 但非数组', content: '{"sourceUrl": "x"}\n' },
{ label: '数组但条目缺 filePath', content: '[{"sourceUrl": "https://a.example/1.png"}]\n' },
{ label: '数组但条目字段非字符串', content: '[{"sourceUrl": 1, "filePath": "a.png"}]\n' },
]
for (const c of cases) {
const mapFile = join(ctx.work, `bad-map-${cases.indexOf(c)}.json`)
await writeFile(mapFile, c.content, 'utf8')
const err: SourceRegisterError = await registerSource(db, {
siteKey: key,
mediaMapPath: mapFile,
allowedRoots: ctx.allowedRoots,
storageRoot: ctx.storageRoot,
reportRoot: ctx.reportRoot,
}).then(
() => {
throw new Error(`用例「${c.label}」不应成功`)
},
(e: SourceRegisterError) => e,
)
expect(err, c.label).toBeInstanceOf(SourceRegisterError)
expect(err.code, c.label).toBe('INVALID_MEDIA_MAP')
expect(err.message, c.label).toMatch(/media-map/)
// 零落库:site 从未创建
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
expect(sites, c.label).toHaveLength(0)
}
})
})
})
it('isUniqueViolation 形状识别:drizzle 包裹(cause.code)/裸 postgres code/其他均正确', () => {
// drizzle 会把 postgres 错误包成 DrizzleQueryError:message 只剩 "Failed query",真 code 在 cause
const drizzleWrapped = Object.assign(new Error('Failed query: insert into sites ...'), {