feat(source): 媒体映射侧车导出与登记
Plan 3 Task 0(评审 Issue 1):旧 cms 库 media_assets 的 source_url→file_path 映射只读导出为侧车 JSON(859 条,覆盖率对照 WXR 855 attachment_url:精确命中 836 / 归一命中 0 / 未命中 19,97.8%),register 新增 --media-map 输入通道 (kind=manual_url_list 快照,形状校验 INVALID_MEDIA_MAP,metadata 含条目数, 幂等复用)。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
// source:register 输入登记服务(Plan 2 Task 7,spec §7.1 / ADR-004/008):
|
||||
// 把 WXR / sitemap / 媒体目录三类输入登记为 content-addressed 快照——
|
||||
// 把 WXR / sitemap / 媒体目录 / 媒体映射侧车(manual_url_list,Plan 3 Task 0)四类输入
|
||||
// 登记为 content-addressed 快照——
|
||||
// 副本落 storage/snapshots/<kind>/<sha256前2>/<sha256>(文件)或 <sha256>/(媒体整树),
|
||||
// 源输入文件从此不再被后续任务读取(导入只寻址副本)。
|
||||
//
|
||||
@@ -25,7 +26,7 @@
|
||||
// 新 uuid 重写)但不再被本服务清理——手工清理策略:CLI 未运行时整目录删除 .tmp/ 即可。
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { createReadStream, createWriteStream, existsSync } from 'node:fs'
|
||||
import { lstat, mkdir, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
@@ -49,13 +50,14 @@ export type SourceDb = NodePgDatabase<typeof schema>
|
||||
*/
|
||||
type SourceDbWriter = Pick<SourceDb, 'select' | 'insert' | 'execute'>
|
||||
|
||||
export type SourceSnapshotKind = 'wxr' | 'sitemap' | 'media_bundle'
|
||||
export type SourceSnapshotKind = 'wxr' | 'sitemap' | 'media_bundle' | 'manual_url_list'
|
||||
|
||||
/** 各 kind 快照的 adapter_version:wxr 用适配器包单源版本;sitemap/media 尚无解析器,占位钉死 */
|
||||
/** 各 kind 快照的 adapter_version:wxr 用适配器包单源版本;sitemap/media/media_map 尚无解析器,占位钉死 */
|
||||
const SNAPSHOT_ADAPTER_VERSION: Record<SourceSnapshotKind, string> = {
|
||||
wxr: `adapter-wp@${ADAPTER_VERSION}`,
|
||||
sitemap: 'sitemap@0',
|
||||
media_bundle: 'media-bundle@1',
|
||||
manual_url_list: 'manual-url-list@1',
|
||||
}
|
||||
|
||||
export type SourceRegisterErrorCode =
|
||||
@@ -66,6 +68,7 @@ export type SourceRegisterErrorCode =
|
||||
| 'INPUT_NOT_DIR'
|
||||
| 'EMPTY_MEDIA_DIR'
|
||||
| 'INPUT_READ_FAILED'
|
||||
| 'INVALID_MEDIA_MAP'
|
||||
| 'COPY_FAILED'
|
||||
|
||||
/** 登记失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */
|
||||
@@ -89,6 +92,12 @@ export interface RegisterInput {
|
||||
wxrPath?: string
|
||||
sitemapPath?: string
|
||||
mediaDir?: string
|
||||
/**
|
||||
* 媒体映射侧车 JSON(Plan 3 Task 0,评审 Issue 1):[{sourceUrl, filePath}] 数组——
|
||||
* 旧 cms 库 media_assets 导出的 URL→扁平文件名映射;登记为 kind=manual_url_list 快照。
|
||||
* 导入管道对拍三级中的第二级数据源(①清单 uploads 路径 → ②本侧车 → ③missing 报告)。
|
||||
*/
|
||||
mediaMapPath?: string
|
||||
/** 输入路径白名单(realpath 后必须位于其中之一);缺省 [repoRoot/storage/incoming] */
|
||||
allowedRoots?: string[]
|
||||
/** 快照存储根(副本落 <storageRoot>/snapshots/...);缺省 repoRoot/storage */
|
||||
@@ -256,9 +265,49 @@ interface Ingested {
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
/** 文件输入(wxr/sitemap):temp 副本上单遍 copy+hash → 按 sha 定名落位(已存在则命中复用) */
|
||||
/**
|
||||
* 校验 media-map 侧车 JSON 形状并数条目:[{sourceUrl: string, filePath: string}] 数组,
|
||||
* 每条两字段均为非空字符串。形状即契约(Plan 3 对拍二级的数据源)——脏输入在登记期
|
||||
* 拒绝(快照不可变,落库后再发现无法补救)。返回条目数入 metadata。
|
||||
*/
|
||||
async function validateMediaMap(path: string): Promise<number> {
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(path, 'utf8'))
|
||||
} catch (err) {
|
||||
throw new SourceRegisterError(
|
||||
'INVALID_MEDIA_MAP',
|
||||
`media-map 不是合法 JSON:${path}(${err instanceof Error ? err.message : String(err)})`,
|
||||
)
|
||||
}
|
||||
if (!Array.isArray(parsed)) {
|
||||
throw new SourceRegisterError(
|
||||
'INVALID_MEDIA_MAP',
|
||||
`media-map 顶层必须是数组([{sourceUrl, filePath}]):${path},实际 ${parsed === null ? 'null' : typeof parsed}`,
|
||||
)
|
||||
}
|
||||
for (let i = 0; i < parsed.length; i++) {
|
||||
const entry = parsed[i] as unknown
|
||||
const ok =
|
||||
entry !== null &&
|
||||
typeof entry === 'object' &&
|
||||
typeof (entry as { sourceUrl?: unknown }).sourceUrl === 'string' &&
|
||||
(entry as { sourceUrl: string }).sourceUrl !== '' &&
|
||||
typeof (entry as { filePath?: unknown }).filePath === 'string' &&
|
||||
(entry as { filePath: string }).filePath !== ''
|
||||
if (!ok) {
|
||||
throw new SourceRegisterError(
|
||||
'INVALID_MEDIA_MAP',
|
||||
`media-map 第 ${i} 条不满足形状 {sourceUrl: string, filePath: string}:${path}`,
|
||||
)
|
||||
}
|
||||
}
|
||||
return parsed.length
|
||||
}
|
||||
|
||||
/** 文件输入(wxr/sitemap/media-map):temp 副本上单遍 copy+hash → 按 sha 定名落位(已存在则命中复用) */
|
||||
async function ingestFile(
|
||||
kind: 'wxr' | 'sitemap',
|
||||
kind: 'wxr' | 'sitemap' | 'manual_url_list',
|
||||
src: string,
|
||||
storageRoot: string,
|
||||
): Promise<Ingested> {
|
||||
@@ -276,6 +325,17 @@ async function ingestFile(
|
||||
`${kind} 输入读取失败:${src}(${err instanceof Error ? err.message : String(err)})`,
|
||||
)
|
||||
}
|
||||
// media-map:形状校验在副本 rename 落位**之前**——脏输入不产 content-addressed 副本
|
||||
//(校验读 temp 副本而非源,登记窗口内源被改写不会使校验结果与哈希脱钩)
|
||||
let entryCount: number | undefined
|
||||
if (kind === 'manual_url_list') {
|
||||
try {
|
||||
entryCount = await validateMediaMap(tmp)
|
||||
} catch (err) {
|
||||
await rm(tmp, { force: true })
|
||||
throw err
|
||||
}
|
||||
}
|
||||
const rel = `snapshots/${kind}/${sha256.slice(0, 2)}/${sha256}`
|
||||
let copied: boolean
|
||||
try {
|
||||
@@ -286,6 +346,8 @@ async function ingestFile(
|
||||
`${kind} 快照副本落位失败:${join(storageRoot, rel)}(${err instanceof Error ? err.message : String(err)})`,
|
||||
)
|
||||
}
|
||||
const metadata: Record<string, unknown> = { sourceName: basename(src) }
|
||||
if (entryCount !== undefined) metadata.entryCount = entryCount
|
||||
return {
|
||||
kind,
|
||||
sourcePath: src,
|
||||
@@ -293,7 +355,7 @@ async function ingestFile(
|
||||
byteSize: bytes,
|
||||
storagePath: rel,
|
||||
copied,
|
||||
metadata: { sourceName: basename(src) },
|
||||
metadata,
|
||||
durationMs: performance.now() - t0,
|
||||
}
|
||||
}
|
||||
@@ -578,20 +640,33 @@ export async function registerSource(db: SourceDb, input: RegisterInput): Promis
|
||||
let wxrReal: string | undefined
|
||||
let sitemapReal: string | undefined
|
||||
let mediaReal: string | undefined
|
||||
let mediaMapReal: string | undefined
|
||||
if (input.wxrPath !== undefined) wxrReal = await validateInputPath(input.wxrPath, 'file', allowedRoots, 'wxr')
|
||||
if (input.sitemapPath !== undefined)
|
||||
sitemapReal = await validateInputPath(input.sitemapPath, 'file', allowedRoots, 'sitemap')
|
||||
if (input.mediaDir !== undefined)
|
||||
mediaReal = await validateInputPath(input.mediaDir, 'directory', allowedRoots, 'media')
|
||||
if (wxrReal === undefined && sitemapReal === undefined && mediaReal === undefined) {
|
||||
throw new SourceRegisterError('NO_INPUTS', '未提供任何输入(wxrPath / sitemapPath / mediaDir 至少其一)')
|
||||
if (input.mediaMapPath !== undefined)
|
||||
mediaMapReal = await validateInputPath(input.mediaMapPath, 'file', allowedRoots, 'media-map')
|
||||
if (
|
||||
wxrReal === undefined &&
|
||||
sitemapReal === undefined &&
|
||||
mediaReal === undefined &&
|
||||
mediaMapReal === undefined
|
||||
) {
|
||||
throw new SourceRegisterError(
|
||||
'NO_INPUTS',
|
||||
'未提供任何输入(wxrPath / sitemapPath / mediaDir / mediaMapPath 至少其一)',
|
||||
)
|
||||
}
|
||||
|
||||
// 阶段 1:摄取(hash + 副本落位)——顺序固定 wxr → sitemap → media(报告输入清单稳定序)
|
||||
// 阶段 1:摄取(hash + 副本落位)——顺序固定 wxr → sitemap → media → media-map(报告输入清单稳定序)
|
||||
const ingested: Ingested[] = []
|
||||
if (wxrReal !== undefined) ingested.push(await ingestFile('wxr', wxrReal, storageRoot))
|
||||
if (sitemapReal !== undefined) ingested.push(await ingestFile('sitemap', sitemapReal, storageRoot))
|
||||
if (mediaReal !== undefined) ingested.push(await ingestMediaDir(mediaReal, storageRoot))
|
||||
if (mediaMapReal !== undefined)
|
||||
ingested.push(await ingestFile('manual_url_list', mediaMapReal, storageRoot))
|
||||
|
||||
// 阶段 2:DB 写入——单事务(site→connection→snapshots 全成或全回滚;"任何失败零落库"
|
||||
// 由此成立)。connection 的 select-then-insert 在事务内靠 advisory xact lock 串行化。
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# 媒体映射侧车导出证据(Plan 3 · Task 0,评审 Issue 1)
|
||||
|
||||
背景(评审实证):真实媒体目录 `D:/project/cms/storage/media/54f91c49-fffb-4c58-87c8-6c2cf6aaef79` 为扁平 GUID 命名(2625 文件,无 yyyy/mm 子目录),与 WXR attachment_url(`https://hesupipe.com/wp-content/uploads/<yyyy>/<mm>/<file>` 形态)**零精确命中**——唯一映射在旧 cms 库 `media_assets` 表(`source_url` → `file_path`)。本任务把该映射只读导出为侧车 JSON 并登记为 `manual_url_list` 快照,作为阶段 3 导入管道对拍三级中第二级的数据源。
|
||||
|
||||
## 导出
|
||||
|
||||
| 项 | 值 |
|
||||
|---|---|
|
||||
| 日期 | 2026-10-21 |
|
||||
| 数据源 | 旧 cms PostgreSQL(容器 `infrastructure-postgres-1`,宿主 5433;凭证按 compose 只读环境 `cms`/`cms`) |
|
||||
| 表/过滤 | `media_assets`,`site_id = '54f91c49-fffb-4c58-87c8-6c2cf6aaef79' AND source_url IS NOT NULL`(867 行中 8 行 source_url 为 NULL 不带出) |
|
||||
| 导出行数 | **859**(与评审预期 859 一致;file_path 全部非空——另行只读核过 0 行空值) |
|
||||
| 输出 | `storage/incoming/media-map-hesupipe.json`(143,945 B,SHA-256 `c935d444f52b6bb8d9fa5e511066cac5b0a684676986041882cfbc54a0da6a2e`;storage/ 已 gitignore) |
|
||||
| 导出脚本 | `scripts/export-media-map.mjs`(幂等可重跑——重跑同 SELECT 覆写同路径,两次运行 SHA-256 一致已验证) |
|
||||
|
||||
实际执行的命令原文(仓库根 `D:\project\cms-one`):
|
||||
|
||||
```bash
|
||||
pnpm exec node scripts/export-media-map.mjs --audit "D:/project/cms/storage/wxr/fd04d6ba8a1316b6.xml"
|
||||
```
|
||||
|
||||
脚本内执行的唯一一条 cms 侧语句(经 `docker exec … psql --csv -t -c`,child_process execFile 参数数组传递、不经 shell):
|
||||
|
||||
```sql
|
||||
SELECT source_url, file_path FROM media_assets
|
||||
WHERE site_id = '54f91c49-fffb-4c58-87c8-6c2cf6aaef79' AND source_url IS NOT NULL
|
||||
ORDER BY source_url
|
||||
```
|
||||
|
||||
连接方式说明(任务给了二选一):选 `docker exec psql`——宿主无需装 pg 客户端、凭证由容器内环境承担;备选 node-postgres 直连 5433 需在脚本里钉死 compose 凭证,弃。
|
||||
|
||||
## 覆盖率预估(侧车 × WXR attachment_url 对照)
|
||||
|
||||
对照口径:WXR(`fd04d6ba8a1316b6.xml`,855 个 `<wp:attachment_url>`、去重后仍 855)逐条对侧车 `sourceUrl` 集合 join。归一化口径(估计用):去首尾空白 + 去尾斜杠 + 整串小写。
|
||||
|
||||
| 指标 | 值 |
|
||||
|---|---|
|
||||
| WXR attachment_url(去重) | 855 |
|
||||
| 侧车条目 | 859 |
|
||||
| **精确命中**(原文相等) | **836** |
|
||||
| 仅归一后命中(大小写/尾斜杠救回) | **0**(本数据集无需归一化救援——未命中即真不在侧车) |
|
||||
| **未命中** | **19** |
|
||||
| 命中率(去重口径) | **97.8%**(836/855) |
|
||||
| 侧车未被 WXR 引用的条目(悬空行) | 23(859 − 836;对拍三级用不上,仅记录) |
|
||||
|
||||
自洽核对:836 命中 + 19 未命中 = 855;859 侧车 − 836 被引用 = 23 悬空。
|
||||
|
||||
### 未命中清单(全部 19 条)
|
||||
|
||||
特征:**全部是 pdf/mp4/zip/txt/特殊字符文件名**,无常规 png/jpg——推测旧 cms 迁移时这批非图片资产未入 `media_assets`(或入在其他 site)。阶段 3 对拍时它们将落入第三级 missing 报告(按计划不插 media_assets 行,只进报告)。
|
||||
|
||||
```text
|
||||
https://hesupipe.com/wp-content/uploads/2019/12/WPPOP-Introduction.pdf
|
||||
https://hesupipe.com/wp-content/uploads/2019/12/WPPOP-Introduction.zip
|
||||
https://hesupipe.com/wp-content/uploads/2019/12/WPPOP-Introduction.txt
|
||||
https://hesupipe.com/wp-content/uploads/2019/12/wppop-ipad-pro-video.mp4
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/hesu-video.mp4
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/HESU-PIPE-ABOUNT.mp4
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/418311fef2a99d2b4bc018.mp4
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/hesuvideo.mp4
|
||||
https://hesupipe.com/wp-content/uploads/2023/05/same-as-never_1_207_translate.pdf
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-1.jpg
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-1-1.jpg
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-2.jpg
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-3.jpg
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/Steel-Belt-Reinforced-PE-Corrugated-Pipe、-6.jpg
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/HDPE-Double-Wall-Corrugated-Pipe.pdf
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/HESU-PPR-PIPES-FITTINGS.pdf
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/HESU-PVC-PIPE-FITTINGS.pdf
|
||||
https://hesupipe.com/wp-content/uploads/2023/12/HESU-HDPE-WATER-PIPES-FITTINGS.pdf
|
||||
https://hesupipe.com/wp-content/uploads/2024/10/HDPE-carat-pipe-(2).jpg
|
||||
```
|
||||
|
||||
## 登记冒烟(临时库,验后已 DROP)
|
||||
|
||||
`--media-map` 输入通道对真实侧车文件的 CLI 冒烟(临时库 `sf_smoke_mediamap_t0`,本项目 PG 5434,非 cms 库;验后 DROP 已执行):
|
||||
|
||||
```bash
|
||||
DATABASE_URL_TEST="postgres://siteforge:siteforge_dev@localhost:5434/sf_smoke_mediamap_t0" \
|
||||
pnpm sf source:register --site hesupipe-smoke --media-map storage/incoming/media-map-hesupipe.json
|
||||
```
|
||||
|
||||
结果:`manual_url_list` 快照 1 行——sha256 `c935d444f52b…`(= 导出文件哈希)、byte_size 143945、metadata `entryCount=859`、adapter_version `manual-url-list@1`、storage_path 指向 content-addressed 副本;重跑复用(snapshot 复用/副本已存在,行数仍 1)。
|
||||
|
||||
## 只读声明
|
||||
|
||||
本任务对 `D:\project\cms`(旧 cms 仓库、其 PostgreSQL 容器、WXR 与媒体文件)**只执行了读取类操作**:`docker exec … psql` 上列 SELECT(及行数核对的 SELECT)与 WXR 文件读取(grep/哈希/正则抽取)。未对 cms 数据库执行任何写语句(INSERT/UPDATE/DELETE/DDL 均无)、未写 cms 文件系统、未挂载任何卷。本项目(cms-one)侧写入仅限自身 storage/ 与 artifacts/。连接凭证信息仅从 compose 文件读得,未修改任何配置。
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env node
|
||||
// 媒体映射侧车导出(Plan 3 · Task 0,评审 Issue 1——真实数据实证):
|
||||
// 旧 cms 库(容器 infrastructure-postgres-1,宿主 5433)media_assets 表存有该站唯一的
|
||||
// WXR attachment_url(yyyy/mm 形态)→ 媒体清单扁平 GUID 文件名 映射。本脚本只读导出为
|
||||
// [{sourceUrl, filePath}] JSON 数组,供 `sf source:register --media-map` 登记
|
||||
// kind=manual_url_list 快照(导入管道对拍三级中的第二级数据源)。
|
||||
//
|
||||
// 连接方式(二选一里选了 docker exec psql):经 child_process execFile 调
|
||||
// `docker exec <container> psql --csv -t -c "<SELECT>"`——宿主无需装 pg 客户端,
|
||||
// SQL 作为单参数传递不经 shell(无注入面),凭证由容器内环境承担(compose:cms/cms)。
|
||||
// 备选 node-postgres 直连 5433 需在脚本里钉死 compose 凭证且依赖仓库外端口可达——弃。
|
||||
//
|
||||
// **只读纪律**:本脚本对 cms 只执行下方 EXPort_SQL 一条 SELECT(无任何写语句、
|
||||
// 不挂载、不 --user 变更);对 cms 文件系统零接触。
|
||||
//
|
||||
// 幂等可重跑:输出文件整体覆写(重跑 = 重新 SELECT + 重写同一路径),无追加语义。
|
||||
//
|
||||
// 用法(仓库根):
|
||||
// pnpm exec node scripts/export-media-map.mjs # 缺省站点/输出路径
|
||||
// pnpm exec node scripts/export-media-map.mjs --audit <wxr路径> # 追加覆盖率对照(只读)
|
||||
// 可选覆盖:--site-id <uuid> / --out <path> / --container <name> / --user <u> / --db <d>
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { execFile as execFileCallback } from 'node:child_process'
|
||||
import { promisify } from 'node:util'
|
||||
|
||||
const execFile = promisify(execFileCallback)
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
|
||||
/** 缺省导出对象:hesupipe 站(评审钉死);输出落 storage/incoming(常规登记通道,已 gitignore) */
|
||||
const DEFAULT_SITE_ID = '54f91c49-fffb-4c58-87c8-6c2cf6aaef79'
|
||||
const DEFAULT_OUT = join(REPO_ROOT, 'storage', 'incoming', 'media-map-hesupipe.json')
|
||||
const DEFAULT_CONTAINER = 'infrastructure-postgres-1'
|
||||
const DEFAULT_USER = 'cms'
|
||||
const DEFAULT_DB = 'cms'
|
||||
|
||||
/** 唯一一条对 cms 执行的 SQL(只读 SELECT;source_url IS NOT NULL——867 中 8 行 NULL 不带出) */
|
||||
function exportSql(siteId) {
|
||||
return (
|
||||
`SELECT source_url, file_path FROM media_assets ` +
|
||||
`WHERE site_id = '${siteId}' AND source_url IS NOT NULL ORDER BY source_url`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 极简 CSV 行解析(psql --csv -t 输出):支持双引号包裹字段(内嵌 "" 转义与逗号)。
|
||||
* 本数据域(URL/GUID 文件名)不出现字段内换行,按行切分安全。
|
||||
*/
|
||||
function parseCsvLine(line) {
|
||||
const fields = []
|
||||
let cur = ''
|
||||
let inQuotes = false
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i]
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (line[i + 1] === '"') {
|
||||
cur += '"'
|
||||
i++
|
||||
} else {
|
||||
inQuotes = false
|
||||
}
|
||||
} else {
|
||||
cur += ch
|
||||
}
|
||||
} else if (ch === '"') {
|
||||
inQuotes = true
|
||||
} else if (ch === ',') {
|
||||
fields.push(cur)
|
||||
cur = ''
|
||||
} else {
|
||||
cur += ch
|
||||
}
|
||||
}
|
||||
fields.push(cur)
|
||||
return fields
|
||||
}
|
||||
|
||||
/** URL 归一(覆盖率对照用):去首尾空白 + 去尾斜杠(可多个)+ 整串小写——估计口径,非精确匹配口径 */
|
||||
function normalizeUrl(url) {
|
||||
return url.trim().replace(/\/+$/, '').toLowerCase()
|
||||
}
|
||||
|
||||
/** 覆盖率对照(只读):侧车 source_url × WXR attachment_url 归一化 join——精确命中/未命中/未命中清单 */
|
||||
async function auditAgainstWxr(mapPath, wxrPath) {
|
||||
const map = JSON.parse(await readFile(mapPath, 'utf8'))
|
||||
const sidecarByExact = new Set(map.map((e) => e.sourceUrl))
|
||||
const sidecarByNormalized = new Set(map.map((e) => normalizeUrl(e.sourceUrl)))
|
||||
const wxr = await readFile(wxrPath, 'utf8')
|
||||
const attachmentUrls = [...wxr.matchAll(/<wp:attachment_url><!\[CDATA\[([^\]]*)\]\]><\/wp:attachment_url>/g)].map(
|
||||
(m) => m[1],
|
||||
)
|
||||
const seen = new Set()
|
||||
const hitsExact = [] // 归一前即完全相等
|
||||
const hitsNormalized = [] // 仅归一后相等(大小写/尾斜杠救回)
|
||||
const misses = []
|
||||
for (const url of attachmentUrls) {
|
||||
if (seen.has(url)) continue
|
||||
seen.add(url)
|
||||
if (sidecarByExact.has(url)) {
|
||||
hitsExact.push(url)
|
||||
} else if (sidecarByNormalized.has(normalizeUrl(url))) {
|
||||
hitsNormalized.push(url)
|
||||
} else {
|
||||
misses.push(url)
|
||||
}
|
||||
}
|
||||
const wxrNormalized = new Set([...seen].map(normalizeUrl))
|
||||
const sidecarHit = hitsExact.length + hitsNormalized.length
|
||||
return {
|
||||
wxrAttachmentUrls: attachmentUrls.length,
|
||||
wxrDistinct: seen.size,
|
||||
sidecarEntries: map.length,
|
||||
hitsExact: hitsExact.length,
|
||||
hitsNormalizedOnly: hitsNormalized.length,
|
||||
misses: misses.length,
|
||||
hitRate: seen.size === 0 ? null : sidecarHit / seen.size,
|
||||
missList: misses.slice(0, 20),
|
||||
missListTruncated: misses.length > 20,
|
||||
/** 反向:侧车中 WXR(含归一口径)从未引用的条目数——导入三级对拍用不上的悬空行 */
|
||||
sidecarUnreferenced: map.filter((e) => !wxrNormalized.has(normalizeUrl(e.sourceUrl))).length,
|
||||
}
|
||||
}
|
||||
|
||||
/** --key <v> / --key=<v> 取值;未知参数报错(防静默拼错键名导出错误范围) */
|
||||
function parseArgs(argv) {
|
||||
const defaults = {
|
||||
siteId: DEFAULT_SITE_ID,
|
||||
out: DEFAULT_OUT,
|
||||
container: DEFAULT_CONTAINER,
|
||||
user: DEFAULT_USER,
|
||||
db: DEFAULT_DB,
|
||||
auditWxr: undefined,
|
||||
}
|
||||
const keys = { '--site-id': 'siteId', '--out': 'out', '--container': 'container', '--user': 'user', '--db': 'db', '--audit': 'auditWxr' }
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i]
|
||||
let key = arg
|
||||
let value = argv[i + 1]
|
||||
if (arg.includes('=')) {
|
||||
key = arg.slice(0, arg.indexOf('='))
|
||||
value = arg.slice(arg.indexOf('=') + 1)
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
const field = keys[key]
|
||||
if (field === undefined) throw new Error(`未知参数:${arg}(支持 ${Object.keys(keys).join(' ')})`)
|
||||
if (value === undefined || value === '') throw new Error(`${key} 缺少值参数`)
|
||||
defaults[field] = value
|
||||
}
|
||||
return defaults
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2))
|
||||
const sql = exportSql(args.siteId)
|
||||
|
||||
process.stderr.write(`[export-media-map] 只读查询:docker exec ${args.container} psql -U ${args.user} -d ${args.db}\n`)
|
||||
process.stderr.write(`[export-media-map] SQL:${sql}\n`)
|
||||
const { stdout } = await execFile('docker', [
|
||||
'exec',
|
||||
args.container,
|
||||
'psql',
|
||||
'-U',
|
||||
args.user,
|
||||
'-d',
|
||||
args.db,
|
||||
'--csv',
|
||||
'-t',
|
||||
'-c',
|
||||
sql,
|
||||
])
|
||||
|
||||
const entries = stdout
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line !== '')
|
||||
.map((line) => {
|
||||
const [sourceUrl, filePath] = parseCsvLine(line)
|
||||
if (sourceUrl === undefined || filePath === undefined || filePath === '') {
|
||||
throw new Error(`CSV 行字段不全(预期 2 列):${line}`)
|
||||
}
|
||||
return { sourceUrl, filePath }
|
||||
})
|
||||
// SQL ORDER BY source_url 已保序;此处复核非降序(防 psql 输出形态意外变化静默乱序)
|
||||
for (let i = 1; i < entries.length; i++) {
|
||||
if (entries[i - 1].sourceUrl > entries[i].sourceUrl) {
|
||||
throw new Error('输出未按 source_url 排序——psql 输出形态与预期不符,中止(不写半坏文件)')
|
||||
}
|
||||
}
|
||||
|
||||
const json = `${JSON.stringify(entries, null, 2)}\n`
|
||||
const outPath = resolve(args.out)
|
||||
await mkdir(dirname(outPath), { recursive: true })
|
||||
await writeFile(outPath, json, 'utf8')
|
||||
const outStat = await stat(outPath)
|
||||
const sha256 = createHash('sha256').update(json, 'utf8').digest('hex')
|
||||
|
||||
process.stderr.write(`[export-media-map] 导出 ${entries.length} 条 → ${outPath}\n`)
|
||||
process.stderr.write(`[export-media-map] bytes=${outStat.size} sha256=${sha256}\n`)
|
||||
|
||||
// 可选覆盖率对照(--audit <wxr>):证据文档数字的复算通道
|
||||
if (args.auditWxr !== undefined) {
|
||||
const audit = await auditAgainstWxr(outPath, resolve(args.auditWxr))
|
||||
process.stderr.write(`[export-media-map] 覆盖率对照(归一:去尾斜杠+小写):\n`)
|
||||
process.stderr.write(` WXR attachment_url:${audit.wxrAttachmentUrls}(去重 ${audit.wxrDistinct})\n`)
|
||||
process.stderr.write(` 侧车条目:${audit.sidecarEntries}\n`)
|
||||
process.stderr.write(` 精确命中:${audit.hitsExact};归一命中:${audit.hitsNormalizedOnly};未命中:${audit.misses}\n`)
|
||||
process.stderr.write(` 命中率(去重口径):${audit.hitRate === null ? 'n/a' : `${(audit.hitRate * 100).toFixed(1)}%`}\n`)
|
||||
process.stderr.write(` 侧车未被 WXR 引用的条目:${audit.sidecarUnreferenced}\n`)
|
||||
if (audit.misses > 0) {
|
||||
process.stderr.write(` 未命中清单(前 ${audit.missList.length}${audit.missListTruncated ? `/${audit.misses}` : ''} 条):\n`)
|
||||
for (const url of audit.missList) process.stderr.write(` - ${url}\n`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
process.stderr.write(`[export-media-map] 失败:${err instanceof Error ? err.message : String(err)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
+11
-3
@@ -66,11 +66,12 @@ program
|
||||
|
||||
program
|
||||
.command('source:register')
|
||||
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录,可多选)')
|
||||
.description('登记来源输入为 content-addressed 快照(WXR / sitemap / 媒体目录 / 媒体映射侧车 JSON,可多选)')
|
||||
.requiredOption('--site <key>', '站点 key(不存在则自动创建)')
|
||||
.option('--wxr <path>', 'WXR 文件路径')
|
||||
.option('--sitemap <path>', 'sitemap 文件路径')
|
||||
.option('--media <dir>', '媒体目录路径(已解包目录,递归登记)')
|
||||
.option('--media-map <path>', '媒体映射侧车 JSON 路径([{sourceUrl, filePath}] 数组,登记为 manual_url_list 快照)')
|
||||
.option('--allow <dir...>', '额外输入白名单根目录(可多个;默认含 storage/incoming 与当前目录)', [])
|
||||
.option('--name <name>', '新建站点时的显示名(缺省 = site key)')
|
||||
.option('--locale <locale>', '新建站点时的默认 locale(缺省 en)')
|
||||
@@ -80,13 +81,19 @@ program
|
||||
wxr?: string
|
||||
sitemap?: string
|
||||
media?: string
|
||||
mediaMap?: string
|
||||
allow: string[]
|
||||
name?: string
|
||||
locale?: string
|
||||
origin?: string
|
||||
}) => {
|
||||
if (opts.wxr === undefined && opts.sitemap === undefined && opts.media === undefined) {
|
||||
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media 之一')
|
||||
if (
|
||||
opts.wxr === undefined &&
|
||||
opts.sitemap === undefined &&
|
||||
opts.media === undefined &&
|
||||
opts.mediaMap === undefined
|
||||
) {
|
||||
throw new SourceRegisterError('NO_INPUTS', '至少提供 --wxr / --sitemap / --media / --media-map 之一')
|
||||
}
|
||||
// fail fast:建站默认列在服务端只在"站点新建"时生效——先做形状校验,避免跑完摄取才报错
|
||||
if (opts.origin !== undefined) {
|
||||
@@ -116,6 +123,7 @@ program
|
||||
if (opts.wxr !== undefined) input.wxrPath = opts.wxr
|
||||
if (opts.sitemap !== undefined) input.sitemapPath = opts.sitemap
|
||||
if (opts.media !== undefined) input.mediaDir = opts.media
|
||||
if (opts.mediaMap !== undefined) input.mediaMapPath = opts.mediaMap
|
||||
const defaultsPassed = opts.name !== undefined || opts.locale !== undefined || opts.origin !== undefined
|
||||
if (defaultsPassed) {
|
||||
const defaults: NonNullable<RegisterInput['siteDefaults']> = {}
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
// ENOENT / 类型不符均清晰报错且零落库;
|
||||
// 6. 报告落 artifacts/register/<runId>/reports/register.json(runId=时间戳+随机);
|
||||
// 7. 并发登记同输入恰好一组行(DB 单事务 + 唯一约束/advisory lock 收敛)、双方同 id;
|
||||
// 8. 空媒体目录拒绝(EMPTY_MEDIA_DIR)、metadata 固化 storageRoot、siteCreated 语义。
|
||||
// 8. 空媒体目录拒绝(EMPTY_MEDIA_DIR)、metadata 固化 storageRoot、siteCreated 语义;
|
||||
// 9. media-map 侧车输入(Plan 3 Task 0):kind=manual_url_list 快照、sha256=文件哈希、
|
||||
// metadata 含条目数、幂等重跑复用;非法 JSON(非数组/条目字段缺失)→ INVALID_MEDIA_MAP 零落库。
|
||||
// 运行于 globalSetup 临时库(ADR-015);storage/报告根用 os.tmpdir 临时目录,finally 清理。
|
||||
// 注:符号链接跳过用目录 junction 验证(Windows 免特权;非 Windows 退化为普通符号链接);
|
||||
// 创建失败(极端沙箱)时该断言静默放行——junction 创建不是本测试的被测契约。
|
||||
@@ -481,6 +483,112 @@ describe('source:register 服务', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('media-map 输入:kind=manual_url_list 快照(sha256=文件哈希、metadata 含条目数),幂等重跑复用', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
await withCtx(async (ctx) => {
|
||||
// 侧车 JSON(Plan 3 Task 0 形态):[{sourceUrl, filePath}] 数组——真实导出脚本的输出同构
|
||||
const entries = [
|
||||
{
|
||||
sourceUrl: 'https://hesupipe.com/wp-content/uploads/2019/12/Amazon.png',
|
||||
filePath: 'f249747a-50d3-48ed-a28a-981cf1d5ee7f.png',
|
||||
},
|
||||
{
|
||||
sourceUrl: 'https://hesupipe.com/wp-content/uploads/2019/12/map-location.jpg',
|
||||
filePath: '0e6d0fd7-8d5f-4f57-a541-40a8f2cb9c02.jpg',
|
||||
},
|
||||
]
|
||||
const mapContent = `${JSON.stringify(entries, null, 2)}\n`
|
||||
const mapFile = join(ctx.work, 'media-map-hesupipe.json')
|
||||
await writeFile(mapFile, mapContent, 'utf8')
|
||||
|
||||
const key = `sfreg-${rand()}`
|
||||
const makeMapInput = (): RegisterInput => ({
|
||||
siteKey: key,
|
||||
mediaMapPath: mapFile,
|
||||
allowedRoots: ctx.allowedRoots,
|
||||
storageRoot: ctx.storageRoot,
|
||||
reportRoot: ctx.reportRoot,
|
||||
})
|
||||
const r1 = await registerSource(db, makeMapInput())
|
||||
try {
|
||||
expect(r1.snapshots).toHaveLength(1)
|
||||
const s = r1.snapshots[0]!
|
||||
expect(s.kind).toBe('manual_url_list')
|
||||
// sha256 = 文件内容哈希(与 wxr/sitemap 文件输入同口径)
|
||||
expect(s.sha256).toBe(sha256(mapContent))
|
||||
expect(s.byteSize).toBe(Buffer.byteLength(mapContent))
|
||||
expect(s.storagePath).toBe(`snapshots/manual_url_list/${s.sha256.slice(0, 2)}/${s.sha256}`)
|
||||
expect(existsSync(join(ctx.storageRoot, s.storagePath))).toBe(true)
|
||||
expect(await readFile(join(ctx.storageRoot, s.storagePath), 'utf8')).toBe(mapContent)
|
||||
// metadata 含条目数(Task 0 契约)与 sourceName;storageRoot 固化
|
||||
expect(s.metadata).toMatchObject({ entryCount: 2, sourceName: 'media-map-hesupipe.json' })
|
||||
expect((s.metadata as { storageRoot: string }).storageRoot).toBe(resolve(ctx.storageRoot))
|
||||
|
||||
// DB 行:kind 枚举值落库正确、metadata jsonb 含条目数
|
||||
const rows = (await db.execute(
|
||||
`select kind, sha256, byte_size, storage_path, metadata from source_snapshots where site_id = '${r1.siteId}'`,
|
||||
)) as { kind: string; sha256: string; byte_size: string; storage_path: string; metadata: { entryCount: number } }[]
|
||||
expect(rows).toHaveLength(1)
|
||||
expect(rows[0]!.kind).toBe('manual_url_list')
|
||||
expect(rows[0]!.sha256).toBe(sha256(mapContent))
|
||||
expect(Number(rows[0]!.byte_size)).toBe(Buffer.byteLength(mapContent))
|
||||
expect(rows[0]!.storage_path).toBe(s.storagePath)
|
||||
expect(rows[0]!.metadata.entryCount).toBe(2)
|
||||
|
||||
// 幂等重跑:UNIQUE(site,kind,sha256) 复用既有行,不新增
|
||||
const r2 = await registerSource(db, makeMapInput())
|
||||
expect(r2.siteId).toBe(r1.siteId)
|
||||
expect(r2.snapshots).toHaveLength(1)
|
||||
expect(r2.snapshots[0]!.id).toBe(s.id)
|
||||
expect(r2.snapshots[0]!.reused).toBe(true)
|
||||
const counts = (await db.execute(
|
||||
`select (select count(*) from source_snapshots where site_id = '${r1.siteId}') as snaps,
|
||||
(select count(*) from source_connections where site_id = '${r1.siteId}') as conns`,
|
||||
)) as any
|
||||
expect(counts[0]).toEqual({ snaps: '1', conns: '1' })
|
||||
} finally {
|
||||
await cleanupSiteRows(db, r1.siteId)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('media-map 非法内容(非 JSON / 非数组 / 条目字段缺失或非串):INVALID_MEDIA_MAP 报错且零落库', async () => {
|
||||
await withTestDb(async (db) => {
|
||||
await withCtx(async (ctx) => {
|
||||
const key = `sfreg-${rand()}`
|
||||
const cases: { label: string; content: string }[] = [
|
||||
{ label: '非 JSON 文本', content: 'not json at all\n' },
|
||||
{ label: 'JSON 但非数组', content: '{"sourceUrl": "x"}\n' },
|
||||
{ label: '数组但条目缺 filePath', content: '[{"sourceUrl": "https://a.example/1.png"}]\n' },
|
||||
{ label: '数组但条目字段非字符串', content: '[{"sourceUrl": 1, "filePath": "a.png"}]\n' },
|
||||
]
|
||||
for (const c of cases) {
|
||||
const mapFile = join(ctx.work, `bad-map-${cases.indexOf(c)}.json`)
|
||||
await writeFile(mapFile, c.content, 'utf8')
|
||||
const err: SourceRegisterError = await registerSource(db, {
|
||||
siteKey: key,
|
||||
mediaMapPath: mapFile,
|
||||
allowedRoots: ctx.allowedRoots,
|
||||
storageRoot: ctx.storageRoot,
|
||||
reportRoot: ctx.reportRoot,
|
||||
}).then(
|
||||
() => {
|
||||
throw new Error(`用例「${c.label}」不应成功`)
|
||||
},
|
||||
(e: SourceRegisterError) => e,
|
||||
)
|
||||
expect(err, c.label).toBeInstanceOf(SourceRegisterError)
|
||||
expect(err.code, c.label).toBe('INVALID_MEDIA_MAP')
|
||||
expect(err.message, c.label).toMatch(/media-map/)
|
||||
// 零落库:site 从未创建
|
||||
const sites = (await db.execute(`select id from sites where key = '${key}'`)) as any
|
||||
expect(sites, c.label).toHaveLength(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('isUniqueViolation 形状识别:drizzle 包裹(cause.code)/裸 postgres code/其他均正确', () => {
|
||||
// drizzle 会把 postgres 错误包成 DrizzleQueryError:message 只剩 "Failed query",真 code 在 cause
|
||||
const drizzleWrapped = Object.assign(new Error('Failed query: insert into sites ...'), {
|
||||
|
||||
Reference in New Issue
Block a user