Files
cms-website/apps/api/src/modules/build/render-snapshot.ts
T
浅念 5434afefff fix(build): 哈希链含尾换行;media 块 first-wins 确定性;job 复用只认 succeeded+崩溃回收;重写器边界锚定
审查六修(Fix 2/3 先于 Task 3 消费六文件落形):
- Fix 1 三哈希 = sha256(文件字节含尾换行)——哈希链与落盘文件互证;测试改按
  原始文件字节独立复算对拍
- Fix 2 media_usages orderBy(fieldPath, publicPath) + first-wins:同 fieldPath
  多资产引用取字典序首位、其余丢弃记 issue;两次物化字节一致有测试钉死
- Fix 3 job 复用只认 succeeded;failed/cancelled/stale 死行与非终态崩溃残留
  (旧行置 stale 宣判)都以 <key>#a<attempt+1> 后缀键建新行自动重跑(attempt
  延续谱系);失败终态化时清走半途 input/ 目录;两路径各有测试
- Fix 4 git rev-parse 等环境探测移出快照事务(单次探测传入)
- Fix 5 属性正则边界锚定 (^|\s):data-src 等连字属性不卷入(\b 会误判 '-s'
  交界)——data-src 命中索引也不改写的用例钉死
- Fix 6 media 引用面只 join ready 资产;非 ready 残留引用单独查出记 issue、
  不进 content.json media 块(quarantined 翻转测试)

354/354(350+4);typecheck 四段、lint 全绿。

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-22 08:12:59 +08:00

524 lines
22 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Render Snapshot 纯物化(Plan 4 Task 2,ADR-011、V2 spec §9.2、快线 §8.4):
// 从 published 修订 + route_registry + media ready 集合物化冻结六文件
// (RENDER_SNAPSHOT_FILES 契约清单),**db 只读、零写入**(磁盘仅做副本存在性探测)。
// 构建编排(job 状态机/落盘/快照行)在 build.service.ts——本文件只负责"读出什么"。
//
// 发布语义(ADR-010):只读 entry_locales.published_revision_id 指向的修订——current
// 分叉(发布后再导入产新修订)不影响本快照;发布后的人工/导入变化须重新 publish +
// 新 build 才进产物。published 指针悬空(目标修订行缺席,指针列无 FK)→ issue 如实
// 报告(§7.6 禁静默),该条目不进 content.json。
//
// bodyHtml URL 重写(计划 Task 2 钉死匹配键):正则提取 src/href → 剥 host 取 path →
// 与 media ready 集合的 public_path / source_url 路径段**双源**比对(覆盖绝对/根相对/
// 协议相对变体与查询串)→ 命中改写为根相对 public_path;未命中保留原样 + unresolved
// 计数(render-manifest.urlRewrite——内链 miss 是常态统计不是错误;外域热链同理)。
// data:/纯片段/相对形态无路径可比对,原样越过不计(与 media-writer 的 data: 口径一致)。
//
// canonical JSON 确定性:六文件全部以 canonicalJson(键递归字典序、无空白)+ 换行落盘
// ——文件字节(含尾换行)即哈希输入(content/route/asset 三哈希 = 各自**文件字节**的
// sha256——哈希链与落盘文件互证),同输入(含 generatedAt 时点参数)两次物化字节一致。
// 集合排序在 JS 侧钉死(UTF-16
// 序,不依赖 PG collation):内容按 (typeKey, slug, locale, contentId)、分类按 (key,
// objectType)、term 按 (sourceId, slug, id)、路由按 (routePath, queryKey nulls-first,
// host)、资产按 publicPath。
//
// 媒体实体路径解析(assets.json 的 copyPath):register 快照 metadata.storageRoot +
// storage_path(content-addressed 副本)为首选,仓库存储根同路径次之,storage/incoming
// 按清单文件名兜底(真实数据扁平 GUID 命名,basename 即唯一键);全部缺席 → issue
// 如实报告(复制防线在 Task 5 构建管线,此处不静默也不阻断)。
import { createHash } from 'node:crypto'
import { existsSync } from 'node:fs'
import { basename, dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { and, desc, eq, inArray, isNotNull, ne } from 'drizzle-orm'
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
import {
ADAPTER_VERSION,
MAPPING_VERSION,
PARSER_VERSION,
SANITIZER_VERSION,
} from '@sf/adapter-wp'
import { RENDER_SNAPSHOT_VERSION, type RenderSnapshotFile } from '@sf/contracts'
import { canonicalJson } from '../../common/canonical.js'
import {
contentEntries,
contentRevisions,
contentTerms,
entryLocales,
mediaAssets,
mediaUsages,
routeRegistry,
siteConfigRevisions,
sourceSnapshots,
taxonomies,
terms,
} from '../../db/schema.js'
import type * as schema from '../../db/schema.js'
/** 本文件位于 <repo>/apps/api/src/modules/build/ → 上五级是仓库根 */
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..')
/** 读取面:db 池与事务句柄都满足(build.service 以 repeatable read 事务传入) */
export type SnapshotDb = NodePgDatabase<typeof schema>
export type SnapshotReader = Pick<SnapshotDb, 'select'>
// ---- URL 重写(纯函数,测试直测) ----
/** media ready 集合的最小匹配面(public_path 与 source_url 双源建键) */
export interface MediaPathSource {
publicPath: string
sourceUrl: string
}
/**
* URL → 可比对路径:剥 host(绝对/协议相对)、剥查询/锚、保证首斜杠;
* data:/纯片段/相对形态返回 null(无路径可比对,原样越过)。
*/
function comparablePathOf(raw: string): string | null {
if (raw === '' || raw.startsWith('data:')) return null
let rest: string
if (raw.startsWith('//')) {
const pathStart = raw.indexOf('/', 2)
if (pathStart === -1) return null // //host 无路径证据——不比对('/' 会误伤首页)
rest = raw.slice(pathStart)
} else {
const abs = /^https?:\/\/[^/]+(.*)$/i.exec(raw)
if (abs !== null) {
rest = abs[1] === '' ? '/' : abs[1]!
} else if (raw.startsWith('/')) {
rest = raw
} else {
return null
}
}
const cut = rest.search(/[?#]/)
const path = cut === -1 ? rest : rest.slice(0, cut)
return path === '' ? '/' : path
}
/**
* media ready 路径索引:path(public_path 本形 + source_url 剥 host 的 path 段)→
* 该资产的 public_path(命中改写目标)。同 path 双源冲突时先登记者胜(register 的
* public_path 唯一键保证本形不冲突;跨源同 path 是数据面事实,任一命中即本地化)。
*/
export function buildMediaPathIndex(assets: MediaPathSource[]): Map<string, string> {
const index = new Map<string, string>()
for (const a of assets) {
index.set(a.publicPath, a.publicPath)
const src = comparablePathOf(a.sourceUrl)
if (src !== null && !index.has(src)) index.set(src, a.publicPath)
}
return index
}
/** 重写统计(入 render-manifest.urlRewrite 与 build 报告 issues 面) */
export interface RewriteStats {
/** 命中 ready 集合的引用数(含已是根相对形态的幂等命中) */
rewritten: number
/** 有路径可比对但未命中 ready 集合的引用数(保留原样) */
unresolved: number
}
/**
* bodyHtml 的 src/href URL 重写(正则提取——清洗信任链上游已保证属性形态规整;
* 门禁期的 HTML 解析复用清洗管线是 Task 6 职责,此处不引入解析器依赖)。
* 属性名边界锚定(审查 Fix 5):前缀必须是空白或串首——`data-src`/`data-href`
* 等连字属性不匹配不卷入(\b 会把 data-src 的 '-s' 交界误判为词边界;懒加载
* 属性的改写是渲染域决策,不是快照域义务)。
* 只替换属性值本身,引号/空白原样保留;命中值与原值相等时计数但零改写。
*/
export function rewriteBodyHtml(html: string, index: Map<string, string>): { html: string; stats: RewriteStats } {
const stats: RewriteStats = { rewritten: 0, unresolved: 0 }
const ATTR_RE = /(^|\s)(src|href)\s*=\s*(["'])([^"']*)\3/gi
const out = html.replace(ATTR_RE, (full, _prefix: string, _attr: string, quote: string, value: string) => {
const path = comparablePathOf(value)
if (path === null) return full
const hit = index.get(path)
if (hit === undefined) {
stats.unresolved++
return full
}
stats.rewritten++
if (hit === value) return full
// match 尾部 = value + 收引号——只换值,保留原引号与属性的原始空白
return `${full.slice(0, full.length - value.length - 1)}${hit}${quote}`
})
return { html: out, stats }
}
// ---- 物化 ----
/** 六文件的 canonical JSON 字符串(含尾换行——落盘字节即哈希输入 + '\n') */
export type RenderSnapshotFileContents = Record<RenderSnapshotFile, string>
export interface MaterializeInput {
siteId: string
/** 快照生成时点(ISO 字符串)——render-manifest.generatedAt;同输入 + 同时点 → 字节一致 */
generatedAt: string
}
export interface MaterializeResult {
files: RenderSnapshotFileContents
sourceHashes: {
content: string
route: string
asset: string
}
counts: {
contents: number
taxonomies: number
routes: number
assets: number
}
urlRewrite: RewriteStats
/** 物化期如实报告的问题(published 悬空/媒体副本缺席/不可服务路由形态)——进 build 报告 */
issues: string[]
/** site.json 来源的 config 修订(build_input_snapshots.site_config_revision_id 落值) */
siteConfigRevisionId: string
}
/** UTF-16 字典序比较器(与 canonicalJson 键排序同语义——钉死不依赖 PG collation) */
function cmp(a: string, b: string): number {
return a < b ? -1 : a > b ? 1 : 0
}
function cmpNull(a: string | null, b: string | null): number {
if (a === null && b === null) return 0
if (a === null) return -1
if (b === null) return 1
return cmp(a, b)
}
/** 三哈希的单一算法来源:sha256(文件字节)——canonicalJson + 尾换行后散列(与落盘文件互证) */
function hashOf(payload: unknown): string {
return createHash('sha256').update(`${canonicalJson(payload)}\n`).digest('hex')
}
/** 仓库存储根的 incoming 媒体兜底目录(真实数据:storage/incoming/<目录>/<扁平 GUID 文件>) */
const REPO_STORAGE = join(REPO_ROOT, 'storage')
/**
* 物化冻结 Render Snapshot 六文件(纯读 + 副本存在性探测)。
* 前置不变量:站点已有 published 指针与 config 修订(build.service 建行前检查;
* 直接调用方破坏不变量时以明错拒绝,不产残缺快照)。
*/
export async function materializeRenderSnapshot(
db: SnapshotReader,
input: MaterializeInput,
): Promise<MaterializeResult> {
const issues: string[] = []
const urlRewrite: RewriteStats = { rewritten: 0, unresolved: 0 }
// site.json:config 修订最新版(Task 1 首发布生成 v1——data={name,origin,locale,brand})
const cfgRows = await db
.select({ id: siteConfigRevisions.id, data: siteConfigRevisions.data })
.from(siteConfigRevisions)
.where(eq(siteConfigRevisions.siteId, input.siteId))
.orderBy(desc(siteConfigRevisions.version))
.limit(1)
const config = cfgRows[0]
if (config === undefined) {
throw new Error(`站点 ${input.siteId} 无 site_config_revisions(publish 首版不变量破裂)——拒绝物化`)
}
// content.json:published 修订(leftJoin 显式暴露悬空指针——§7.6 禁静默)
const localeRows = await db
.select({
contentId: contentEntries.id,
typeKey: contentEntries.typeKey,
locale: entryLocales.locale,
localeSlug: entryLocales.slug,
revisionId: contentRevisions.id,
checksum: contentRevisions.checksum,
data: contentRevisions.data,
})
.from(entryLocales)
.innerJoin(contentEntries, eq(entryLocales.contentEntryId, contentEntries.id))
.leftJoin(contentRevisions, eq(entryLocales.publishedRevisionId, contentRevisions.id))
.where(and(eq(entryLocales.siteId, input.siteId), isNotNull(entryLocales.publishedRevisionId)))
const published = localeRows.filter((r) => r.revisionId !== null)
for (const row of localeRows) {
if (row.revisionId === null) {
issues.push(
`published 指针悬空(entry=${row.contentId}/${row.locale}):published_revision_id 指向的修订行缺席(指针列无 FK)——该条目不进 content.json`,
)
}
}
// media ready 集合:资产面(assets.json)与 URL 重写索引共用的单一查询
const assetRows = await db
.select({
publicPath: mediaAssets.publicPath,
sourceUrl: mediaAssets.sourceUrl,
storagePath: mediaAssets.storagePath,
sha256: mediaAssets.sha256,
detectedMimeType: mediaAssets.detectedMimeType,
byteSize: mediaAssets.byteSize,
width: mediaAssets.width,
height: mediaAssets.height,
altText: mediaAssets.altText,
})
.from(mediaAssets)
.where(and(eq(mediaAssets.siteId, input.siteId), eq(mediaAssets.status, 'ready')))
assetRows.sort((a, b) => cmp(a.publicPath, b.publicPath))
const mediaIndex = buildMediaPathIndex(assetRows)
// published 修订的 media_usages 引用面(fieldPath → public_path;content.json 的 media 块)。
// 确定性(审查 Fix 2):fieldPath/publicPath 双键 orderBy 钉死 first-wins 的"首"——
// 同 fieldPath 多资产引用(PK 是 资产×修订×fieldPath,跨资产可重)时取字典序首位,
// 其余丢弃并记 issue(§7.6 禁静默);只 join ready 资产(media 块只呈现可服务引用,
// 审查 Fix 6)——非 ready 资产的残留引用单独查出版 issue,不进块。
const publishedIds = published.map((r) => r.revisionId!)
const usageMap = new Map<string, Map<string, string>>()
if (publishedIds.length > 0) {
const usages = await db
.select({
contentRevisionId: mediaUsages.contentRevisionId,
fieldPath: mediaUsages.fieldPath,
publicPath: mediaAssets.publicPath,
})
.from(mediaUsages)
.innerJoin(mediaAssets, eq(mediaUsages.mediaAssetId, mediaAssets.id))
.where(
and(
eq(mediaUsages.siteId, input.siteId),
inArray(mediaUsages.contentRevisionId, publishedIds),
eq(mediaAssets.status, 'ready'),
),
)
.orderBy(mediaUsages.fieldPath, mediaAssets.publicPath)
for (const u of usages) {
let byField = usageMap.get(u.contentRevisionId)
if (byField === undefined) {
byField = new Map()
usageMap.set(u.contentRevisionId, byField)
}
const first = byField.get(u.fieldPath)
if (first === undefined) {
byField.set(u.fieldPath, u.publicPath)
} else if (first !== u.publicPath) {
issues.push(
`同 fieldPath 多资产引用(revision=${u.contentRevisionId} fieldPath=${u.fieldPath}):first-wins 取 ${first},丢弃 ${u.publicPath}`,
)
}
}
// 非 ready 资产的残留引用(登记后状态翻转等数据面事实):不进 media 块,如实记 issue
const nonReady = await db
.select({
contentRevisionId: mediaUsages.contentRevisionId,
fieldPath: mediaUsages.fieldPath,
publicPath: mediaAssets.publicPath,
status: mediaAssets.status,
})
.from(mediaUsages)
.innerJoin(mediaAssets, eq(mediaUsages.mediaAssetId, mediaAssets.id))
.where(
and(
eq(mediaUsages.siteId, input.siteId),
inArray(mediaUsages.contentRevisionId, publishedIds),
ne(mediaAssets.status, 'ready'),
),
)
for (const u of nonReady) {
issues.push(
`media 引用非 ready 资产(revision=${u.contentRevisionId} fieldPath=${u.fieldPath} public_path=${u.publicPath} status=${u.status})——该引用不进 content.json media 块`,
)
}
}
const contentPayload = published.map((row) => {
const data = (row.data ?? {}) as Record<string, unknown>
let entryData = data
if (typeof data.bodyHtml === 'string') {
const r = rewriteBodyHtml(data.bodyHtml, mediaIndex)
urlRewrite.rewritten += r.stats.rewritten
urlRewrite.unresolved += r.stats.unresolved
entryData = { ...data, bodyHtml: r.html }
}
// slug 以修订 data.slug(发布时点冻结值)为准,缺席回退 entry_locales.slug(current 跟随面)
const slug = typeof data.slug === 'string' && data.slug !== '' ? data.slug : row.localeSlug
const media = usageMap.get(row.revisionId!)
const entry: Record<string, unknown> = {
contentId: row.contentId,
typeKey: row.typeKey,
locale: row.locale,
slug,
data: entryData,
revisionId: row.revisionId,
checksum: row.checksum,
}
if (media !== undefined && media.size > 0) {
entry.media = Object.fromEntries([...media.entries()].sort(([a], [b]) => cmp(a, b)))
}
return entry
})
contentPayload.sort((a, b) =>
cmp(a.typeKey as string, b.typeKey as string)
|| cmp(a.slug as string, b.slug as string)
|| cmp(a.locale as string, b.locale as string)
|| cmp(a.contentId as string, b.contentId as string),
)
// taxonomies.json:taxonomies × terms 全量 + content_terms 关联(auto: 自建行 source_id 原样)
const taxonomyRows = await db
.select({
id: taxonomies.id,
key: taxonomies.key,
objectType: taxonomies.objectType,
publicBasePath: taxonomies.publicBasePath,
})
.from(taxonomies)
.where(eq(taxonomies.siteId, input.siteId))
const termRows = await db
.select({
id: terms.id,
taxonomyId: terms.taxonomyId,
sourceId: terms.sourceId,
slug: terms.slug,
name: terms.name,
parentTermId: terms.parentTermId,
data: terms.data,
})
.from(terms)
.where(eq(terms.siteId, input.siteId))
const relationRows = await db
.select({ termId: contentTerms.termId, contentEntryId: contentTerms.contentEntryId })
.from(contentTerms)
.where(eq(contentTerms.siteId, input.siteId))
const entriesByTerm = new Map<string, string[]>()
for (const rel of relationRows) {
const list = entriesByTerm.get(rel.termId) ?? []
list.push(rel.contentEntryId)
entriesByTerm.set(rel.termId, list)
}
const termsByTaxonomy = new Map<string, typeof termRows>()
for (const t of termRows) {
const list = termsByTaxonomy.get(t.taxonomyId) ?? []
list.push(t)
termsByTaxonomy.set(t.taxonomyId, list)
}
const taxonomiesPayload = taxonomyRows.map((tax) => {
const group = (termsByTaxonomy.get(tax.id) ?? []).map((t) => ({
id: t.id,
sourceId: t.sourceId,
slug: t.slug,
name: t.name,
parentTermId: t.parentTermId,
data: (t.data ?? {}) as Record<string, unknown>,
contentEntryIds: (entriesByTerm.get(t.id) ?? []).sort(cmp),
}))
group.sort((a, b) => cmp(a.sourceId, b.sourceId) || cmp(a.slug, b.slug) || cmp(a.id, b.id))
return { key: tax.key, objectType: tax.objectType, publicBasePath: tax.publicBasePath, terms: group }
})
taxonomiesPayload.sort((a, b) => cmp(a.key, b.key) || cmp(a.objectType, b.objectType))
// routes.json:可服务路由形态(html/asset/redirect/gone);dynamic/unresolved 不进产物
const routeRows = await db
.select({
host: routeRegistry.host,
routePath: routeRegistry.routePath,
routeKind: routeRegistry.routeKind,
queryPolicy: routeRegistry.queryPolicy,
queryKey: routeRegistry.queryKey,
targetStatus: routeRegistry.targetStatus,
targetContentId: routeRegistry.targetContentId,
targetFile: routeRegistry.targetFile,
redirectStatus: routeRegistry.redirectStatus,
redirectLocation: routeRegistry.redirectLocation,
indexable: routeRegistry.indexable,
priority: routeRegistry.priority,
verificationStatus: routeRegistry.verificationStatus,
})
.from(routeRegistry)
.where(eq(routeRegistry.siteId, input.siteId))
const SERVICABLE = new Set(['html', 'asset', 'redirect', 'gone'])
const unservicable = routeRows.filter((r) => !SERVICABLE.has(r.routeKind))
if (unservicable.length > 0) {
issues.push(
`路由表含 ${unservicable.length} 条不可服务形态(dynamic/unresolved)——不进 routes.json(处置是 SP3+ 查询路由域)`,
)
}
const routesPayload = routeRows.filter((r) => SERVICABLE.has(r.routeKind))
routesPayload.sort(
(a, b) => cmp(a.routePath, b.routePath) || cmpNull(a.queryKey, b.queryKey) || cmp(a.host, b.host),
)
// assets.json:ready 资产 + 实体副本路径解析(content-addressed 首选 → 仓库存储根 → incoming 兜底)
const bundleRows = await db
.select({ metadata: sourceSnapshots.metadata })
.from(sourceSnapshots)
.where(and(eq(sourceSnapshots.siteId, input.siteId), eq(sourceSnapshots.kind, 'media_bundle')))
.orderBy(desc(sourceSnapshots.capturedAt), desc(sourceSnapshots.createdAt))
.limit(1)
const bundleRootRaw = bundleRows[0]?.metadata as { storageRoot?: unknown } | undefined
const bundleRoot = typeof bundleRootRaw?.storageRoot === 'string' ? bundleRootRaw.storageRoot : null
const assetsPayload = assetRows.map((a) => {
const candidates: string[] = []
if (bundleRoot !== null && bundleRoot !== '') candidates.push(join(bundleRoot, a.storagePath))
const repoCopy = join(REPO_STORAGE, a.storagePath)
candidates.push(repoCopy)
// incoming 兜底:清单文件名(真实数据扁平 GUID 命名,basename 唯一)
candidates.push(join(REPO_STORAGE, 'incoming', basename(a.storagePath)))
let copyPath = candidates[0] ?? repoCopy
for (const c of candidates) {
if (existsSync(c)) {
copyPath = c
break
}
}
if (!existsSync(copyPath)) {
issues.push(
`媒体副本缺席(public_path=${a.publicPath}):候选 ${candidates.join(' ; ')} 均不存在——assets.json 记录首选路径,复制阶段(Task 5 管线)将失败`,
)
}
return {
publicPath: a.publicPath,
storagePath: a.storagePath,
copyPath,
sha256: a.sha256,
detectedMimeType: a.detectedMimeType,
byteSize: a.byteSize,
width: a.width,
height: a.height,
altText: a.altText,
}
})
// 三哈希(各文件 canonicalJson 的 sha256——字节可复算)+ render-manifest 汇聚
const sourceHashes = {
content: hashOf(contentPayload),
route: hashOf(routesPayload),
asset: hashOf(assetsPayload),
}
const counts = {
contents: contentPayload.length,
taxonomies: taxonomiesPayload.length,
routes: routesPayload.length,
assets: assetsPayload.length,
}
const manifestPayload = {
renderSnapshotVersion: RENDER_SNAPSHOT_VERSION,
versions: {
adapter: ADAPTER_VERSION,
mapping: MAPPING_VERSION,
sanitizer: SANITIZER_VERSION,
parser: PARSER_VERSION,
},
counts,
sourceHashes,
urlRewrite,
generatedAt: input.generatedAt,
}
const file = (payload: unknown): string => `${canonicalJson(payload)}\n`
const files: RenderSnapshotFileContents = {
'site.json': file(config.data),
'content.json': file(contentPayload),
'taxonomies.json': file(taxonomiesPayload),
'routes.json': file(routesPayload),
'assets.json': file(assetsPayload),
'render-manifest.json': file(manifestPayload),
}
return { files, sourceHashes, counts, urlRewrite, issues, siteConfigRevisionId: config.id }
}