diff --git a/apps/api/src/modules/build/build.service.ts b/apps/api/src/modules/build/build.service.ts index 75f7607..c7afcbd 100644 --- a/apps/api/src/modules/build/build.service.ts +++ b/apps/api/src/modules/build/build.service.ts @@ -1,10 +1,11 @@ -// build_jobs 生命周期服务（Plan 4 Task 2，V2 spec §9、ADR-010/ADR-011）： +// build_jobs 生命周期服务（Plan 4 Task 2/5，V2 spec §9、ADR-010/ADR-011）： // 解析站点 → publish 状态检查（快照只读 published——空发布集无物化意义）→ 幂等建 // job（queued）→ snapshotting（render-snapshot.ts 纯物化六文件 + build_input_snapshots -// 行，repeatable read 单事务一致性读取）→ validating（**空门禁**——Task 6 填真实 -// 门禁；Task 5 接真实构建时**新建 job 不改行**：build_input_snapshots 有不可变触发器） -// → succeeded；报告落 /jobs//reports/build.json（temp+rename -// 原子写、**先于终态落行**——import 同款纪律）。 +// 行，repeatable read 单事务一致性读取）→ **building（Task 5：pipeline.ts 真实构建 +// ——spawn astro → 媒体复制 → sitemap/robots → route-manifest → build_artifacts +// dist/manifest 登记）**→ validating（**空门禁**——Task 6 填真实门禁）→ succeeded； +// 报告落 /jobs//reports/build.json + build_artifacts report 行 +// （temp+rename 原子写、**先于终态落行**——import 同款纪律）。 // // 幂等：build_jobs.idempotency_key UNIQUE——重复提交同 key：**只有 succeeded 复用** // （产物完整）；failed/cancelled/stale 终态死行与 queued/snapshotting/building/ @@ -33,9 +34,17 @@ import { fileURLToPath } from 'node:url' import { desc, eq } from 'drizzle-orm' import type { NodePgDatabase } from 'drizzle-orm/node-postgres' import { canonicalJson } from '../../common/canonical.js' -import { buildInputSnapshots, buildJobs, entryLocales, siteConfigRevisions, sites } from '../../db/schema.js' +import { + buildArtifacts, + buildInputSnapshots, + buildJobs, + entryLocales, + siteConfigRevisions, + sites, +} from '../../db/schema.js' import type * as schema from '../../db/schema.js' import { isUniqueViolation } from '../source/register.service.js' +import { PipelineError, runBuildPipeline, themeDistDir, type BuildPipelineResult } from './pipeline.js' import { materializeRenderSnapshot, type RewriteStats } from './render-snapshot.js' /** 本文件位于 /apps/api/src/modules/build/ → 上五级是仓库根 */ @@ -51,6 +60,7 @@ export type BuildErrorCode = | 'NO_SITE_CONFIG' | 'SNAPSHOT_FAILED' | 'REPORT_WRITE_FAILED' + | 'ARTIFACT_REGISTER_FAILED' /** 构建失败（可预期错误）：CLI 以 code 区分呈现；不裹程序 bug（那些原样上抛） */ export class BuildError extends Error { @@ -106,6 +116,8 @@ export interface CreateBuildJobResult { counts?: { contents: number; taxonomies: number; routes: number; assets: number } urlRewrite?: RewriteStats issues?: string[] + /** Task 5 产物面（dist/route-manifest 路径与哈希 + 计数）——复用路径无值（产物在原 job 报告） */ + artifacts?: BuildPipelineResult durationMs: number } @@ -135,6 +147,16 @@ export interface BuildReport { } files: Array<{ name: string; path: string; sha256: string; byteSize: number }> urlRewrite: RewriteStats + /** Task 5 产物面（与 build_artifacts 行互证） */ + artifacts: { + distPath: string + distSha256: string + distFileCount: number + routeManifestPath: string + routeManifestSha256: string + sitemapEntries: number + mediaCopied: number + } issues: string[] durationMs: number } @@ -227,9 +249,11 @@ const RETRY_KEY_BOUND = 100 * 解析（site → published 检查 → config 检查）→ 幂等建行（queued；**只有 succeeded * 可复用**——终态死行/崩溃残留按阶段 3 注记的策略自动恢复）→ snapshotting * （repeatable read 单事务：物化 + 六文件落盘 + build_input_snapshots 行）→ - * validating（空——Task 6）→ 报告落盘 → succeeded。 - * dist 产物与 build_artifacts 登记是 Task 5 职责（本任务 dist 尚空）。 - * 快照后失败 = job 行 failed 终态 + 半途 input/ 目录清场 + 原样上抛。 + * building（pipeline.ts：spawn astro → 媒体复制 → sitemap/robots → route-manifest + * → build_artifacts dist/manifest 两行）→ validating（空——Task 6）→ 报告落盘 + + * build_artifacts report 行 → succeeded。 + * 任一阶段失败 = job 行 failed 终态（PipelineError.code 进 error_code、stderr 标记进 + * error_summary）+ 半途产物清场（input/ 与 dist/——失败无产物语义）+ 原样上抛。 */ export async function createBuildJob(db: BuildDb, input: CreateBuildJobInput): Promise { const startedAt = performance.now() @@ -387,11 +411,23 @@ export async function createBuildJob(db: BuildDb, input: CreateBuildJobInput): P { isolationLevel: 'repeatable read' }, ) - // 阶段 6：validating（**空门禁**——Task 6 填真实门禁；状态机占位完整） + // 阶段 6：building——真实构建（Task 5 pipeline.ts：spawn/媒体/sitemap/manifest/ + // build_artifacts dist+manifest 登记）。失败以 PipelineError 上抛（code 进 error_code， + // stderr 标记进 error_summary——收口见下方 catch）。 + await db.update(buildJobs).set({ status: 'building' }).where(eq(buildJobs.id, jobId)) + input.onStatus?.('building', jobId) + const built = await runBuildPipeline(db, { + jobId: jobId!, + siteId: site.id, + inputDir, + artifactsRoot, + }) + + // 阶段 7：validating（**空门禁**——Task 6 填真实门禁；状态机占位完整） await db.update(buildJobs).set({ status: 'validating' }).where(eq(buildJobs.id, jobId)) input.onStatus?.('validating', jobId) - // 阶段 7：报告先行落盘（temp+rename 原子替换），终态落行在后——行内即文件在 + // 阶段 8：报告先行落盘（temp+rename 原子替换），终态落行在后——行内即文件在 const report: BuildReport = { jobId, generatedAt: buildTimestamp.toISOString(), @@ -425,12 +461,22 @@ export async function createBuildJob(db: BuildDb, input: CreateBuildJobInput): P byteSize: Buffer.byteLength(content, 'utf8'), })), urlRewrite: mat.urlRewrite, + artifacts: { + distPath: built.distPath, + distSha256: built.distSha256, + distFileCount: built.distFileCount, + routeManifestPath: built.routeManifestPath, + routeManifestSha256: built.routeManifestSha256, + sitemapEntries: built.sitemapEntries, + mediaCopied: built.mediaCopied, + }, issues: mat.issues, durationMs: performance.now() - startedAt, } + const reportText = `${JSON.stringify(report, null, 2)}\n` try { await mkdir(reportDir, { recursive: true }) - await writeAtomically(reportPath, `${JSON.stringify(report, null, 2)}\n`) + await writeAtomically(reportPath, reportText) } catch (err) { throw new BuildError( 'REPORT_WRITE_FAILED', @@ -438,7 +484,23 @@ export async function createBuildJob(db: BuildDb, input: CreateBuildJobInput): P ) } - // 阶段 8：succeeded 终态 + // 阶段 9：report 行登记（build_artifacts 第三行——报告文件已落盘，行内即文件在） + try { + await db.insert(buildArtifacts).values({ + buildJobId: jobId, + kind: 'report', + storagePath: reportPath, + sha256: createHash('sha256').update(reportText, 'utf8').digest('hex'), + byteSize: Buffer.byteLength(reportText, 'utf8'), + }) + } catch (err) { + throw new BuildError( + 'ARTIFACT_REGISTER_FAILED', + `report 产物登记失败（job=${jobId}）：${err instanceof Error ? err.message : String(err)}`, + ) + } + + // 阶段 10：succeeded 终态 await db.update(buildJobs).set({ status: 'succeeded', finishedAt: new Date() }).where(eq(buildJobs.id, jobId)) input.onStatus?.('succeeded', jobId) @@ -454,23 +516,29 @@ export async function createBuildJob(db: BuildDb, input: CreateBuildJobInput): P counts: mat.counts, urlRewrite: mat.urlRewrite, issues: mat.issues, + artifacts: built, durationMs: performance.now() - startedAt, } } catch (err) { - const code = err instanceof BuildError ? err.code : 'SNAPSHOT_FAILED' + // 阶段敏感的错误码映射：BuildError 保自身码（解析/报告/登记）；PipelineError 带 + // 管线阶段码（spawn/媒体/manifest——stderr 标记在 message 内一并入 error_summary）； + // 其余（快照事务/物化）沿用 SNAPSHOT_FAILED 总括码。 + const code = err instanceof BuildError ? err.code : err instanceof PipelineError ? err.code : 'SNAPSHOT_FAILED' const summary = err instanceof Error ? `${err.name}: ${err.message}` : String(err) await db .update(buildJobs) .set({ status: 'failed', finishedAt: new Date(), errorCode: code, errorSummary: `[build] ${summary}` }) .where(eq(buildJobs.id, jobId)) - // 失败清场（审查 Fix 3）：半途落盘的 input/ 目录移除——failed 终态无产物语义， - // 不留"行说 failed 而目录半存"的歧义现场。快照行若已插入则由不可变触发器保留为 - // 审计记录（manifest_path 悬空可接受：job 已 failed，无下游消费方）。清场失败 - // 不掩盖原始错误（残留目录由操作者处置）。 - try { - await rm(inputDir, { recursive: true, force: true }) - } catch { - // best-effort：保留原错误上抛 + // 失败清场（审查 Fix 3 + Task 5 扩面）：半途落盘的 input/ 与 dist/（job 级产物目录 + // + 主题包暂存 dist）移除——failed 终态无产物语义，不留"行说 failed 而目录半存" + // 的歧义现场。快照行若已插入则由不可变触发器保留为审计记录（manifest_path 悬空 + // 可接受：job 已 failed，无下游消费方）。清场失败不掩盖原始错误（残留目录由操作者处置）。 + for (const dir of [inputDir, join(artifactsRoot, 'jobs', jobId, 'dist'), themeDistDir()]) { + try { + await rm(dir, { recursive: true, force: true }) + } catch { + // best-effort：保留原错误上抛 + } } throw err } diff --git a/apps/api/src/modules/build/pipeline.ts b/apps/api/src/modules/build/pipeline.ts new file mode 100644 index 0000000..e44c2e7 --- /dev/null +++ b/apps/api/src/modules/build/pipeline.ts @@ -0,0 +1,470 @@ +// 构建编排管线（Plan 4 Task 5，V2 spec §9.3/§12.1、快线 §8.4）： +// spawn astro 构建（SF_BUILD_INPUT 六文件输入）→ dist 定位（主题包 outDir）→ +// 媒体复制（copyPath→dist/public_path，sha256 全量校验失配即败）→ sitemap.xml/ +// robots.txt（origin 来自 site.json）→ route-manifest.json（§12.1 字段面，哈希链复用 +// render-manifest 的 route 快照哈希）→ build_artifacts 登记（dist/manifest 两行； +// report 行是 build.service 落报告后的职责）。 +// +// 本文件是"纯编排段"：不做 job 状态机推进（building→validating→succeeded 与失败收口 +// 都在 build.service.ts），失败以 PipelineError 上抛（code 区分阶段，message 带 +// stderr 摘要——Windows 下 pnpm 对子进程失败会叠 127/EPERM/libuv 噪音，判据是 +// **非零退出 + stderr 标记**，不断言精确退出码）。 +// +// dist 双位置纪律：主题包内 dist（astro outDir）是**暂存**——spawn 产物 + 媒体 + +// sitemap/robots 在此汇齐后整体拷出到 /jobs//dist（job 级 +// 权威产物，行内即文件在），暂存随即清空（成功失败都清——不留跨 job 共享可变态）。 +// 跨盘拷贝用 copy 不用 rename（artifacts 根常在另一卷，rename 撞 EXDEV）。 +// +// 无网纪律（五禁③的实现侧配合）：spawn env 显式 ASTRO_TELEMETRY_DISABLED=1—— +// @astrojs/telemetry 在构建期真调 global fetch POST（实测撞无网 stub）；离线构建 +// 纪律下遥测必须关。NODE_OPTIONS 原样透传（spread 继承 + 本管线不自加标志的合并 +// 语义）：测试可注入 `--import ` 预载钩子证明构建进程树不联网。 +import { spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { copyFile, mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { NodePgDatabase } from 'drizzle-orm/node-postgres' +import { + ROUTE_MANIFEST_VERSION, + type AssetEntry, + type RouteEntry, + type RouteManifest, + type RouteManifestRoute, +} from '@sf/contracts' +import { buildArtifacts } from '../../db/schema.js' +import type * as schema from '../../db/schema.js' + +/** 本文件位于 /apps/api/src/modules/build/ → 上五级是仓库根 */ +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..') + +/** Plan 4 唯一主题：包全名 filter（Task 3 备忘——勿用目录名短 filter） */ +const THEME_PACKAGE = '@sf/theme-pilot-minimal' +const THEME_DIR = join(REPO_ROOT, 'themes', 'pilot-minimal') + +/** 主题包内暂存 dist（astro outDir='./dist'）——产物汇齐后拷出、随即清空 */ +export function themeDistDir(): string { + return join(THEME_DIR, 'dist') +} + +export type PipelineDb = NodePgDatabase + +export type PipelineErrorCode = 'INPUT_INVALID' | 'THEME_BUILD_FAILED' | 'MEDIA_COPY_FAILED' | 'MANIFEST_WRITE_FAILED' + +/** 管线阶段失败（可预期错误）：code 区分阶段；stderr 摘要进 message（落 error_summary） */ +export class PipelineError extends Error { + readonly code: PipelineErrorCode + constructor(code: PipelineErrorCode, message: string) { + super(message) + this.name = 'PipelineError' + this.code = code + } +} + +export interface RunBuildPipelineInput { + jobId: string + siteId: string + /** 六文件输入目录（SF_BUILD_INPUT——build_input_snapshots 物化产物） */ + inputDir: string + /** 产物根：/jobs//{input,dist,reports,route-manifest.json} */ + artifactsRoot: string +} + +export interface BuildPipelineResult { + distPath: string + /** 目录哈希（公式见 hashDirectory——递归文件清单 sha256） */ + distSha256: string + distByteSize: number + distFileCount: number + routeManifestPath: string + routeManifestSha256: string + routeManifestByteSize: number + sitemapEntries: number + mediaCopied: number +} + +// ---- 输入读取 ---- + +async function readJsonInput(inputDir: string, file: string): Promise { + let text: string + try { + text = await readFile(join(inputDir, file), 'utf8') + } catch (err) { + throw new PipelineError( + 'INPUT_INVALID', + `管线输入缺失或不可读：${join(inputDir, file)}（${err instanceof Error ? err.message : String(err)}）`, + ) + } + try { + return JSON.parse(text) + } catch (err) { + throw new PipelineError( + 'INPUT_INVALID', + `管线输入不是合法 JSON：${file}（${err instanceof Error ? err.message : String(err)}）`, + ) + } +} + +// ---- spawn ---- + +/** spawn 超时上界（含 pnpm 解析与 astro 冷启——Windows 实测全链 ~10s，宽限到分钟级） */ +const SPAWN_TIMEOUT_MS = 300_000 +/** stdout/stderr 各自的累积上界（超出丢弃并标记截断——防失控输出撑爆内存） */ +const STREAM_CAP = 2 * 1024 * 1024 + +function spawnEnv(inputDir: string): NodeJS.ProcessEnv { + // NODE_OPTIONS 合并语义：spread 继承调用方（含测试注入的 --import 预载），本管线 + // 不追加不清洗；自己的覆盖只有 SF_BUILD_INPUT 与遥测关闭（见模块头无网纪律注记）。 + return { + ...process.env, + SF_BUILD_INPUT: inputDir, + ASTRO_TELEMETRY_DISABLED: '1', + } +} + +/** astro 构建 spawn：cwd=仓库根（pnpm workspace 解析），退出非零即败（stderr 摘要入 message） */ +function spawnAstroBuild(inputDir: string): Promise { + return new Promise((resolvePromise, rejectPromise) => { + let settled = false + let stderr = '' + let stderrTruncated = false + const child = spawn('pnpm', ['--filter', THEME_PACKAGE, 'exec', 'astro', 'build'], { + cwd: REPO_ROOT, + shell: true, // Windows：pnpm 是 pnpm.cmd，须经 shell 解析（参数均为无空格字面量） + env: spawnEnv(inputDir), + stdio: ['ignore', 'pipe', 'pipe'], + }) + const timer = setTimeout(() => { + child.kill('SIGKILL') + if (!settled) { + settled = true + rejectPromise( + new PipelineError('THEME_BUILD_FAILED', `astro 构建超时（>${SPAWN_TIMEOUT_MS}ms）——SIGKILL 终结`), + ) + } + }, SPAWN_TIMEOUT_MS) + child.stderr?.on('data', (chunk: Buffer) => { + if (stderr.length >= STREAM_CAP) { + stderrTruncated = true + return + } + stderr += chunk.toString('utf8') + if (stderr.length > STREAM_CAP) { + stderr = stderr.slice(0, STREAM_CAP) + stderrTruncated = true + } + }) + // stdout 只排水不累积（页面清单对诊断无增量价值；异常路径的信号在 stderr） + child.stdout?.on('data', () => {}) + child.on('error', (err) => { + clearTimeout(timer) + if (!settled) { + settled = true + rejectPromise(new PipelineError('THEME_BUILD_FAILED', `astro 构建 spawn 失败：${err.message}`)) + } + }) + child.on('close', (code, signal) => { + clearTimeout(timer) + if (settled) return + settled = true + if (code === 0) { + resolvePromise() + return + } + // 判据 = 非零退出 + stderr 标记（Windows pnpm 会叠 127/libuv 噪音——勿匹配精确码）。 + // 摘要取"头 500 + 尾 4000"：astro 的错误块（消息+栈）靠 stderr 尾部，但纯尾截会把 + // 标记行拦腰截断（消息在错误块头部）——头部保上下文，尾部保完整错误块。 + const head = stderr.length > 4500 ? `${stderr.slice(0, 500)}\n…（中段省略）\n` : '' + const tail = stderr.length > 4000 ? stderr.slice(-4000) : stderr + const excerpt = `${head}${tail}` + const truncNote = stderrTruncated ? '（stderr 超 2MB 已截断）' : '' + rejectPromise( + new PipelineError( + 'THEME_BUILD_FAILED', + `astro 构建失败（exit=${String(code)} signal=${String(signal)}）${truncNote}——stderr 摘要：\n${excerpt}`, + ), + ) + }) + }) +} + +// ---- dist 后处理 ---- + +/** + * 路由 → dist 相对产物路径（映射规则唯一权威——route-writer 的 html target_file 落 NULL + * 即"由构建器落地"；与主题侧 sf-literal-html-paths 折平集成对齐）： + * '/' → index.html；尾斜杠 → /index.html；.html 后缀 → 字面文件。 + */ +export function deriveTargetFile(routePath: string): string { + if (routePath === '/') return 'index.html' + const rel = routePath.replace(/^\//, '').replace(/\/+$/, '') + return rel.endsWith('.html') ? rel : `${rel}/index.html` +} + +/** temp+rename 原子写（与 build.service 同款纪律——不留半成品） */ +async function writeAtomically(path: string, content: string): Promise { + const tmpPath = `${path}.tmp` + try { + await writeFile(tmpPath, content, 'utf8') + await rename(tmpPath, path) + } catch (err) { + await rm(tmpPath, { force: true }) + throw err + } +} + +/** 媒体复制：copyPath→dist/public_path；**每条**落盘后 sha256 全量校验（非抽检）失配即败 */ +async function copyMedia(distDir: string, assets: AssetEntry[]): Promise { + for (const a of assets) { + // public_path 必须是根相对安全路径（media-writer 上游已按此登记——此处守越界写） + if (!a.publicPath.startsWith('/') || a.publicPath.split('/').includes('..')) { + throw new PipelineError( + 'MEDIA_COPY_FAILED', + `媒体 public_path 非根相对安全路径（${a.publicPath}）——拒绝越界落盘`, + ) + } + const src = resolve(REPO_ROOT, a.copyPath) + const dst = join(distDir, a.publicPath) + try { + await mkdir(dirname(dst), { recursive: true }) + await copyFile(src, dst) + } catch (err) { + throw new PipelineError( + 'MEDIA_COPY_FAILED', + `媒体复制失败（${a.publicPath} ← ${a.copyPath}）：${err instanceof Error ? err.message : String(err)}`, + ) + } + const copied = createHash('sha256').update(await readFile(dst)).digest('hex') + if (copied !== a.sha256) { + throw new PipelineError( + 'MEDIA_COPY_FAILED', + `媒体 sha256 失配（${a.publicPath}）：assets.json=${a.sha256} ≠ 落盘=${copied}——输入侧副本与清单破裂`, + ) + } + } + return assets.length +} + +/** XML 属性/文本转义（编码 URL 原样贯穿的边界：%xx 不受影响，裸 &<>" 转义） */ +function xmlEscape(s: string): string { + return s.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"') +} + +/** UTF-16 字典序（与物化端排序同语义——钉死不依赖 locale） */ +function cmp(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0 +} +function cmpNull(a: string | null, b: string | null): number { + if (a === null && b === null) return 0 + if (a === null) return -1 + if (b === null) return 1 + return cmp(a, b) +} + +function buildSitemap(origin: string, routes: RouteEntry[]): { xml: string; entries: number } { + const rows = routes + .filter((r) => r.routeKind === 'html' && r.indexable) + .sort((a, b) => cmp(a.routePath, b.routePath) || cmpNull(a.queryKey, b.queryKey)) + const items = rows.map((r) => { + const url = r.queryKey === null ? `${origin}${r.routePath}` : `${origin}${r.routePath}?${r.queryKey}` + return ` ${xmlEscape(url)}` + }) + return { + xml: `\n\n${items.join('\n')}\n\n`, + entries: rows.length, + } +} + +function buildRobots(origin: string): string { + return `User-agent: *\nAllow: /\n\nSitemap: ${origin}/sitemap.xml\n` +} + +/** route-manifest 行映射：exact_key 带查询形态；target_file 派生映射；redirect 带 location */ +function toManifestRoute(r: RouteEntry, assetsByPath: Map): RouteManifestRoute { + // redirect 行 redirectStatus 是权威（targetStatus 同值兜底——数据面两处冗余） + const rawStatus = r.routeKind === 'redirect' ? (r.redirectStatus ?? Number(r.targetStatus)) : Number(r.targetStatus) + if (!Number.isInteger(rawStatus) || rawStatus < 100 || rawStatus > 599) { + throw new PipelineError( + 'MANIFEST_WRITE_FAILED', + `路由 ${r.routePath} 的 HTTP 状态非法（${String(rawStatus)}）——target_status/redirect_status 数据面破裂`, + ) + } + const row: RouteManifestRoute = { + exact_key: r.queryKey === null ? r.routePath : `${r.routePath}?${r.queryKey}`, + query_policy: r.queryPolicy, + query_key: r.queryKey, + kind: r.routeKind, + status: rawStatus, + indexable: r.indexable, + } + if (r.routeKind === 'html') { + row.target_file = deriveTargetFile(r.routePath) + row.content_type = 'text/html' + } else if (r.routeKind === 'asset') { + row.target_file = r.targetFile ?? r.routePath + const asset = assetsByPath.get(r.routePath) + if (asset !== undefined) row.content_type = asset.detectedMimeType + } else if (r.routeKind === 'redirect') { + if (r.redirectLocation !== null) row.location = r.redirectLocation + } + return row +} + +/** + * 目录哈希（build_artifacts dist 行的 sha256 口径）：递归文件清单（posix 相对路径 + * 字典序）每行 `rel\truntime sha256`，清单文本（含尾换行）的 sha256——文件集合与 + * 内容任一变化即变，操作者可离线复算互证。byteSize = 递归文件字节和。 + */ +async function hashDirectory(dir: string): Promise<{ sha256: string; byteSize: number; fileCount: number }> { + const rels: string[] = [] + const walk = async (d: string): Promise => { + for (const e of await readdir(d, { withFileTypes: true })) { + const full = join(d, e.name) + if (e.isDirectory()) await walk(full) + else rels.push(relative(dir, full).replaceAll('\\', '/')) + } + } + await walk(dir) + rels.sort() + let byteSize = 0 + const lines: string[] = [] + for (const rel of rels) { + const buf = await readFile(join(dir, rel)) + byteSize += buf.byteLength + lines.push(`${rel}\t${createHash('sha256').update(buf).digest('hex')}`) + } + return { sha256: createHash('sha256').update(`${lines.join('\n')}\n`).digest('hex'), byteSize, fileCount: rels.length } +} + +async function copyTree(src: string, dst: string): Promise { + await mkdir(dst, { recursive: true }) + for (const e of await readdir(src, { withFileTypes: true })) { + if (e.isDirectory()) await copyTree(join(src, e.name), join(dst, e.name)) + else await copyFile(join(src, e.name), join(dst, e.name)) + } +} + +/** + * 构建管线主体（编排顺序见模块头）。前置不变量：inputDir 六文件齐备（service 物化 + * 已保证；直调方破坏不变量时以 INPUT_INVALID 拒绝，不产残缺产物）。 + * build_artifacts 登记 dist/manifest 两行；report 行由 build.service 落报告后补登。 + */ +export async function runBuildPipeline(db: PipelineDb, input: RunBuildPipelineInput): Promise { + const routes = (await readJsonInput(input.inputDir, 'routes.json')) as RouteEntry[] + const assets = (await readJsonInput(input.inputDir, 'assets.json')) as AssetEntry[] + const siteRaw = (await readJsonInput(input.inputDir, 'site.json')) as { origin?: unknown } + const inputManifest = (await readJsonInput(input.inputDir, 'render-manifest.json')) as { + sourceHashes?: { route?: unknown } + } + if (!Array.isArray(routes) || routes.length === 0) { + throw new PipelineError('INPUT_INVALID', 'routes.json 为空或非数组——正常发布站点至少含首页路由') + } + if (!Array.isArray(assets)) { + throw new PipelineError('INPUT_INVALID', 'assets.json 非数组——物化面破裂') + } + if (typeof siteRaw.origin !== 'string' || siteRaw.origin === '') { + throw new PipelineError('INPUT_INVALID', 'site.json origin 缺席/为空——sitemap 与 manifest 的 origin 无来源') + } + const origin = siteRaw.origin + const routeHash = + typeof inputManifest.sourceHashes?.route === 'string' ? inputManifest.sourceHashes.route : '' + if (routeHash === '') { + throw new PipelineError('INPUT_INVALID', 'render-manifest.json sourceHashes.route 缺席——哈希链断链') + } + + // ① spawn（主题包 dist 暂存先行清空——跨 build 残留归零，astro 自身清 outDir 之外双保险） + const scratchDist = themeDistDir() + await rm(scratchDist, { recursive: true, force: true }) + await spawnAstroBuild(input.inputDir) + + // ② dist 定位与完整性：每条 html 行 target_file（派生映射）在暂存 dist 存在且是文件 + // （主题侧折平集成已对齐映射——此处失配即主题/管线映射破裂，禁静默放行） + const scratchStat = await stat(scratchDist).catch(() => null) + if (scratchStat === null || !scratchStat.isDirectory()) { + throw new PipelineError('THEME_BUILD_FAILED', `主题构建产物目录缺席：${scratchDist}`) + } + for (const r of routes) { + if (r.routeKind !== 'html') continue + const rel = deriveTargetFile(r.routePath) + const st = await stat(join(scratchDist, rel)).catch(() => null) + if (st === null || !st.isFile()) { + throw new PipelineError( + 'THEME_BUILD_FAILED', + `dist 产物缺失（${r.routePath} → ${rel}）——主题产物与路由表映射破裂（禁静默放行）`, + ) + } + } + + // ③ 媒体复制（sha256 全量校验）④ sitemap/robots（写入暂存 dist——它们是站点服务面文件） + const mediaCopied = await copyMedia(scratchDist, assets) + const sitemap = buildSitemap(origin, routes) + await writeAtomically(join(scratchDist, 'sitemap.xml'), sitemap.xml) + await writeAtomically(join(scratchDist, 'robots.txt'), buildRobots(origin)) + + // ⑤ 暂存拷出到 job 级权威产物目录，暂存清空（见模块头双位置纪律） + const distPath = join(input.artifactsRoot, 'jobs', input.jobId, 'dist') + await rm(distPath, { recursive: true, force: true }) + await copyTree(scratchDist, distPath) + await rm(scratchDist, { recursive: true, force: true }) + + // ⑥ route-manifest（§12.1：物化哈希链复用 + 行序 = routes.json 物化序） + const assetsByPath = new Map(assets.map((a) => [a.publicPath, a])) + const manifest: RouteManifest = { + manifest_version: ROUTE_MANIFEST_VERSION, + site_id: input.siteId, + build_job_id: input.jobId, + origin, + generated_from_route_snapshot_hash: routeHash, + routes: routes.map((r) => toManifestRoute(r, assetsByPath)), + } + const manifestText = `${JSON.stringify(manifest, null, 2)}\n` + const manifestPath = join(input.artifactsRoot, 'jobs', input.jobId, 'route-manifest.json') + try { + await mkdir(dirname(manifestPath), { recursive: true }) + await writeAtomically(manifestPath, manifestText) + } catch (err) { + throw new PipelineError( + 'MANIFEST_WRITE_FAILED', + `route-manifest 落盘失败（${manifestPath}）：${err instanceof Error ? err.message : String(err)}`, + ) + } + + // ⑦ build_artifacts 登记（dist/manifest 两行；UNIQUE(build_job_id,kind,storage_path)） + const dirHash = await hashDirectory(distPath) + const manifestSha256 = createHash('sha256').update(manifestText, 'utf8').digest('hex') + try { + await db.insert(buildArtifacts).values([ + { + buildJobId: input.jobId, + kind: 'dist', + storagePath: distPath, + sha256: dirHash.sha256, + byteSize: dirHash.byteSize, + }, + { + buildJobId: input.jobId, + kind: 'manifest', + storagePath: manifestPath, + sha256: manifestSha256, + byteSize: Buffer.byteLength(manifestText, 'utf8'), + }, + ]) + } catch (err) { + throw new PipelineError( + 'MANIFEST_WRITE_FAILED', + `build_artifacts 登记失败（job=${input.jobId}）：${err instanceof Error ? err.message : String(err)}`, + ) + } + + return { + distPath, + distSha256: dirHash.sha256, + distByteSize: dirHash.byteSize, + distFileCount: dirHash.fileCount, + routeManifestPath: manifestPath, + routeManifestSha256: manifestSha256, + routeManifestByteSize: Buffer.byteLength(manifestText, 'utf8'), + sitemapEntries: sitemap.entries, + mediaCopied, + } +} diff --git a/scripts/sf.ts b/scripts/sf.ts index 8ee9b85..0d44937 100644 --- a/scripts/sf.ts +++ b/scripts/sf.ts @@ -16,6 +16,7 @@ import { fileURLToPath } from 'node:url' import { Command } from 'commander' import { renderReportHtml, runWxrAdapterReport, type WxrAdapterReport } from '@sf/adapter-wp' import { getDb } from '../apps/api/src/db/client.js' +import { PipelineError } from '../apps/api/src/modules/build/pipeline.js' import { BuildError, createBuildJob } from '../apps/api/src/modules/build/build.service.js' import { ImportError, @@ -301,7 +302,7 @@ program program .command('build:create') - .description('冻结 Render Snapshot：物化六文件（site/content/taxonomies/routes/assets/render-manifest）+ build_input_snapshots 行；dist 产物是 Task 5') + .description('冻结 Render Snapshot 并构建：物化六文件 + build_input_snapshots 行 → astro 构建（媒体落盘/sitemap/robots/route-manifest）→ dist 产物 + build_artifacts 登记') .requiredOption('--site ', '站点 key') .option('--key ', '幂等键（缺省 = site+published 集合哈希+主题版本 自动派生；重复提交同 key 复用既有 job 不重跑）') .option('--artifacts ', '产物根目录（缺省 repoRoot/artifacts）') @@ -335,6 +336,12 @@ program ` hashes: content=${result.sourceHashes.content.slice(0, 12)}… route=${result.sourceHashes.route.slice(0, 12)}… asset=${result.sourceHashes.asset.slice(0, 12)}…`, ) console.log(` report: ${result.reportPath}`) + if (result.artifacts !== undefined) { + console.log( + ` dist: ${result.artifacts.distPath}（${result.artifacts.distFileCount} 文件 sha256=${result.artifacts.distSha256.slice(0, 12)}…）`, + ) + console.log(` route-manifest: ${result.artifacts.routeManifestPath}`) + } console.log(` total ${result.durationMs.toFixed(1)}ms`) if (result.status === 'failed') process.exitCode = 1 } finally { @@ -343,7 +350,7 @@ program }) // parseAsync：action 内抛错走此 catch（commander 默认会 process.exit(1) 并打印堆栈， -// 这里收敛为单行错误信息 + exitCode——SourceRegisterError/ImportError/PublishError/BuildError 打 code，其余原样呈现） +// 这里收敛为单行错误信息 + exitCode——SourceRegisterError/ImportError/PublishError/BuildError/PipelineError 打 code，其余原样呈现） program.parseAsync(process.argv).catch((err: unknown) => { if (err instanceof SourceRegisterError) { console.error(`[sf] ${err.code}: ${err.message}`) @@ -356,6 +363,9 @@ program.parseAsync(process.argv).catch((err: unknown) => { console.error(`[sf] ${err.code}: ${err.message}`) } else if (err instanceof BuildError) { console.error(`[sf] ${err.code}: ${err.message}`) + } else if (err instanceof PipelineError) { + // 构建管线阶段失败（spawn/媒体/manifest）——code + stderr 摘要（job 行已有 error 面） + console.error(`[sf] ${err.code}: ${err.message}`) } else { console.error(`[sf] ${err instanceof Error ? err.message : String(err)}`) } diff --git a/tests/integration/build-pipeline.test.ts b/tests/integration/build-pipeline.test.ts new file mode 100644 index 0000000..1ed27cd --- /dev/null +++ b/tests/integration/build-pipeline.test.ts @@ -0,0 +1,561 @@ +// Plan 4 Task 5 失败测试（先于实现）：构建管线（astro spawn → 媒体落盘 → sitemap/robots +// → route-manifest → build_artifacts 登记）。覆盖契约（对照计划 Task 5）： +// 1. 全链（mini 夹具 DB 物化输入）：状态机 queued→snapshotting→**building**→validating +// →succeeded；dist 按映射规则落盘（manifest target_file 权威：每 html 行存在且 +// isFile）；媒体 copyPath→dist/public_path（sha256 全量校验）；sitemap.xml（origin + +// indexable html 行、条目数=indexable html 数）；robots.txt；route-manifest §12.1 +// 字段面（含 redirect 行与 ?attachment_id= 查询形态）；build_artifacts 三行 +// （dist=目录哈希/manifest/report）+ UNIQUE(build_job_id,kind,storage_path)； +// 2. 主题夹具直跑管线（Task 3 render-input 作 SF_BUILD_INPUT）：编码 URL sitemap 原样 +// （%e5… 字节不变、无解码孪生）、.html 字面文件、3 条媒体全量 sha256、redirect 行； +// 3. 五禁断言：① loader 单一入口（主题源码数据访问面只在 load-render-data） +// ② DATABASE_URL 置空构建成功 ③ 无网证明（NODE_OPTIONS 注入 stub：global fetch / +// net.connect 抛错——构建仍成功）④ 无 seed（fixtures 引用只在 test/）⑤ 不读源快照 +// （storage/snapshots 零命中）； +// 4. 失败路径：SF_BUILD_INPUT 六文件面破裂（抽走 content.json）→ spawn 非零退出 + +// stderr 标记 → job failed（error_summary 含标记）+ dist 残留清场。 +// Windows 判据纪律（Task 3 备忘）：astro 失败 = 非零退出 + stderr 标记，不断言精确退出码。 +import { createHash } from 'node:crypto' +import { existsSync, rmSync, statSync } from 'node:fs' +import { copyFile, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, relative, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { describe, expect, it } from 'vitest' +import { isRouteManifest, ROUTE_MANIFEST_VERSION } from '@sf/contracts' +import { canonicalJson } from '../../apps/api/src/common/canonical.js' +import { runBuildPipeline, themeDistDir } from '../../apps/api/src/modules/build/pipeline.js' +import { createBuildJob } from '../../apps/api/src/modules/build/build.service.js' +import { createImportJob } from '../../apps/api/src/modules/import/import.service.js' +import { publishSiteContent } from '../../apps/api/src/modules/publish/publish.service.js' +import { registerSource } from '../../apps/api/src/modules/source/register.service.js' +import { + createSiteFixture, + rand, + rejectionMessage, + withImmutableCleanup, + withTestDb, + type Db, +} from './helpers.js' + +/** 本文件位于 /tests/integration/ → 上两级是仓库根 */ +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const FIXTURES_ROOT = fileURLToPath(new URL('../fixtures', import.meta.url)) +const WXR_FIXTURE = resolve(FIXTURES_ROOT, 'wxr', 'mini.xml') +const THEME_SRC = join(REPO_ROOT, 'themes', 'pilot-minimal', 'src') +const THEME_FIXTURE_INPUT = join(REPO_ROOT, 'themes', 'pilot-minimal', 'test', 'fixtures', 'render-input') + +const PNG_SIG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + +/** 伪媒体体（签名 + 唯一种子载荷——内容唯一避免跨测试撞哈希） */ +function mediaBytes(seed: string): Buffer { + return Buffer.concat([PNG_SIG, Buffer.from(`media-body:${seed}`, 'latin1')]) +} +function sha256of(buf: Buffer | string): string { + return createHash('sha256').update(buf).digest('hex') +} +function plain(rows: any[]): any[] { + return rows.map((r) => ({ ...r })) +} +async function countOf(db: Db, sql: string): Promise { + const rows = (await db.execute(sql)) as any + return Number(rows[0].n) +} + +/** 路由 → dist 相对文件路径（映射规则：尾斜杠→/index.html、.html 后缀→字面文件） */ +function distRel(routePath: string): string { + if (routePath === '/') return 'index.html' + const rel = routePath.replace(/^\//, '').replace(/\/+$/, '') + return rel.endsWith('.html') ? rel : `${rel}/index.html` +} + +/** 目录哈希独立复算（与管线同公式：排序相对路径\t文件 sha256 行清单的 sha256） */ +async function dirSha256(dir: string): Promise<{ hash: string; bytes: number; files: number }> { + const rels: string[] = [] + const walk = async (d: string): Promise => { + for (const e of await readdir(d, { withFileTypes: true })) { + const full = join(d, e.name) + if (e.isDirectory()) await walk(full) + else rels.push(relative(dir, full).replaceAll('\\', '/')) + } + } + await walk(dir) + rels.sort() + let bytes = 0 + const lines: string[] = [] + for (const r of rels) { + const buf = await readFile(join(dir, r)) + bytes += buf.byteLength + lines.push(`${r}\t${sha256of(buf)}`) + } + return { hash: sha256of(`${lines.join('\n')}\n`), bytes, files: rels.length } +} + +/** 主题 src 全文件面（posix 相对路径 → 内容） */ +async function scanThemeSrc(): Promise> { + const out = new Map() + const walk = async (d: string): Promise => { + for (const e of await readdir(d, { withFileTypes: true })) { + const full = join(d, e.name) + if (e.isDirectory()) await walk(full) + else out.set(relative(THEME_SRC, full).replaceAll('\\', '/'), await readFile(full, 'utf8')) + } + } + await walk(THEME_SRC) + return out +} + +/** 本文件写入的行清场：构建域（build_artifacts 先于 build_jobs——FK）+ 全链反序 */ +async function cleanupSiteRows(db: Db, siteId: string): Promise { + await db.execute(`delete from build_artifacts where build_job_id in + (select id from build_jobs where site_id = '${siteId}')`) + await withImmutableCleanup(db, 'build_input_snapshots', 'build_input_snapshots_immutable', `site_id = '${siteId}'`) + await db.execute(`delete from build_jobs where site_id = '${siteId}'`) + await withImmutableCleanup(db, 'audit_logs', 'audit_logs_immutable', `site_id = '${siteId}'`) + await db.execute(`delete from site_config_revisions where site_id = '${siteId}'`) + await db.execute(`delete from route_registry where site_id = '${siteId}'`) + await db.execute(`delete from media_usages where site_id = '${siteId}'`) + await db.execute(`delete from media_assets where site_id = '${siteId}'`) + await db.execute(`delete from content_terms where site_id = '${siteId}'`) + await db.execute(`delete from terms where site_id = '${siteId}'`) + await db.execute(`delete from taxonomies where site_id = '${siteId}'`) + await db.execute(`delete from source_records where site_id = '${siteId}'`) + await db.execute(`delete from entry_locales where site_id = '${siteId}'`) + await withImmutableCleanup(db, 'content_revisions', 'content_revisions_immutable', `site_id = '${siteId}'`) + await db.execute(`delete from content_entries where site_id = '${siteId}'`) + await db.execute(`delete from import_jobs where site_id = '${siteId}'`) + await withImmutableCleanup(db, 'source_snapshots', 'source_snapshots_immutable', `site_id = '${siteId}'`) + await db.execute(`delete from source_connections where site_id = '${siteId}'`) + await db.execute(`delete from sites where id = '${siteId}'`) +} + +interface Work { + work: string + storageRoot: string + allowedRoots: string[] + pngA: Buffer + pngB: Buffer +} + +async function makeWork(prefix: string): Promise { + const work = await mkdtemp(join(tmpdir(), `${prefix}-`)) + const mediaDir = join(work, 'media') + await mkdir(join(mediaDir, '2026', '10'), { recursive: true }) + const pngA = mediaBytes(`pipe-a-${rand()}`) + const pngB = mediaBytes(`pipe-b-${rand()}`) + await writeFile(join(mediaDir, '2026', '10', 'fixture-a.png'), pngA) + await writeFile(join(mediaDir, '2026', '10', 'fixture-b.png'), pngB) + return { work, storageRoot: join(work, 'storage'), allowedRoots: [FIXTURES_ROOT, work], pngA, pngB } +} + +/** mini 全链（register wxr+media → import → publish）：3 条 published + 2 ready 媒体 */ +async function seedPublishedSite(db: Db, w: Work): Promise<{ key: string; siteId: string }> { + const key = `sfpipe-${rand()}` + const reg = await registerSource(db, { + siteKey: key, + wxrPath: WXR_FIXTURE, + mediaDir: join(w.work, 'media'), + allowedRoots: w.allowedRoots, + storageRoot: w.storageRoot, + reportRoot: join(w.work, 'artifacts', 'register'), + }) + const result = await createImportJob(db, { siteKey: key, reportRoot: join(w.work, 'artifacts', 'import') }) + expect(result.status).toBe('succeeded_with_issues') + const dry = await publishSiteContent(db, { siteKey: key, dryRun: true }) + const run = await publishSiteContent(db, { siteKey: key, confirmToken: dry.confirmToken! }) + expect(run.publishedCount).toBe(3) + return { key, siteId: reg.siteId } +} + +describe('构建管线全链（mini 夹具 DB 物化输入）', () => { + it('状态机含 building；dist target_file 权威落盘 + 媒体 sha256 全量 + sitemap/robots + route-manifest §12.1 + build_artifacts 三行', { timeout: 300_000 }, async () => { + await withTestDb(async (db) => { + const w = await makeWork('sf-pipe-full') + try { + const { key, siteId } = await seedPublishedSite(db, w) + try { + const statuses: string[] = [] + const result = await createBuildJob(db, { + siteKey: key, + artifactsRoot: join(w.work, 'artifacts'), + onStatus: (s) => statuses.push(s), + }) + expect(result.reused).toBe(false) + expect(result.status).toBe('succeeded') + // 状态机：Task 5 起 snapshotting 后进 building（真实 astro 构建）再 validating + expect(statuses).toStrictEqual(['queued', 'snapshotting', 'building', 'validating', 'succeeded']) + + const inputDir = result.inputDir + const distDir = join(w.work, 'artifacts', 'jobs', result.jobId, 'dist') + const routes = JSON.parse(await readFile(join(inputDir, 'routes.json'), 'utf8')) + const assets = JSON.parse(await readFile(join(inputDir, 'assets.json'), 'utf8')) + const site = JSON.parse(await readFile(join(inputDir, 'site.json'), 'utf8')) + const inputManifest = JSON.parse(await readFile(join(inputDir, 'render-manifest.json'), 'utf8')) + const htmlRoutes = routes.filter((r: any) => r.routeKind === 'html') + + // ---- dist 落盘：manifest target_file 权威（每 html 行存在且 isFile）---- + expect(distDir).toBe(result.artifacts!.distPath) + for (const r of htmlRoutes) { + const rel = distRel(r.routePath) + const f = join(distDir, rel) + // isFile 而非 exists：.html 字面路径若被折成同名目录，existsSync 仍真——形态必须钉死 + expect(statSync(f).isFile(), `${r.routePath} → ${rel}（须为文件）`).toBe(true) + } + // 尾斜杠→index.html / .html 字面 两形态都有（mini 夹具两族齐备） + expect(statSync(join(distDir, 'index.html')).isFile()).toBe(true) + expect(statSync(join(distDir, 'news', 'fixture-news-a.html')).isFile()).toBe(true) + + // ---- 媒体复制：assets.json 逐条 copyPath→dist/public_path，sha256 全量校验 ---- + expect(assets).toHaveLength(2) + for (const a of assets) { + const copied = await readFile(join(distDir, a.publicPath)) + expect(sha256of(copied)).toBe(a.sha256) + expect(copied.byteLength).toBe(Number(a.byteSize)) + } + expect(sha256of(await readFile(join(distDir, '/wp-content/uploads/2026/10/fixture-a.png')))).toBe(sha256of(w.pngA)) + expect(sha256of(await readFile(join(distDir, '/wp-content/uploads/2026/10/fixture-b.png')))).toBe(sha256of(w.pngB)) + + // ---- sitemap.xml：origin + indexable html 行；条目数=indexable html 数 ---- + const sitemap = await readFile(join(distDir, 'sitemap.xml'), 'utf8') + const indexable = htmlRoutes.filter((r: any) => r.indexable) + const locs = [...sitemap.matchAll(/(.*?)<\/loc>/g)].map((m) => m[1]!) + expect(locs).toHaveLength(indexable.length) + expect(locs).toStrictEqual(indexable.map((r: any) => `${site.origin}${r.routePath}`).sort()) + expect(sitemap.startsWith('')).toBe(true) + expect(sitemap).toContain('') + + // ---- robots.txt ---- + expect(await readFile(join(distDir, 'robots.txt'), 'utf8')).toBe( + `User-agent: *\nAllow: /\n\nSitemap: ${site.origin}/sitemap.xml\n`, + ) + + // ---- route-manifest.json：§12.1 字段面全断言 ---- + const manifestPath = join(w.work, 'artifacts', 'jobs', result.jobId, 'route-manifest.json') + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) + expect(isRouteManifest(manifest)).toBe(true) + expect(manifest.manifest_version).toBe(ROUTE_MANIFEST_VERSION) + expect(manifest.site_id).toBe(siteId) + expect(manifest.build_job_id).toBe(result.jobId) + expect(manifest.origin).toBe(site.origin) + // 哈希链复用：generated_from = 输入 render-manifest 的 route 快照哈希 + expect(manifest.generated_from_route_snapshot_hash).toBe(inputManifest.sourceHashes.route) + expect(manifest.routes).toHaveLength(routes.length) + // 序 = routes.json 序（物化排序确定） + expect(manifest.routes.map((r: any) => r.exact_key)).toStrictEqual( + routes.map((r: any) => (r.queryKey === null ? r.routePath : `${r.routePath}?${r.queryKey}`)), + ) + // html 行：target_file 派生映射 + 200 + text/html + indexable + const home = manifest.routes.find((r: any) => r.exact_key === '/') + expect(home).toStrictEqual({ + exact_key: '/', + query_policy: 'ignore', + query_key: null, + kind: 'html', + target_file: 'index.html', + status: 200, + content_type: 'text/html', + indexable: true, + }) + const literal = manifest.routes.find((r: any) => r.exact_key === '/news/fixture-news-a.html') + expect(literal.target_file).toBe('news/fixture-news-a.html') + // redirect 查询形态：?attachment_id= 查询键 → exact_key 带查询串 + const redirectA = manifest.routes.find((r: any) => r.exact_key === '/?attachment_id=201') + expect(redirectA).toStrictEqual({ + exact_key: '/?attachment_id=201', + query_policy: 'exact', + query_key: 'attachment_id=201', + kind: 'redirect', + status: 301, + location: '/wp-content/uploads/2026/10/fixture-a.png', + indexable: false, + }) + // asset 行：target_file=public_path + content_type=mime + 200 + const assetRow = manifest.routes.find( + (r: any) => r.exact_key === '/wp-content/uploads/2026/10/fixture-a.png', + ) + expect(assetRow).toStrictEqual({ + exact_key: '/wp-content/uploads/2026/10/fixture-a.png', + query_policy: 'ignore', + query_key: null, + kind: 'asset', + target_file: '/wp-content/uploads/2026/10/fixture-a.png', + status: 200, + content_type: 'image/png', + indexable: false, + }) + + // ---- build_artifacts 三行（dist/manifest/report）---- + const rows = plain( + (await db.execute(`select kind, storage_path, sha256, byte_size from build_artifacts + where build_job_id = '${result.jobId}' order by kind`)) as any, + ) + expect(rows.map((r) => r.kind)).toStrictEqual(['dist', 'manifest', 'report']) + const distInfo = await dirSha256(distDir) + const distRow = rows.find((r) => r.kind === 'dist')! + expect(distRow.storage_path).toBe(distDir) + expect(distRow.sha256).toBe(distInfo.hash) + expect(Number(distRow.byte_size)).toBe(distInfo.bytes) + const manifestRow = rows.find((r) => r.kind === 'manifest')! + expect(manifestRow.storage_path).toBe(manifestPath) + expect(manifestRow.sha256).toBe(sha256of(await readFile(manifestPath, 'utf8'))) + expect(Number(manifestRow.byte_size)).toBe((await readFile(manifestPath)).byteLength) + const reportRow = rows.find((r) => r.kind === 'report')! + expect(reportRow.storage_path).toBe(result.reportPath) + expect(reportRow.sha256).toBe(sha256of(await readFile(result.reportPath, 'utf8'))) + + // UNIQUE(build_job_id, kind, storage_path)：同三元组重复登记被拒 + const dupMsg = await rejectionMessage( + db, + `insert into build_artifacts (build_job_id, kind, storage_path, sha256, byte_size) + values ('${result.jobId}', 'dist', '${distDir}', 'x', 1)`, + ) + expect(dupMsg).toContain('build_artifacts_uq') + + // 报告含产物面（dist/manifest 路径与哈希） + const report = JSON.parse(await readFile(result.reportPath, 'utf8')) + expect(report.artifacts.distPath).toBe(distDir) + expect(report.artifacts.distSha256).toBe(distInfo.hash) + expect(report.artifacts.routeManifestPath).toBe(manifestPath) + expect(report.artifacts.mediaCopied).toBe(2) + + // 主题包内 dist 是管线暂存（spawn 后处理完已拷出并清空）——产物权威在 artifacts 根 + expect(existsSync(themeDistDir())).toBe(false) + } finally { + await cleanupSiteRows(db, siteId) + } + } finally { + await rm(w.work, { recursive: true, force: true }) + } + }) + }) + + it('失败路径：SF_BUILD_INPUT 六文件面破裂 → spawn 非零退出 + stderr 标记 → job failed + error_summary 含标记 + dist 清残', { timeout: 300_000 }, async () => { + await withTestDb(async (db) => { + const w = await makeWork('sf-pipe-fail') + try { + const { key, siteId } = await seedPublishedSite(db, w) + try { + const artifactsRoot = join(w.work, 'artifacts') + let failingJobId = '' + const err: unknown = await createBuildJob(db, { + siteKey: key, + artifactsRoot, + onStatus: (s, jobId) => { + // building 挂钩点：物化已完成、spawn 未起——抽走 content.json 迫使主题构建 + // 失败（管线预读 routes/assets/site/render-manifest 四文件，content.json 是 + // 六文件面里"只有主题读"的成员——失败确定落在 spawn 而非管线预检） + if (s === 'building') { + failingJobId = jobId + rmSync(join(artifactsRoot, 'jobs', jobId, 'input', 'content.json'), { force: true }) + } + }, + }).then( + () => { + throw new Error('不应成功') + }, + (e: unknown) => e, + ) + expect(err).toBeInstanceOf(Error) + expect((err as Error).message).toContain('astro') + + // job 行：failed 终态 + 错误面（Windows 判据：非零退出 + stderr 标记，不断言精确码） + const job = plain( + (await db.execute(`select status, error_code, error_summary from build_jobs + where id = '${failingJobId}'`)) as any, + ) + expect(job).toHaveLength(1) + expect(job[0].status).toBe('failed') + expect(job[0].error_code).toBe('THEME_BUILD_FAILED') + expect(job[0].error_summary).toContain('Render Snapshot 文件缺失或不可读') + expect(job[0].error_summary).toContain('content.json') + + // dist 清残：产物目录与主题暂存 dist 均无残留；input/ 同样清场（failed 无产物语义） + expect(existsSync(join(artifactsRoot, 'jobs', failingJobId, 'dist'))).toBe(false) + expect(existsSync(themeDistDir())).toBe(false) + expect(existsSync(join(artifactsRoot, 'jobs', failingJobId, 'input'))).toBe(false) + // 半途失败无 build_artifacts 行 + expect( + await countOf(db, `select count(*)::int as n from build_artifacts where build_job_id = '${failingJobId}'`), + ).toBe(0) + } finally { + await cleanupSiteRows(db, siteId) + } + } finally { + await rm(w.work, { recursive: true, force: true }) + } + }) + }) +}) + +describe('主题夹具直跑管线（Task 3 render-input 作 SF_BUILD_INPUT）', () => { + it('编码 URL sitemap 原样 + .html 字面 + 3 媒体全量 sha256 + redirect 行；暂存 dist 清空', { timeout: 240_000 }, async () => { + await withTestDb(async (db) => { + const w = await mkdtemp(join(tmpdir(), 'sf-pipe-fixture-')) + const siteFix = await createSiteFixture(db, 'sfpipe') + try { + // 手构 job 行（building 态——管线直跑不经过 service 状态机） + const job = (await db.execute(`insert into build_jobs (id, site_id, status, idempotency_key, attempt) + values (gen_random_uuid(), '${siteFix.siteId}', 'building', 'build-pipe-${siteFix.rand}', 1) + returning id`)) as any + const jobId = job[0].id + + // 输入 = 主题夹具六文件 + 真媒体注入（copyPath/sha256/byteSize 换真值——夹具占位哈希无对应实体） + const inputDir = join(w, 'input') + await mkdir(inputDir, { recursive: true }) + for (const name of ['site.json', 'content.json', 'taxonomies.json', 'routes.json', 'render-manifest.json']) { + await copyFile(join(THEME_FIXTURE_INPUT, name), join(inputDir, name)) + } + const mediaFiles: Array<{ pub: string; buf: Buffer }> = [ + { pub: '/wp-content/uploads/2026/10/fixture-a.png', buf: mediaBytes(`fx-a-${rand()}`) }, + { pub: '/wp-content/uploads/2026/10/fixture-b.jpg', buf: mediaBytes(`fx-b-${rand()}`) }, + { pub: '/wp-content/uploads/2026/10/fixture-c.jpg', buf: mediaBytes(`fx-c-${rand()}`) }, + ] + const binDir = join(w, 'media-bin') + await mkdir(binDir, { recursive: true }) + const assets = JSON.parse(await readFile(join(THEME_FIXTURE_INPUT, 'assets.json'), 'utf8')) + expect(assets).toHaveLength(3) + const rewritten: any[] = [] + for (let i = 0; i < assets.length; i++) { + const m = mediaFiles[i]! + const file = join(binDir, `media-${i}.bin`) + await writeFile(file, m.buf) + rewritten.push({ ...assets[i], copyPath: file, sha256: sha256of(m.buf), byteSize: m.buf.byteLength }) + } + await writeFile(join(inputDir, 'assets.json'), `${canonicalJson(rewritten)}\n`, 'utf8') + + const artifactsRoot = join(w, 'artifacts') + const result = await runBuildPipeline(db, { jobId, siteId: siteFix.siteId, inputDir, artifactsRoot }) + expect(result.mediaCopied).toBe(3) + expect(result.sitemapEntries).toBe(14) + + const distDir = join(artifactsRoot, 'jobs', jobId, 'dist') + expect(result.distPath).toBe(distDir) + const routes = JSON.parse(await readFile(join(inputDir, 'routes.json'), 'utf8')) + + // 全量落盘映射：14 html 行 target_file 存在且 isFile（含编码目录与 .html 字面） + const htmlRoutes = routes.filter((r: any) => r.routeKind === 'html' && r.queryKey === null) + expect(htmlRoutes).toHaveLength(14) + for (const r of htmlRoutes) { + expect(statSync(join(distDir, distRel(r.routePath))).isFile(), r.routePath).toBe(true) + } + const encoded = '%e5%85%b3%e4%ba%8e%e6%88%91%e4%bb%ac' + expect(statSync(join(distDir, encoded, 'index.html')).isFile()).toBe(true) + expect(statSync(join(distDir, 'news', 'fixture-news-a.html')).isFile()).toBe(true) + + // 3 媒体全量 sha256 校验（非抽检——夹具 3 条全对） + for (const m of mediaFiles) { + expect(sha256of(await readFile(join(distDir, m.pub)))).toBe(sha256of(m.buf)) + } + + // sitemap：编码 URL 原样（%xx 字节不变、无解码孪生、无二次编码）+ 条目数断言 + const origin = 'https://demo.example' + const sitemap = await readFile(join(distDir, 'sitemap.xml'), 'utf8') + const locs = [...sitemap.matchAll(/(.*?)<\/loc>/g)].map((m) => m[1]!) + expect(locs).toHaveLength(14) + expect(locs).toContain(`${origin}/${encoded}/`) + expect(sitemap).not.toContain('关于我们') + expect(sitemap).not.toContain('%25e5') + expect(await readFile(join(distDir, 'robots.txt'), 'utf8')).toBe( + `User-agent: *\nAllow: /\n\nSitemap: ${origin}/sitemap.xml\n`, + ) + + // route-manifest：15 行全量（html 14 + redirect 1）+ redirect 字段面 + const manifest = JSON.parse(await readFile(result.routeManifestPath, 'utf8')) + expect(isRouteManifest(manifest)).toBe(true) + expect(manifest.routes).toHaveLength(15) + expect(manifest.site_id).toBe(siteFix.siteId) + expect(manifest.build_job_id).toBe(jobId) + expect(manifest.origin).toBe(origin) + const redirect = manifest.routes.find((r: any) => r.exact_key === '/old-path/') + expect(redirect).toStrictEqual({ + exact_key: '/old-path/', + query_policy: 'ignore', + query_key: null, + kind: 'redirect', + status: 301, + location: '/', + indexable: false, + }) + + // build_artifacts 两行（dist/manifest——report 行是 service 落报告后的职责） + const rows = plain( + (await db.execute(`select kind from build_artifacts where build_job_id = '${jobId}' order by kind`)) as any, + ) + expect(rows.map((r) => r.kind)).toStrictEqual(['dist', 'manifest']) + expect(rows.length).toBe(2) + + // 暂存 dist 清空（主题包目录回零——产物权威在 artifacts 根） + expect(existsSync(themeDistDir())).toBe(false) + } finally { + await db.execute(`delete from build_artifacts where build_job_id in + (select id from build_jobs where site_id = '${siteFix.siteId}')`) + await db.execute(`delete from build_jobs where site_id = '${siteFix.siteId}'`) + await siteFix.cleanup() + await rm(w, { recursive: true, force: true }) + } + }) + }) +}) + +describe('五禁断言（主题构建期纪律）', () => { + it('① loader 单一入口：主题 src 数据访问面只在 load-render-data；④ 无 seed；⑤ 不读源快照', async () => { + const src = await scanThemeSrc() + expect(src.size).toBeGreaterThan(0) + // ① fs/DB/env 数据访问只准出现在 lib/load-render-data.ts（其余模块经它取数） + const dataAccess = /node:fs|readFileSync|DATABASE_URL|postgres\b|from\s+'pg'/ + const offenders = [...src.entries()].filter(([, c]) => dataAccess.test(c)).map(([p]) => p) + expect(offenders).toStrictEqual(['lib/load-render-data.ts']) + // 动态分派页确实经唯一入口取数（结构保证不是纸面声明） + expect(src.get('pages/[...slug].astro')).toContain("from '../lib/load-render-data'") + // ④ fixtures 引用只在 test/（src 零命中——无站点专属 seed） + expect([...src.entries()].filter(([, c]) => /fixtures/i.test(c))).toStrictEqual([]) + // ⑤ 源快照路径零命中（storage/snapshots 是 register 域布局，主题不读原始输入） + expect([...src.entries()].filter(([, c]) => /storage[\\/](snapshots|incoming)/.test(c))).toStrictEqual([]) + }) + + it('② DATABASE_URL 置空 + ③ 无网 stub（global fetch/net.connect 抛错）→ 构建仍成功', { timeout: 300_000 }, async () => { + await withTestDb(async (db) => { + const w = await makeWork('sf-pipe-nonet') + // 无网 stub：--import 预载——构建进程树内任何 fetch/net.connect 即抛错 + const stub = join(w.work, 'no-net-stub.mjs') + await writeFile( + stub, + [ + '// SiteForge 无网证明 stub（Plan 4 Task 5 五禁⑤）：构建进程树不得联网。', + '// 只改 net.default.*（CJS module.exports——require 消费方所见）；ESM 命名空间导出', + '// 是只读绑定，直接赋值 net.connect 会在预载期崩溃（实测 TypeError）。', + "const boom = () => { throw new Error('sf-no-network: 构建进程树出现 fetch/net.connect 调用') }", + 'globalThis.fetch = () => boom()', + "const net = await import('node:net')", + 'net.default.connect = () => boom()', + 'net.default.createConnection = () => boom()', + ].join('\n'), + 'utf8', + ) + const savedUrl = process.env.DATABASE_URL + const savedNodeOptions = process.env.NODE_OPTIONS + process.env.DATABASE_URL = '' + process.env.NODE_OPTIONS = `--import ${pathToFileURL(stub).href}` + try { + const { key, siteId } = await seedPublishedSite(db, w) + try { + const result = await createBuildJob(db, { + siteKey: key, + artifactsRoot: join(w.work, 'artifacts'), + }) + expect(result.status).toBe('succeeded') + // 构建真实发生且产物落盘（spawn 树在 stub 钳制下完成——主题/astro 构建期不联网） + const distDir = join(w.work, 'artifacts', 'jobs', result.jobId, 'dist') + expect(statSync(join(distDir, 'index.html')).isFile()).toBe(true) + expect(statSync(join(distDir, 'sitemap.xml')).isFile()).toBe(true) + } finally { + await cleanupSiteRows(db, siteId) + } + } finally { + process.env.DATABASE_URL = savedUrl + process.env.NODE_OPTIONS = savedNodeOptions + await rm(w.work, { recursive: true, force: true }) + } + }) + }) +}) diff --git a/tests/integration/build-snapshot.test.ts b/tests/integration/build-snapshot.test.ts index 645bc1f..f356082 100644 --- a/tests/integration/build-snapshot.test.ts +++ b/tests/integration/build-snapshot.test.ts @@ -1,7 +1,7 @@ // Plan 4 Task 2 失败测试（先于实现）：Render Snapshot 物化 + build job（ADR-011、V2 spec §9）。 // 覆盖契约（对照计划 Task 2）： -// 1. build job 状态机：queued→snapshotting→validating（空，Task 6 填）→succeeded（dist 尚空 -// ——Task 5 接真实构建时新建 job 不改行，build_input_snapshots 有不可变触发器）+ +// 1. build job 状态机：queued→snapshotting→building（Task 5 真实构建）→validating +// （空，Task 6 填）→succeeded + // build_input_snapshots 行（content/route/asset 三哈希 = 各自**文件字节**（canonicalJson+ // 尾换行）的 sha256——哈希链与落盘文件互证、site_config_revision_id 指向 config v1、 // theme_key='pilot-minimal'、node_version 等版本信息列真实读 process.versions）+ @@ -71,8 +71,10 @@ async function countOf(db: Db, sql: string): Promise { return Number(rows[0].n) } -/** 本文件写入的行清场：构建域（不可变快照先行）+ 发布/导入/来源全链反序 */ +/** 本文件写入的行清场：构建域（build_artifacts 先于 build_jobs——FK；不可变快照先行）+ 发布/导入/来源全链反序 */ async function cleanupSiteRows(db: Db, siteId: string): Promise { + await db.execute(`delete from build_artifacts where build_job_id in + (select id from build_jobs where site_id = '${siteId}')`) await withImmutableCleanup(db, 'build_input_snapshots', 'build_input_snapshots_immutable', `site_id = '${siteId}'`) await db.execute(`delete from build_jobs where site_id = '${siteId}'`) await withImmutableCleanup(db, 'audit_logs', 'audit_logs_immutable', `site_id = '${siteId}'`) @@ -158,8 +160,9 @@ describe('build job + Render Snapshot 物化', () => { }) expect(result.reused).toBe(false) expect(result.status).toBe('succeeded') - // 状态机序列（validating 空：Task 6 填真实门禁） - expect(statuses).toStrictEqual(['queued', 'snapshotting', 'validating', 'succeeded']) + // 状态机序列（Task 5 起 snapshotting 后进 building：真实 astro 构建管线； + // validating 空：Task 6 填真实门禁） + expect(statuses).toStrictEqual(['queued', 'snapshotting', 'building', 'validating', 'succeeded']) // build_jobs 行：终态 + attempt/时间戳/error 空（epoch 投影——时间戳非空身份断言； // raw execute 的 timestamptz 回串不做 Date 形状断言） diff --git a/themes/pilot-minimal/src/pages/[...slug].astro b/themes/pilot-minimal/src/pages/[...slug].astro index 0db6cd3..9e65b95 100644 --- a/themes/pilot-minimal/src/pages/[...slug].astro +++ b/themes/pilot-minimal/src/pages/[...slug].astro @@ -17,6 +17,8 @@ // 前缀 /product-category/ 或 /category/ → CategoryList（term 从 taxonomies 按 slug 段查） // '/news/' → NewsList（主列表特判——真实数据 posts 落根路径， // /news/ 是 route-writer finish 期生成的主列表行） +// '/products/' → ProductList（产品主列表特判——同 finish 期主列表行， +// targetContentId=null 无内容锚点；Task 5 集成补面） // 无法分派的 html 行 → 构建期明错（§7.6 禁静默——不允许空页混过）。 import { loadRenderData } from '../lib/load-render-data' import type { ContentEntry, RenderData, RouteEntry, TaxonomyTerm } from '../lib/load-render-data' @@ -26,6 +28,7 @@ import Home from '../templates/Home.astro' import NewsDetail from '../templates/NewsDetail.astro' import NewsList from '../templates/NewsList.astro' import ProductDetail from '../templates/ProductDetail.astro' +import ProductList from '../templates/ProductList.astro' /** 分类列表前缀（term-writer 前置事实 9 常量的两族 basePath；publicBasePath 尾斜杠已归一） */ const CATEGORY_PREFIXES = ['/product-category/', '/category/'] @@ -37,6 +40,7 @@ type View = | { kind: 'product'; content: ContentEntry } | { kind: 'news-detail'; content: ContentEntry } | { kind: 'news-list' } + | { kind: 'product-list' } function dispatch(route: RouteEntry, data: RenderData): View { if (route.routePath === '/') return { kind: 'home' } @@ -70,6 +74,10 @@ function dispatch(route: RouteEntry, data: RenderData): View { if (route.routePath === '/news/') return { kind: 'news-list' } + // 产品主列表特判（Task 5 集成补面）：route-writer finish() 对 productsPath 落的主列表 + // 行 targetContentId=null——真实物化形状无内容锚点，须与 /news/ 对称的列表落点 + if (route.routePath === '/products/') return { kind: 'product-list' } + throw new Error( `路由 ${route.routePath} 无法分派到任何模板（html 纯路径行必须有落点；主列表/分类/内容页规则均未命中）——拒绝静默空页`, ) @@ -99,3 +107,4 @@ const view = dispatch(route, data) {view.kind === 'product' && } {view.kind === 'news-detail' && } {view.kind === 'news-list' && } +{view.kind === 'product-list' && } diff --git a/themes/pilot-minimal/src/templates/ProductList.astro b/themes/pilot-minimal/src/templates/ProductList.astro new file mode 100644 index 0000000..356121e --- /dev/null +++ b/themes/pilot-minimal/src/templates/ProductList.astro @@ -0,0 +1,47 @@ +--- +// 产品主列表模板（Plan 4 Task 5 集成补面）：route-writer finish() 对 productsPath（默认 +// /products/）落的主列表行 targetContentId=null——Task 4 的分派器只覆盖了 news 主列表 +// 特判，产品主列表在主题夹具里被"targetContentId 占位"绕过（见 CategoryList 头注记）， +// DB 真实物化形状（targetContentId=null）即无处落点。本模板补齐：全部 product 卡片 +// 单页（latestEntries 同款排序），无落点内容不出卡（禁虚构路径）。 +// 与 news-list 同构（主列表两兄弟）；manifest 的 'product-list' 模板键声明性覆盖本页 +// （appliesTo 单字符串不承载多前缀——/products/ 主列表由分派器特判，/category/ 别名 +// 前缀同先例）。 +import BaseLayout from '../layouts/BaseLayout.astro' +import Breadcrumbs from '../components/Breadcrumbs.astro' +import ProductCard from '../components/ProductCard.astro' +import type { RenderData, RouteEntry } from '../lib/load-render-data' +import { contentTitle, coverRef, latestEntries, metaDescription, routeHrefFor } from '../lib/ui' + +interface Props { + data: RenderData + route: RouteEntry +} + +const { data, route } = Astro.props +const page = { path: route.routePath, title: 'Products' } +const allProducts = latestEntries(data.contents, 'product') +--- + + + +

Products

+
+ {allProducts.map((c) => { + const href = routeHrefFor(data.routes, c.contentId) + if (href === undefined) return null + return + })} +
+ + + diff --git a/themes/pilot-minimal/theme.manifest.ts b/themes/pilot-minimal/theme.manifest.ts index cd9320e..5484dce 100644 --- a/themes/pilot-minimal/theme.manifest.ts +++ b/themes/pilot-minimal/theme.manifest.ts @@ -5,7 +5,9 @@ // 'route:' 精确路由命中（home= '/'，not-found = '/404'） // 'type:' 按内容类型分派（详情页） // 'path-prefix:' 按路径前缀分派（列表页；product-list 的 /category/ 别名前缀 -// 由 Task 4 分派器一并处理——单一 appliesTo 字符串不承载多前缀） +// 由 Task 4 分派器一并处理——单一 appliesTo 字符串不承载多前缀； +// /products/ 产品主列表同此先例：Task 5 集成补面的 ProductList +// 模板由分派器精确路由特判，不另立模板键） import type { ThemeManifest } from '@sf/contracts' export const themeManifest: ThemeManifest = {