fix(import): 失败报告部分冲刷;列表路径校验(建行前);JSDoc 求和措辞;reason 归档
Task 6 质量评审四项修复: - 失败报告部分冲刷(Fix 1):orchestratedPipeline 分派/finish 包 try/catch—— 中途抛错时把可安全取得的部分产出冲入 ctx.report 后重抛:term/content 的 finish 是纯返回(已算成复用、未算成就地补算 trySyncFinish,自身抛错放弃 该部分不掩盖原错)、已收集的 adapter 问题/清洗统计、进度计数 eventsDispatched(orchestrator 键,仅失败路径出现——成功路径不占 18 键面)。 media/route finish 依赖库与先序产物,失败路径不调用(其键缺席如实反映)。 - 列表路径校验(Fix 3):ImportErrorCode 增 INVALID_LIST_PATH;解析期防线 移至阶段 2.5(建 job 行前——建行/置 running 后再抛会留悬挂行);复用 route-writer 导出的 isSafePath;normalizeListPath 自带安全拒绝(写入器 自调用位双覆盖)。CLI 无需新码(既有 ImportError 通道按 code 呈现)。 - JSDoc 求和措辞(Fix 2):content-writer/media-writer 的 counts 键契约 注记由"返回值覆盖同名键 CLOBBER"改为"同名键求和(Task 6 钉死——防覆盖 安全网)",与 ImportPipelineOutcome.counts 契约对齐。 - reason 归档(Fix 4):分派循环对 unsupported/error 事件把 reason/issue 推入 adapter 问题组(此前被静默丢弃)。 测试:import-full 增 2 例(手构流 3 事件后爆炸 → failed + eventsDispatched=3 + 部分 content/term 计数 + unsupported/error reason 双归档;--products-path /../x/ 等 → INVALID_LIST_PATH 零 job 行)。全量 325 通过;typecheck/lint 干净。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -51,7 +51,13 @@ import { isUniqueViolation } from '../source/register.service.js'
|
||||
import { createContentEventHandler } from './content-writer.js'
|
||||
import { createTermEventHandler } from './term-writer.js'
|
||||
import { createMediaEventHandlerFromContext } from './media-writer.js'
|
||||
import { DEFAULT_NEWS_PATH, DEFAULT_PRODUCTS_PATH, createRouteEventHandler, normalizeListPath } from './route-writer.js'
|
||||
import {
|
||||
DEFAULT_NEWS_PATH,
|
||||
DEFAULT_PRODUCTS_PATH,
|
||||
createRouteEventHandler,
|
||||
isSafePath,
|
||||
normalizeListPath,
|
||||
} from './route-writer.js'
|
||||
|
||||
/** 本文件位于 <repo>/apps/api/src/modules/import/ → 上五级是仓库根 */
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../../..')
|
||||
@@ -66,6 +72,7 @@ export type ImportErrorCode =
|
||||
| 'NO_WXR_SNAPSHOT'
|
||||
| 'PIPELINE_FAILED'
|
||||
| 'REPORT_WRITE_FAILED'
|
||||
| 'INVALID_LIST_PATH'
|
||||
|
||||
/** 导入失败(可预期错误):CLI 以 code 区分呈现;不裹程序 bug(那些原样上抛) */
|
||||
export class ImportError extends Error {
|
||||
@@ -139,7 +146,9 @@ export interface ImportJobContext {
|
||||
*
|
||||
* 汇合规则(Task 6 钉死):最终产出 = ctx.report 累加器 ⊕ 管道返回值——counts 按键
|
||||
* **求和**、issues 串联(累加器在前、返回值在后)、issueGroups 逐键串联、sanitize
|
||||
* 数值相加。管道抛错时无返回值,累加器现值原样进报告(部分产出可审计)。
|
||||
* 数值相加。管道抛错时无返回值,累加器现值进报告(部分产出可审计)——缺省编排管道
|
||||
* 在重抛前自行冲刷部分产出(term/content 纯返回 finish + adapter 问题/清洗统计 +
|
||||
* eventsDispatched 进度计数,见 orchestratedPipeline),注入管道的冲刷行为由注入方自理。
|
||||
*/
|
||||
export interface ImportPipelineOutcome {
|
||||
/**
|
||||
@@ -149,7 +158,8 @@ export interface ImportPipelineOutcome {
|
||||
* 各写入器仍须各持独占键名并在自身 JSDoc 声明归属(term:termsProcessed/
|
||||
* autoCreatedTerms/contentTermsCreated;content:contentEntries/revisionsCreated/
|
||||
* sourceRecords/skipped/unsupportedRecords/failedRecords;media 六键 media 前缀;
|
||||
* route 五键 routes 前缀——新写入器勿复用他家键名)。
|
||||
* route 五键 routes 前缀;orchestrator:eventsDispatched——仅失败路径部分冲刷时
|
||||
* 出现,成功路径不占键——新写入器勿复用他家键名)。
|
||||
*/
|
||||
counts: Record<string, number>
|
||||
issues: string[]
|
||||
@@ -202,9 +212,12 @@ export interface CreateImportJobInput {
|
||||
events?: AsyncIterable<AdapterEvent>
|
||||
/** 管道钩子(测试注入);缺省 = 编排管道 */
|
||||
pipeline?: ImportPipeline
|
||||
/** 主列表页路径(前置事实 9:CLI 参数来源,规范化值记入 job 报告);缺省 /products/ */
|
||||
/**
|
||||
* 主列表页路径(前置事实 9:CLI 参数来源,规范化值记入 job 报告);缺省 /products/。
|
||||
* 解析期防线(阶段 2.5,建 job 行前):不安全段(穿越/控制字符等)→ INVALID_LIST_PATH
|
||||
*/
|
||||
productsPath?: string
|
||||
/** 新闻列表页路径(同上);缺省 /news/ */
|
||||
/** 新闻列表页路径(同上,缺省 /news/;防线同 productsPath) */
|
||||
newsPath?: string
|
||||
}
|
||||
|
||||
@@ -300,13 +313,34 @@ export function defaultIdempotencyKey(siteKey: string, wxrSnapshotId: string): s
|
||||
return `import:${siteKey}:${wxrSnapshotId}:${ADAPTER_VERSION}/${MAPPING_VERSION}/${SANITIZER_VERSION}`
|
||||
}
|
||||
|
||||
/**
|
||||
* 主列表路径解析(阶段 2.5 输入防线,Task 6 审查 Fix 3):安全检查(route-writer 的
|
||||
* isSafePath 同判据)→ normalizeListPath 规范化(补首尾斜杠)。不安全段 →
|
||||
* INVALID_LIST_PATH(CLI 无需新码——既有 ImportError 通道按 code 呈现)。
|
||||
*/
|
||||
function resolveListPath(raw: string, cliFlag: string): string {
|
||||
if (!isSafePath(raw)) {
|
||||
throw new ImportError(
|
||||
'INVALID_LIST_PATH',
|
||||
`${cliFlag}=${raw} 含不安全段(NUL/反斜杠/控制字符/穿越段 '..')——拒绝(完整校验规则 SP3 验证器)`,
|
||||
)
|
||||
}
|
||||
try {
|
||||
return normalizeListPath(raw)
|
||||
} catch (err) {
|
||||
// normalizeListPath 自带同判据拒绝(双覆盖的理论重叠分支)——归码为可预期错误
|
||||
throw new ImportError('INVALID_LIST_PATH', `${cliFlag}=${raw}:${err instanceof Error ? err.message : String(err)}`)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 主入口 ----
|
||||
|
||||
/**
|
||||
* 创建(并就地推进到底)一个导入 job:
|
||||
* 解析(site → wxr 快照 → 侧快照)→ 幂等建行(queued)→ running(attempt=1、started_at)
|
||||
* → 空管道/注入管道 → succeeded|failed(error_code/error_summary/finished_at)→ 报告落盘。
|
||||
* 失败口径:解析阶段失败零 job 行;管道阶段失败 job 行保留(failed 终态 + error 列)。
|
||||
* 失败口径:解析阶段失败零 job 行(含主列表路径防线 INVALID_LIST_PATH——建行前抛出,
|
||||
* 不留悬挂 running 行);管道阶段失败 job 行保留(failed 终态 + error 列)。
|
||||
*/
|
||||
export async function createImportJob(
|
||||
db: ImportDb,
|
||||
@@ -391,6 +425,13 @@ export async function createImportJob(
|
||||
const mediaBundleSnapshot = mediaBundleRow === null ? null : toRef(mediaBundleRow)
|
||||
const manualUrlListSnapshot = manualUrlListRow === null ? null : toRef(manualUrlListRow)
|
||||
|
||||
// 阶段 2.5:主列表路径解析(Task 6 审查 Fix 3——**建 job 行前**的输入防线:穿越段等
|
||||
// 不安全输入 → INVALID_LIST_PATH 零 job 行残留;建行/置 running 后再抛会留悬挂行)。
|
||||
// normalizeListPath 自带同判据安全拒绝(route-writer 侧双覆盖),此处前置检查给出
|
||||
// CLI 参数归属的错误信息。复用命中同样过检:非法输入恒拒(与 SITE_NOT_FOUND 同纪律)。
|
||||
const productsPath = resolveListPath(input.productsPath ?? DEFAULT_PRODUCTS_PATH, '--products-path')
|
||||
const newsPath = resolveListPath(input.newsPath ?? DEFAULT_NEWS_PATH, '--news-path')
|
||||
|
||||
// 阶段 3:幂等建 job(queued)——命中既有 key 直接复用返回(不重跑)。
|
||||
// 复用口径忠实化:只返回 job 行锚定字段(含行内 mapping_version/report_path),
|
||||
// 快照与版本块不回读(权威记录在既有 job 报告——reportPath 指向它)。
|
||||
@@ -434,8 +475,7 @@ export async function createImportJob(
|
||||
// 事件流三分:注入 events 优先(手构流 + 缺省编排管道 = 测试形态)→ 注入 pipeline 而
|
||||
// 无 events = 空流(Task 1 语义保持)→ 双缺席 = 真流(wxr 快照副本 + locale 接线,
|
||||
// 前置事实 8——缺省空串会让全部行被 content-writer 以 locale 不匹配拒收)。
|
||||
const productsPath = normalizeListPath(input.productsPath ?? DEFAULT_PRODUCTS_PATH)
|
||||
const newsPath = normalizeListPath(input.newsPath ?? DEFAULT_NEWS_PATH)
|
||||
// productsPath/newsPath 已在阶段 2.5 解析(建行前防线 + 报告/管道共用规范化值)。
|
||||
const useRealStream = input.events === undefined && input.pipeline === undefined
|
||||
const adapterTimings: AdapterTimings | null = useRealStream ? { parseMs: 0, sanitizeMs: 0 } : null
|
||||
const events: AsyncIterable<AdapterEvent> =
|
||||
@@ -688,7 +728,12 @@ function mergeOutcomes(acc: ImportPipelineOutcome, ret: ImportPipelineOutcome):
|
||||
* (media_assets 唯一写入方),content-writer 对其只留 source_records record-only 足迹
|
||||
* (前置事实 11)——两家各写各域,单一 owner;
|
||||
* - counts 按键求和聚合(安全网——四家键前缀互斥);清洗统计与 row 事件行级问题在
|
||||
* 分派循环收集(RowResult 契约:issues 逐条收录为行级错误)。
|
||||
* 分派循环收集(RowResult 契约:issues 逐条收录为行级错误);unsupported/error 事件
|
||||
* 的 reason/issue 同归 adapter 组(审查 Fix 4——不静默丢弃诊断信息);
|
||||
* - **失败报告部分冲刷**(审查 Fix 1):分派/finish 中途抛错时,把可安全取得的部分产出
|
||||
* 冲入 ctx.report 累加器后重抛——term/content 的 finish 是纯返回(无库依赖,就地补算);
|
||||
* media/route 的 finish 依赖库与先序产物,失败路径**不调用**(其键/issue 缺席于失败报告);
|
||||
* 进度以 orchestrator 键 eventsDispatched 落 counts(仅失败路径出现——成功路径不占键)。
|
||||
*/
|
||||
function orchestratedPipeline(paths: { productsPath: string; newsPath: string }): ImportPipeline {
|
||||
return async (ctx) => {
|
||||
@@ -719,44 +764,86 @@ function orchestratedPipeline(paths: { productsPath: string; newsPath: string })
|
||||
|
||||
const sanitize: ImportSanitizeStats = { changedRows: 0, byDiffType: {} }
|
||||
const adapterIssues: string[] = []
|
||||
for await (const ev of ctx.events) {
|
||||
if (ev.kind === 'row') {
|
||||
if (ev.changed) sanitize.changedRows++
|
||||
for (const diff of ev.sanitizeDiffs) {
|
||||
sanitize.byDiffType[diff.type] = (sanitize.byDiffType[diff.type] ?? 0) + 1
|
||||
}
|
||||
for (const issue of ev.issues) {
|
||||
adapterIssues.push(`行级问题(${ev.row.sourceType}/${ev.row.sourceId}):${issue}`)
|
||||
let eventsDispatched = 0
|
||||
// 已算成的纯返回 finish(media/route finish 抛错时直接复用,不重调)
|
||||
let finished: { content: ImportPipelineOutcome; term: ImportPipelineOutcome } | null = null
|
||||
try {
|
||||
for await (const ev of ctx.events) {
|
||||
if (ev.kind === 'row') {
|
||||
if (ev.changed) sanitize.changedRows++
|
||||
for (const diff of ev.sanitizeDiffs) {
|
||||
sanitize.byDiffType[diff.type] = (sanitize.byDiffType[diff.type] ?? 0) + 1
|
||||
}
|
||||
for (const issue of ev.issues) {
|
||||
adapterIssues.push(`行级问题(${ev.row.sourceType}/${ev.row.sourceId}):${issue}`)
|
||||
}
|
||||
} else if (ev.kind === 'unsupported') {
|
||||
adapterIssues.push(`不支持对象(${ev.sourceType}/${ev.postId ?? '?'}, ${ev.rawLocation}):${ev.reason}`)
|
||||
} else if (ev.kind === 'error') {
|
||||
adapterIssues.push(`适配器错误(${ev.sourceType}/${ev.postId ?? '?'}, ${ev.rawLocation}):${ev.issue}`)
|
||||
}
|
||||
await content.handleEvent(ev)
|
||||
await term.handleEvent(ev)
|
||||
await media.handleEvent(ev)
|
||||
await route.handleEvent(ev)
|
||||
eventsDispatched++
|
||||
}
|
||||
await content.handleEvent(ev)
|
||||
await term.handleEvent(ev)
|
||||
await media.handleEvent(ev)
|
||||
await route.handleEvent(ev)
|
||||
}
|
||||
const t = term.finish()
|
||||
const c = content.finish()
|
||||
const m = await media.finish()
|
||||
const r = await route.finish()
|
||||
const t = term.finish()
|
||||
const c = content.finish()
|
||||
finished = { content: c, term: t }
|
||||
const m = await media.finish()
|
||||
const r = await route.finish()
|
||||
|
||||
const counts: Record<string, number> = {}
|
||||
sumCounts(counts, c.counts)
|
||||
sumCounts(counts, t.counts)
|
||||
sumCounts(counts, m.counts)
|
||||
sumCounts(counts, r.counts)
|
||||
return {
|
||||
counts,
|
||||
issues: [...c.issues, ...t.issues, ...m.issues, ...r.issues, ...adapterIssues],
|
||||
issueGroups: {
|
||||
content: c.issues,
|
||||
term: t.issues,
|
||||
media: m.issues,
|
||||
route: r.issues,
|
||||
const counts: Record<string, number> = {}
|
||||
sumCounts(counts, c.counts)
|
||||
sumCounts(counts, t.counts)
|
||||
sumCounts(counts, m.counts)
|
||||
sumCounts(counts, r.counts)
|
||||
return {
|
||||
counts,
|
||||
issues: [...c.issues, ...t.issues, ...m.issues, ...r.issues, ...adapterIssues],
|
||||
issueGroups: {
|
||||
content: c.issues,
|
||||
term: t.issues,
|
||||
media: m.issues,
|
||||
route: r.issues,
|
||||
adapter: adapterIssues,
|
||||
orchestrator: [],
|
||||
},
|
||||
sanitize,
|
||||
}
|
||||
} catch (err) {
|
||||
// 失败报告部分冲刷(审查 Fix 1):重抛前把可审计的部分产出落入累加器——
|
||||
// createImportJob 对抛错管道取 ctx.report 现值进报告,失败诊断不再只剩 error 块。
|
||||
const c = finished?.content ?? trySyncFinish(content)
|
||||
const t = finished?.term ?? trySyncFinish(term)
|
||||
const partial: Record<string, number> = { eventsDispatched }
|
||||
if (c !== null) sumCounts(partial, c.counts)
|
||||
if (t !== null) sumCounts(partial, t.counts)
|
||||
sumCounts(ctx.report.counts, partial)
|
||||
const partialIssues = [...(c?.issues ?? []), ...(t?.issues ?? []), ...adapterIssues]
|
||||
ctx.report.issues.push(...partialIssues)
|
||||
ctx.report.issueGroups = {
|
||||
content: c?.issues ?? [],
|
||||
term: t?.issues ?? [],
|
||||
media: [],
|
||||
route: [],
|
||||
adapter: adapterIssues,
|
||||
orchestrator: [],
|
||||
},
|
||||
sanitize,
|
||||
}
|
||||
ctx.report.sanitize = sanitize
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 失败路径的安全 finish:content/term 的 finish 是纯返回(无库依赖)——已算成则由调用方
|
||||
* 复用,此处只对未算成的就地补算;finish 本身抛错则放弃该部分(不掩盖原错误) */
|
||||
function trySyncFinish(handler: { finish(): ImportPipelineOutcome }): ImportPipelineOutcome | null {
|
||||
try {
|
||||
return handler.finish()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user