From e1e97dac2c450b6de1a752f93da8488e78a4a6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Tue, 20 Oct 2026 21:16:46 +0800 Subject: [PATCH] =?UTF-8?q?fix(site-import):=20FetchError=20=E7=99=BD?= =?UTF-8?q?=E8=AF=9D=E5=8C=96=E6=98=A0=E5=B0=84=EF=BC=88spec=20=C2=A76=20?= =?UTF-8?q?=E8=A1=8C1=EF=BC=89+=20=E9=A2=84=E7=AE=97=E8=AF=AD=E4=B9=89?= =?UTF-8?q?=E6=B3=A8=E8=AE=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _pipeline 增 FetchError 分支（先于通用异常兜底）：入口页不可达/超时—— 最高频真实失败路径——映射白话文案「网站打不开或响应太慢——检查 URL」 + phase=crawl（不可 retry）；ImportTask docstring 的抓取失败归档口径 同步对齐。crawler 模块 docstring 注记 MAX_PAGES 预算语义（只计成功入编 页，链接农场广度消耗由总超时/并发/频控三层兜底）。 Co-Authored-By: Claude Code --- app/platform/site_import/crawler.py | 4 +++- app/platform/site_import/task.py | 9 +++++++-- tests/test_site_import_task.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/app/platform/site_import/crawler.py b/app/platform/site_import/crawler.py index 3c8b84f..493e5ca 100644 --- a/app/platform/site_import/crawler.py +++ b/app/platform/site_import/crawler.py @@ -7,7 +7,9 @@ CSS/logo 追抓（15s 超时）→ 组装 crawl_result dict 供生成器。 <200 字符 → SPA 空壳（子页过薄只跳过不失败——JS 渲染的产品子页常见， 整站不该陪葬）。单页 >500KB 截断解析（clean_html 未闭合块二遍清洗兜底）。 discover_links 会回吐入口自身（href="#"/"/"）——visited 预置 start URL -去重，防浪费 ≤10 页名额（Task 3 备忘）。 +去重，防浪费 ≤10 页名额（Task 3 备忘）。MAX_PAGES 只计成功入编的页 +（薄页/非 html/失败页不占名额）——链接农场的广度消耗由 7 分钟总超时/ +并发 ≤2/频控三层兜底（预算语义记档）。 """ from __future__ import annotations diff --git a/app/platform/site_import/task.py b/app/platform/site_import/task.py index f3ecd68..35ca14a 100644 --- a/app/platform/site_import/task.py +++ b/app/platform/site_import/task.py @@ -20,6 +20,7 @@ from collections import defaultdict from app.platform.site_import import crawler, generator from app.platform.site_import.crawler import CrawlError, INSUFFICIENT_MESSAGE from app.platform.site_import.generator import GenerateError +from app.platform.site_import.safe_fetch import FetchError TASK_TTL = 30 * 60 TOTAL_TIMEOUT = 7 * 60 @@ -89,8 +90,8 @@ class ImportTask: """一条导入任务：状态机 + 阶段产物 + 强引用的后台 aio task。 failed_phase："generate" 可 retry=generate 复活（复用 crawl_result， - 不重抓）；"crawl"（SPA/内容不足/抓取失败）与 None（超时落在抓取段/ - 未知异常）不可复活——重开任务。 + 不重抓）；"crawl"（SPA/内容不足/FetchError 网站打不开）与 None（超时 + 落在抓取段/未知异常）不可复活——重开任务。 """ def __init__(self, slug: str, task_id: str, url: str | None) -> None: @@ -201,6 +202,10 @@ async def _pipeline(task_obj: ImportTask) -> None: phase="generate" if task_obj.crawl_result is not None else None) except CrawlError as exc: task_obj.fail(str(exc), phase="crawl") # SPA/内容不足——不可 retry + except FetchError: + # spec §6 行1：入口页不可达/超时是最高频真实失败路径——白话文案， + # 不漏进通用兜底裸抛英文内部信息 + task_obj.fail("网站打不开或响应太慢——检查 URL", phase="crawl") except GenerateError as exc: task_obj.fail(str(exc), phase="generate") # retry=generate 复活 except Exception as exc: # 防御：未知异常不裸抛（任务表对象完整性优先） diff --git a/tests/test_site_import_task.py b/tests/test_site_import_task.py index 7fa3597..1292388 100644 --- a/tests/test_site_import_task.py +++ b/tests/test_site_import_task.py @@ -433,6 +433,18 @@ class TestTasks: with pytest.raises(ValueError): await task.start("t1", None, retry_task_id=tid_b) + async def test_FetchError白话化映射(self, monkeypatch): + # spec §6 行1：站点不可达/超时是最高频真实失败路径——不漏进 + # 通用异常兜底裸抛英文内部信息 + stubs = _Stubs().install(monkeypatch) + stubs.crawl_errors.append(safe_fetch.FetchError( + "fetch failed for 'https://a.com/': timed out")) + tid = await task.start("t1", "https://a.com/") + t = await _wait("t1", tid) + assert t.state == "failed" + assert t.error == "网站打不开或响应太慢——检查 URL" + assert t.failed_phase == "crawl" + async def test_30分钟惰性清理(self, monkeypatch): _Stubs().install(monkeypatch) tid = await task.start("t1", "https://a.com/")