diff --git a/app/platform/site_import/crawler.py b/app/platform/site_import/crawler.py index 3c8b84f..493e5ca 100644 --- a/app/platform/site_import/crawler.py +++ b/app/platform/site_import/crawler.py @@ -7,7 +7,9 @@ CSS/logo 追抓(15s 超时)→ 组装 crawl_result dict 供生成器。 <200 字符 → SPA 空壳(子页过薄只跳过不失败——JS 渲染的产品子页常见, 整站不该陪葬)。单页 >500KB 截断解析(clean_html 未闭合块二遍清洗兜底)。 discover_links 会回吐入口自身(href="#"/"/")——visited 预置 start URL -去重,防浪费 ≤10 页名额(Task 3 备忘)。 +去重,防浪费 ≤10 页名额(Task 3 备忘)。MAX_PAGES 只计成功入编的页 +(薄页/非 html/失败页不占名额)——链接农场的广度消耗由 7 分钟总超时/ +并发 ≤2/频控三层兜底(预算语义记档)。 """ from __future__ import annotations diff --git a/app/platform/site_import/task.py b/app/platform/site_import/task.py index f3ecd68..35ca14a 100644 --- a/app/platform/site_import/task.py +++ b/app/platform/site_import/task.py @@ -20,6 +20,7 @@ from collections import defaultdict from app.platform.site_import import crawler, generator from app.platform.site_import.crawler import CrawlError, INSUFFICIENT_MESSAGE from app.platform.site_import.generator import GenerateError +from app.platform.site_import.safe_fetch import FetchError TASK_TTL = 30 * 60 TOTAL_TIMEOUT = 7 * 60 @@ -89,8 +90,8 @@ class ImportTask: """一条导入任务:状态机 + 阶段产物 + 强引用的后台 aio task。 failed_phase:"generate" 可 retry=generate 复活(复用 crawl_result, - 不重抓);"crawl"(SPA/内容不足/抓取失败)与 None(超时落在抓取段/ - 未知异常)不可复活——重开任务。 + 不重抓);"crawl"(SPA/内容不足/FetchError 网站打不开)与 None(超时 + 落在抓取段/未知异常)不可复活——重开任务。 """ def __init__(self, slug: str, task_id: str, url: str | None) -> None: @@ -201,6 +202,10 @@ async def _pipeline(task_obj: ImportTask) -> None: phase="generate" if task_obj.crawl_result is not None else None) except CrawlError as exc: task_obj.fail(str(exc), phase="crawl") # SPA/内容不足——不可 retry + except FetchError: + # spec §6 行1:入口页不可达/超时是最高频真实失败路径——白话文案, + # 不漏进通用兜底裸抛英文内部信息 + task_obj.fail("网站打不开或响应太慢——检查 URL", phase="crawl") except GenerateError as exc: task_obj.fail(str(exc), phase="generate") # retry=generate 复活 except Exception as exc: # 防御:未知异常不裸抛(任务表对象完整性优先) diff --git a/tests/test_site_import_task.py b/tests/test_site_import_task.py index 7fa3597..1292388 100644 --- a/tests/test_site_import_task.py +++ b/tests/test_site_import_task.py @@ -433,6 +433,18 @@ class TestTasks: with pytest.raises(ValueError): await task.start("t1", None, retry_task_id=tid_b) + async def test_FetchError白话化映射(self, monkeypatch): + # spec §6 行1:站点不可达/超时是最高频真实失败路径——不漏进 + # 通用异常兜底裸抛英文内部信息 + stubs = _Stubs().install(monkeypatch) + stubs.crawl_errors.append(safe_fetch.FetchError( + "fetch failed for 'https://a.com/': timed out")) + tid = await task.start("t1", "https://a.com/") + t = await _wait("t1", tid) + assert t.state == "failed" + assert t.error == "网站打不开或响应太慢——检查 URL" + assert t.failed_phase == "crawl" + async def test_30分钟惰性清理(self, monkeypatch): _Stubs().install(monkeypatch) tid = await task.start("t1", "https://a.com/")