浅念
e1e97dac2c
_pipeline 增 FetchError 分支（先于通用异常兜底）：入口页不可达/超时—— 最高频真实失败路径——映射白话文案「网站打不开或响应太慢——检查 URL」 + phase=crawl（不可 retry）；ImportTask docstring 的抓取失败归档口径 同步对齐。crawler 模块 docstring 注记 MAX_PAGES 预算语义（只计成功入编 页，链接农场广度消耗由总超时/并发/频控三层兜底）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
494 lines
21 KiB
Python
494 lines
21 KiB
Python
"""site_import 任务编排与内存任务表（Task 6）。
|
||
|
||
crawler 段全打桩 safe_fetch.fetch（构造真 httpx.Response，不连网）；
|
||
task 段打桩 crawler.crawl + generator 三函数与 make_provider。频控/清理
|
||
的时间推进照 test_auth.py 先例直改内部计数或 created_at。
|
||
"""
|
||
import asyncio
|
||
import base64
|
||
import weakref
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.platform.site_import import crawler, generator, safe_fetch, task
|
||
|
||
|
||
def _resp(url, html, status=200, content_type="text/html; charset=utf-8"):
|
||
return httpx.Response(
|
||
status, text=html, headers={"content-type": content_type},
|
||
request=httpx.Request("GET", url))
|
||
|
||
|
||
def _bin_resp(url, data, content_type):
|
||
return httpx.Response(
|
||
200, content=data, headers={"content-type": content_type},
|
||
request=httpx.Request("GET", url))
|
||
|
||
|
||
def _fake_fetch(pages: dict, log: list | None = None):
|
||
"""url → Response 桩；未登记 url 抛 FetchError（子页 404 形态）。"""
|
||
async def fetch(url, timeout=10.0):
|
||
if log is not None:
|
||
log.append((url, timeout))
|
||
if url not in pages:
|
||
raise safe_fetch.FetchError(f"fetch failed for {url!r}: 404")
|
||
return pages[url]
|
||
return fetch
|
||
|
||
|
||
def _page(text: str) -> str:
|
||
return f"<!doctype html><html><body><p>{text}</p></body></html>"
|
||
|
||
|
||
def _fat(marker: str, repeat: int = 40) -> str:
|
||
return _page(f"{marker} " + "guardrail highway safety manufacturer " * repeat)
|
||
|
||
|
||
PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 64
|
||
JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"\x00" * 64
|
||
|
||
HOME_FULL = """<!doctype html>
|
||
<html><head><title>Acme</title>
|
||
<link rel="stylesheet" href="/style.css">
|
||
<style>body{color:#fff}</style></head>
|
||
<body>
|
||
<header><nav><a href="/">Home</a><a href="/about">About us</a>
|
||
<a href="/products">Products</a></nav></header>
|
||
<img src="/uploads/2024/05/logo-acme.png" alt="logo">
|
||
<a href="https://wa.me/+8613800138000">WhatsApp</a>
|
||
<a href="mailto:sales@acme.com">Mail</a>
|
||
<p style="color:#aa1122">Acme builds guardrail systems for highways since 1999.</p>
|
||
<p>Acme manufactures guardrail, safety barriers and crash cushions for
|
||
highway contractors worldwide. Certified production, galvanized finish,
|
||
custom lengths available for export projects. Acme manufactures guardrail,
|
||
safety barriers and crash cushions for highway contractors worldwide.
|
||
Certified production, galvanized finish, custom lengths available.</p>
|
||
<footer>copyright noise</footer>
|
||
<script>var x=1;</script>
|
||
</body></html>"""
|
||
|
||
CSS_FULL = "body{color:#bb3344} a{color:#bb3344} h1{color:#bb3344}"
|
||
|
||
|
||
def _full_site_pages() -> dict:
|
||
base = "https://acme.com"
|
||
return {
|
||
f"{base}/": _resp(f"{base}/", HOME_FULL),
|
||
f"{base}/about": _resp(f"{base}/about", _fat("About us marker")),
|
||
f"{base}/products": _resp(f"{base}/products", _fat("Products marker")),
|
||
f"{base}/style.css": _resp(f"{base}/style.css", CSS_FULL,
|
||
content_type="text/css; charset=utf-8"),
|
||
f"{base}/uploads/2024/05/logo-acme.png": _bin_resp(
|
||
f"{base}/uploads/2024/05/logo-acme.png", PNG_BYTES, "image/png"),
|
||
}
|
||
|
||
|
||
class TestCrawl:
|
||
|
||
async def test_首页关于产品页全抓并清洗(self, monkeypatch):
|
||
pages = _full_site_pages()
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages))
|
||
result = await crawler.crawl("https://acme.com/")
|
||
# 三页正文都在，script/footer 噪音不在
|
||
assert "About us marker" in result["pages_text"]
|
||
assert "Products marker" in result["pages_text"]
|
||
assert "Acme builds guardrail" in result["pages_text"]
|
||
assert "var x=1" not in result["pages_text"]
|
||
assert "copyright noise" not in result["pages_text"]
|
||
# CSS #bb3344×3 压过内联 #aa1122×1；联系方式/logo 就位
|
||
assert result["color"] == "#bb3344"
|
||
assert result["whatsapp"] == "+8613800138000"
|
||
assert result["email"] == "sales@acme.com"
|
||
assert result["logo_data_url"].startswith("data:image/png;base64,")
|
||
decoded = base64.b64decode(result["logo_data_url"].split(",", 1)[1])
|
||
assert decoded == PNG_BYTES
|
||
assert result["pages_fetched"] == 3
|
||
assert result["warnings"] == []
|
||
|
||
async def test_深度与页数上限(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
links = "".join(f'<a href="/p{i}">page {i}</a>' for i in range(15))
|
||
home = _page("Acme industrial guardrail home. "
|
||
+ "highway safety barrier manufacturer " * 20 + links)
|
||
pages = {f"{base}/": _resp(f"{base}/", home)}
|
||
for i in range(15):
|
||
pages[f"{base}/p{i}"] = _resp(f"{base}/p{i}", _fat(f"P{i} marker"))
|
||
log: list = []
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages, log))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["pages_fetched"] == crawler.MAX_PAGES
|
||
assert len(log) == crawler.MAX_PAGES # 入口 + 9 个子页，其余不再抓
|
||
assert "P8 marker" in result["pages_text"]
|
||
assert "P9 marker" not in result["pages_text"]
|
||
|
||
async def test_薄子页跳过不失败(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
home = _page("Acme guardrail home. "
|
||
+ "highway safety barrier manufacturer " * 20
|
||
+ ' <a href="/about">about</a><a href="/tiny">tiny</a>')
|
||
pages = {
|
||
f"{base}/": _resp(f"{base}/", home),
|
||
f"{base}/about": _resp(f"{base}/about", _fat("About marker")),
|
||
f"{base}/tiny": _resp(f"{base}/tiny", _page("thin page")),
|
||
}
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert "About marker" in result["pages_text"]
|
||
assert "thin page" not in result["pages_text"]
|
||
assert result["pages_fetched"] == 2
|
||
assert any("薄" in w for w in result["warnings"])
|
||
|
||
async def test_入口页SPA空壳抛CrawlError(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
pages = {f"{base}/": _resp(f"{base}/", _page("bare shell"))}
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages))
|
||
with pytest.raises(crawler.CrawlError) as ei:
|
||
await crawler.crawl(f"{base}/")
|
||
assert "浏览器渲染" in str(ei.value)
|
||
|
||
async def test_内容不足_总文本1K以下抛CrawlError(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
# 清洗后 ≥200（过 SPA 门）但总文本 <1K（无子页）
|
||
entry = _page("Acme guardrail maker. " * 12)
|
||
assert 200 <= len(entry.split("<p>")[1].split("</p>")[0]) < 1000
|
||
pages = {f"{base}/": _resp(f"{base}/", entry)}
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages))
|
||
with pytest.raises(crawler.CrawlError) as ei:
|
||
await crawler.crawl(f"{base}/")
|
||
assert str(ei.value) == crawler.INSUFFICIENT_MESSAGE
|
||
|
||
async def test_入口页非html抛CrawlError(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
pages = {f"{base}/": _resp(f"{base}/", "%PDF-1.7 fake",
|
||
content_type="application/pdf")}
|
||
log: list = []
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages, log))
|
||
with pytest.raises(crawler.CrawlError) as ei:
|
||
await crawler.crawl(f"{base}/")
|
||
assert str(ei.value) == crawler.INSUFFICIENT_MESSAGE
|
||
assert len(log) == 1 # 零子页追抓
|
||
|
||
async def test_logo下载_格式大小校验(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
fat_home = _page(
|
||
'<img src="/logo.png" alt="logo"> '
|
||
+ "Acme industrial guardrail manufacturer. " * 60)
|
||
|
||
def _pages(logo_resp) -> dict:
|
||
return {
|
||
f"{base}/": _resp(f"{base}/", fat_home),
|
||
f"{base}/logo.png": logo_resp,
|
||
}
|
||
|
||
# png 通过 → data URL
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(_pages(
|
||
_bin_resp(f"{base}/logo.png", PNG_BYTES, "image/png"))))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["logo_data_url"].startswith("data:image/png;base64,")
|
||
|
||
# svg → None + warning
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(_pages(
|
||
_bin_resp(f"{base}/logo.png", b"<svg xmlns='x'></svg>",
|
||
"image/svg+xml"))))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["logo_data_url"] is None
|
||
assert any("logo" in w and "格式" in w for w in result["warnings"])
|
||
|
||
# >2MB → None + warning
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(_pages(
|
||
_bin_resp(f"{base}/logo.png",
|
||
PNG_BYTES + b"\x00" * (2 * 1024 * 1024), "image/png"))))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["logo_data_url"] is None
|
||
assert any("logo" in w and "2MB" in w for w in result["warnings"])
|
||
|
||
# Content-Type 与魔数不符（头谎报 png 实为 jpeg）→ None + warning
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(_pages(
|
||
_bin_resp(f"{base}/logo.png", JPEG_BYTES, "image/png"))))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["logo_data_url"] is None
|
||
assert any("logo" in w for w in result["warnings"])
|
||
|
||
async def test_默认色回退(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
pages = {f"{base}/": _resp(f"{base}/", _fat("Acme home", 60))}
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", _fake_fetch(pages))
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["color"] == crawler.DEFAULT_COLOR
|
||
|
||
async def test_CSS追抓15s超时不阻塞(self, monkeypatch):
|
||
base = "https://acme.com"
|
||
broken_css = dict(_full_site_pages())
|
||
broken_css[f"{base}/style.css"] = _resp(
|
||
f"{base}/style.css", "", content_type="text/css; charset=utf-8")
|
||
|
||
async def fetch(url, timeout=10.0):
|
||
if url == f"{base}/style.css":
|
||
raise safe_fetch.FetchError(
|
||
f"fetch failed for {url!r}: timed out")
|
||
return broken_css[url]
|
||
|
||
log: list = []
|
||
original = fetch
|
||
|
||
async def fetch_logged(url, timeout=10.0):
|
||
log.append((url, timeout))
|
||
return await original(url, timeout)
|
||
|
||
monkeypatch.setattr(crawler.safe_fetch, "fetch", fetch_logged)
|
||
# CSS 失败不阻塞：抓取完成，内联色兜底生效
|
||
result = await crawler.crawl(f"{base}/")
|
||
assert result["color"] == "#aa1122"
|
||
# CSS 追抓用的是 15s 资产超时（页 10s）
|
||
assert (f"{base}/style.css", crawler.ASSET_TIMEOUT) in log
|
||
|
||
|
||
# ---- task 段 ---------------------------------------------------------------
|
||
|
||
_DEFAULT_CRAWL = {
|
||
"pages_text": "Acme guardrail manufacturer since 1999 certified. " * 40,
|
||
"color": "#123456",
|
||
"whatsapp": "+8613800138000",
|
||
"email": "sales@acme.com",
|
||
"logo_data_url": None,
|
||
"warnings": [],
|
||
"pages_fetched": 3,
|
||
}
|
||
|
||
|
||
class _Stubs:
|
||
"""crawler/generator 全量确定性桩 + 调用计数。
|
||
|
||
*_errors 为异常队列出列（首错只撞一次），delay 模拟慢阶段。
|
||
"""
|
||
|
||
def __init__(self):
|
||
self.calls = {"crawl": 0, "analyze": 0, "generate": 0, "build": 0}
|
||
self.crawl_result = dict(_DEFAULT_CRAWL)
|
||
self.analysis = {"company_name": "Acme",
|
||
"product_lines": [{"name": "guardrail"}]}
|
||
self.gen_out = {"身份描述": "外贸销售"}
|
||
self.draft = {"brand": {"title": "Acme", "color": "#123456"}}
|
||
self.crawl_errors: list = []
|
||
self.generate_errors: list = []
|
||
self.delay = 0.0
|
||
|
||
async def _slow(self):
|
||
if self.delay:
|
||
await asyncio.sleep(self.delay)
|
||
|
||
async def crawl_(self, url):
|
||
self.calls["crawl"] += 1
|
||
await self._slow()
|
||
if self.crawl_errors:
|
||
raise self.crawl_errors.pop(0)
|
||
return dict(self.crawl_result)
|
||
|
||
async def analyze_(self, provider, source):
|
||
self.calls["analyze"] += 1
|
||
await self._slow()
|
||
return dict(self.analysis)
|
||
|
||
async def generate_(self, provider, analysis):
|
||
self.calls["generate"] += 1
|
||
await self._slow()
|
||
if self.generate_errors:
|
||
raise self.generate_errors.pop(0)
|
||
return dict(self.gen_out)
|
||
|
||
def build_(self, gen, analysis, crawl, source):
|
||
self.calls["build"] += 1
|
||
assert "pages_text" in crawl # build 消费 crawl_result 原形
|
||
return dict(self.draft)
|
||
|
||
def install(self, monkeypatch):
|
||
monkeypatch.setattr(crawler, "crawl", self.crawl_)
|
||
monkeypatch.setattr(generator, "analyze", self.analyze_)
|
||
monkeypatch.setattr(generator, "generate", self.generate_)
|
||
monkeypatch.setattr(generator, "build", self.build_)
|
||
monkeypatch.setattr(generator, "make_provider",
|
||
lambda config_data=None: object())
|
||
return self
|
||
|
||
|
||
async def _wait(slug: str, tid: str) -> "task.ImportTask":
|
||
t = task.get(slug, tid)
|
||
assert t is not None
|
||
await t._aio_task
|
||
return t
|
||
|
||
|
||
class TestTasks:
|
||
|
||
async def test_状态机流转到done(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
tid = await task.start("t1", "https://acme.com/")
|
||
t = await _wait("t1", tid)
|
||
assert t.state == "done"
|
||
assert t.draft == stubs.draft
|
||
assert t.error is None
|
||
assert t.pages_fetched == 3
|
||
assert t.finished_at is not None
|
||
assert stubs.calls == {"crawl": 1, "analyze": 1, "generate": 1,
|
||
"build": 1}
|
||
|
||
async def test_slug与taskId联合索引_跨租户查404(self, monkeypatch):
|
||
_Stubs().install(monkeypatch)
|
||
tid = await task.start("t1", "https://acme.com/")
|
||
await _wait("t1", tid)
|
||
assert task.get("t1", tid) is not None
|
||
assert task.get("other", tid) is None
|
||
|
||
async def test_全平台并发2_第三个拒绝(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
stubs.delay = 0.3
|
||
ta = await task.start("t1", "https://a.com/")
|
||
tb = await task.start("t2", "https://b.com/") # 全平台（跨租户同算）
|
||
assert task.active_count() == 2
|
||
with pytest.raises(task.ConcurrentLimit):
|
||
await task.start("t3", "https://c.com/")
|
||
await _wait("t1", ta)
|
||
await _wait("t2", tb)
|
||
|
||
async def test_频控3次每15分钟_第4次429(self, monkeypatch):
|
||
_Stubs().install(monkeypatch)
|
||
for i in range(3):
|
||
tid = await task.start("t1", f"https://a.com/{i}")
|
||
await _wait("t1", tid) # 顺序完成，不撞并发上限
|
||
with pytest.raises(task.ImportRateLimited) as ei:
|
||
await task.start("t1", "https://a.com/again")
|
||
assert "3 次" in str(ei.value)
|
||
|
||
async def test_频控新建与retry同计(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
stubs.generate_errors.append(
|
||
generator.GenerateError("内容生成：输出无法解析"))
|
||
tid = await task.start("t1", "https://a.com/")
|
||
await _wait("t1", tid) # 计数 1（新建）
|
||
tid2 = await task.start("t1", None, retry_task_id=tid)
|
||
assert tid2 == tid
|
||
await _wait("t1", tid) # 计数 2（retry 同计）
|
||
tid_b = await task.start("t1", "https://a.com/b")
|
||
await _wait("t1", tid_b) # 计数 3
|
||
with pytest.raises(task.ImportRateLimited):
|
||
await task.start("t1", "https://a.com/c")
|
||
|
||
async def test_7分钟总超时强failed(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
monkeypatch.setattr(task, "TOTAL_TIMEOUT", 0.05)
|
||
# 超时落在抓取段：无产物 → 不可 retry
|
||
stubs.delay = 1.0
|
||
tid = await task.start("t1", "https://a.com/")
|
||
t = await _wait("t1", tid)
|
||
assert t.state == "failed"
|
||
assert "导入超时" in t.error
|
||
assert t.failed_phase is None
|
||
# 超时落在生成段（抓取产物已在）→ 标 generate，retry=generate 复活
|
||
stubs.delay = 0.0
|
||
stubs.crawl_result["pages_fetched"] = 5
|
||
async def slow_analyze(provider, source):
|
||
await asyncio.sleep(1.0)
|
||
return {"product_lines": [{"name": "x"}]}
|
||
monkeypatch.setattr(generator, "analyze", slow_analyze)
|
||
tid2 = await task.start("t1", "https://b.com/")
|
||
t2 = await _wait("t1", tid2)
|
||
assert t2.state == "failed"
|
||
assert "导入超时" in t2.error
|
||
assert t2.failed_phase == "generate"
|
||
|
||
async def test_generate失败可retry同task复活且不重抓(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
stubs.generate_errors.append(
|
||
generator.GenerateError("内容生成：AI 调用超时（120s）"))
|
||
tid = await task.start("t1", "https://a.com/")
|
||
t = await _wait("t1", tid)
|
||
assert t.state == "failed"
|
||
assert t.failed_phase == "generate"
|
||
assert stubs.calls["crawl"] == 1
|
||
tid2 = await task.start("t1", None, retry_task_id=tid)
|
||
assert tid2 == tid
|
||
t = await _wait("t1", tid2)
|
||
assert t.state == "done"
|
||
assert t.draft == stubs.draft
|
||
assert stubs.calls["crawl"] == 1 # 未重抓
|
||
assert stubs.calls["generate"] == 2
|
||
assert stubs.calls["build"] == 1
|
||
|
||
async def test_retry校验_不存在404与非法态409(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
with pytest.raises(LookupError):
|
||
await task.start("t1", None, retry_task_id="nope")
|
||
tid = await task.start("t1", "https://a.com/")
|
||
await _wait("t1", tid)
|
||
with pytest.raises(ValueError): # done 态不可 retry
|
||
await task.start("t1", None, retry_task_id=tid)
|
||
# 跨租户 retry 他人 task → LookupError（404 语义）
|
||
with pytest.raises(LookupError):
|
||
await task.start("t2", None, retry_task_id=tid)
|
||
# crawl 失败态（不可复活）→ ValueError
|
||
stubs.crawl_errors.append(crawler.CrawlError("抓取失败"))
|
||
tid_b = await task.start("t1", "https://b.com/")
|
||
await _wait("t1", tid_b)
|
||
with pytest.raises(ValueError):
|
||
await task.start("t1", None, retry_task_id=tid_b)
|
||
|
||
async def test_FetchError白话化映射(self, monkeypatch):
|
||
# spec §6 行1：站点不可达/超时是最高频真实失败路径——不漏进
|
||
# 通用异常兜底裸抛英文内部信息
|
||
stubs = _Stubs().install(monkeypatch)
|
||
stubs.crawl_errors.append(safe_fetch.FetchError(
|
||
"fetch failed for 'https://a.com/': timed out"))
|
||
tid = await task.start("t1", "https://a.com/")
|
||
t = await _wait("t1", tid)
|
||
assert t.state == "failed"
|
||
assert t.error == "网站打不开或响应太慢——检查 URL"
|
||
assert t.failed_phase == "crawl"
|
||
|
||
async def test_30分钟惰性清理(self, monkeypatch):
|
||
_Stubs().install(monkeypatch)
|
||
tid = await task.start("t1", "https://a.com/")
|
||
t = await _wait("t1", tid)
|
||
assert task.get("t1", tid) is not None
|
||
t.created_at -= task.TASK_TTL + 1 # 拨到 30 分钟窗外（免真实等待）
|
||
assert task.get("t1", tid) is None
|
||
assert ("t1", tid) not in task._tasks
|
||
# 新建路径同样顺带扫
|
||
tid2 = await task.start("t1", "https://b.com/")
|
||
await _wait("t1", tid2)
|
||
assert ("t1", tid) not in task._tasks
|
||
|
||
async def test_强引用_任务对象在表内(self, monkeypatch):
|
||
stubs = _Stubs().install(monkeypatch)
|
||
stubs.delay = 0.3
|
||
tid = await task.start("t1", "https://a.com/")
|
||
t = task.get("t1", tid)
|
||
ref = weakref.ref(t)
|
||
assert task.get("t1", tid) is t
|
||
assert ref() is not None # 表内强引用存活（未被 GC）
|
||
assert isinstance(t._aio_task, asyncio.Task)
|
||
await t._aio_task
|
||
|
||
async def test_内容不足三触发映射失败文案(self, monkeypatch):
|
||
# 触发 1：入口页非 text/html（crawler 层零页 → CrawlError）
|
||
stubs = _Stubs().install(monkeypatch)
|
||
stubs.crawl_errors.append(
|
||
crawler.CrawlError(crawler.INSUFFICIENT_MESSAGE))
|
||
tid = await task.start("t1", "https://a.com/")
|
||
t = await _wait("t1", tid)
|
||
assert t.state == "failed"
|
||
assert t.error == crawler.INSUFFICIENT_MESSAGE
|
||
assert t.failed_phase == "crawl" # 内容不足类不可 retry
|
||
# 触发 2：SPA 空壳（专属文案）
|
||
stubs.crawl_errors.append(crawler.CrawlError(crawler.SPA_MESSAGE))
|
||
tid = await task.start("t1", "https://b.com/")
|
||
t = await _wait("t1", tid)
|
||
assert "浏览器渲染" in t.error
|
||
assert t.failed_phase == "crawl"
|
||
# 触发 3：调用 1 产品线为空（任务层映射内容不足文案）
|
||
stubs.analysis = {"company_name": "Acme", "product_lines": []}
|
||
tid = await task.start("t1", "https://c.com/")
|
||
t = await _wait("t1", tid)
|
||
assert t.error == crawler.INSUFFICIENT_MESSAGE
|
||
assert t.failed_phase == "crawl"
|