浅念
8f2a73c5d4
crawler：BFS ≤10 页（深度 ≤2）+ 入口双重门（非 html 零页/SPA 空壳）+ 薄子页跳过 + CSS/logo 15s 追抓（MIME+魔数双验 ≤2MB → base64 data URL） + 总文本 <1K 内容不足 + 主色默认 #066fc0 回退；logo 相对路径对入口绝对化。 task：(slug, task_id) 联合索引内存表 + 30 分钟惰性清理 + 强引用防 GC + 全平台并发 ≤2（复活同占名额）+ 频控 3 次/15min（新建与 retry 同计， SlidingWindowCounter 同款独立实例）+ 7 分钟总超时兜底（生成段超时标 generate 可 retry=generate 复活不重抓）+ 内容不足三触发失败文案映射。 conftest 增 autouse 复位夹具（任务表/频控单例跨测试隔离）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
192 lines
7.3 KiB
Python
192 lines
7.3 KiB
Python
"""site_import 抓取编排（spec §3.1/§3.3 的 crawl 段）。
|
||
|
||
BFS 选页 ≤10（深度 ≤2）→ safe_fetch 逐页 → extract 清洗抽取 →
|
||
CSS/logo 追抓（15s 超时）→ 组装 crawl_result dict 供生成器。
|
||
|
||
入口页双重门（spec §3.1/§6）：非 text/html → 零页内容不足；清洗后
|
||
<200 字符 → SPA 空壳（子页过薄只跳过不失败——JS 渲染的产品子页常见，
|
||
整站不该陪葬）。单页 >500KB 截断解析（clean_html 未闭合块二遍清洗兜底）。
|
||
discover_links 会回吐入口自身（href="#"/"/"）——visited 预置 start URL
|
||
去重，防浪费 ≤10 页名额（Task 3 备忘）。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import re
|
||
from collections import deque
|
||
from urllib.parse import urljoin, urlsplit, urlunsplit
|
||
|
||
from app.platform.site_import import safe_fetch
|
||
from app.platform.site_import.extract import (
|
||
clean_html, discover_links, extract_color, extract_contacts,
|
||
extract_logo_url, is_thin,
|
||
)
|
||
from app.platform.site_import.safe_fetch import FetchError
|
||
|
||
MAX_PAGES = 10
|
||
MAX_DEPTH = 2
|
||
DEFAULT_COLOR = "#066fc0" # extract_color None 的函数层不兜底原则（Task 3 备忘）
|
||
PAGE_TIMEOUT = 10.0
|
||
ASSET_TIMEOUT = 15.0
|
||
MAX_PAGE_CHARS = 500 * 1024 # 单页 500KB 上限（截断解析）
|
||
LOGO_MAX_BYTES = 2 * 1024 * 1024
|
||
LOGO_OK_MIME = {"image/png", "image/jpeg", "image/webp"}
|
||
INSUFFICIENT_TOTAL_CHARS = 1000
|
||
|
||
SPA_MESSAGE = "该网站需要浏览器渲染才能看到内容，暂不支持自动导入——请在配置中心手动配置"
|
||
INSUFFICIENT_MESSAGE = "网站内容不足以生成配置，请在配置中心手动配置"
|
||
|
||
# 首个外链样式表发现（link 标签内 rel 含 stylesheet → 提取 href）
|
||
_RE_LINK_TAG = re.compile(r"<link\b[^>]*>", re.IGNORECASE)
|
||
_RE_REL_STYLESHEET = re.compile(r"""rel\s*=\s*["']?\s*stylesheet\b""",
|
||
re.IGNORECASE)
|
||
_RE_HREF_IN_TAG = re.compile(r"""href\s*=\s*["']([^"']+)["']""",
|
||
re.IGNORECASE)
|
||
|
||
|
||
class CrawlError(Exception):
|
||
"""抓取阶段不可恢复失败（SPA 空壳/内容不足）——phase=crawl，不可 retry。"""
|
||
|
||
|
||
def _sniff_image(data: bytes) -> str | None:
|
||
"""魔数指纹（与 Content-Type 双验——防伪造头）。"""
|
||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||
return "image/png"
|
||
if data.startswith(b"\xff\xd8\xff"):
|
||
return "image/jpeg"
|
||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||
return "image/webp"
|
||
return None
|
||
|
||
|
||
def _normalize(url: str) -> str:
|
||
"""同源同径去重键（与 discover_links 的清洗口径对齐：去查询去锚点）。"""
|
||
parts = urlsplit(url)
|
||
return urlunsplit((parts.scheme.lower(), parts.netloc.lower(),
|
||
parts.path, "", ""))
|
||
|
||
|
||
def _html_text(resp) -> str | None:
|
||
"""Content-Type 非 text/html → None（调用方跳页/判零页）；500KB 截断。"""
|
||
ct = resp.headers.get("content-type", "").lower()
|
||
if "text/html" not in ct:
|
||
return None
|
||
return resp.text[:MAX_PAGE_CHARS]
|
||
|
||
|
||
def _first_css_url(base_url: str, html_text: str) -> str | None:
|
||
for m in _RE_LINK_TAG.finditer(html_text):
|
||
tag = m.group(0)
|
||
if _RE_REL_STYLESHEET.search(tag):
|
||
hm = _RE_HREF_IN_TAG.search(tag)
|
||
if hm:
|
||
return urljoin(base_url, hm.group(1).strip())
|
||
return None
|
||
|
||
|
||
async def _fetch_logo(url: str, warnings: list[str]) -> str | None:
|
||
"""logo 下载校验（spec §3.1）：MIME 白名单 + 魔数双验 + ≤2MB →
|
||
base64 data URL；不合格 → None + warning 注记（审核页无头像预览，
|
||
保存链路头像项空置——零新后端端点）。"""
|
||
try:
|
||
resp = await safe_fetch.fetch(url, ASSET_TIMEOUT)
|
||
except FetchError:
|
||
warnings.append("logo 下载失败，已跳过头像")
|
||
return None
|
||
data = resp.content
|
||
if len(data) > LOGO_MAX_BYTES:
|
||
warnings.append("logo 超过 2MB 上限，已跳过头像")
|
||
return None
|
||
mime = resp.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||
sniffed = _sniff_image(data)
|
||
if mime not in LOGO_OK_MIME or sniffed is None or sniffed != mime:
|
||
warnings.append("logo 为不支持的格式，已跳过头像")
|
||
return None
|
||
return f"data:{sniffed};base64," + base64.b64encode(data).decode("ascii")
|
||
|
||
|
||
async def crawl(start_url: str) -> dict:
|
||
"""整站抓取编排，返回 crawl_result dict。
|
||
|
||
FetchError 不在此包装——交任务层 fail（白话化 + phase 归档）。
|
||
"""
|
||
warnings: list[str] = []
|
||
|
||
# 1) 入口页：非 text/html → 零页（内容不足三触发之一）
|
||
entry = await safe_fetch.fetch(start_url, PAGE_TIMEOUT)
|
||
entry_html = _html_text(entry)
|
||
if entry_html is None:
|
||
raise CrawlError(INSUFFICIENT_MESSAGE)
|
||
entry_clean = clean_html(entry_html)
|
||
if is_thin(entry_clean):
|
||
raise CrawlError(SPA_MESSAGE)
|
||
|
||
# 2) BFS：入口 depth 0 → 子页 ≤MAX_DEPTH；关键词优先序即抓取优先级
|
||
pages_text = entry_clean
|
||
htmls = [entry_html]
|
||
fetched = 1
|
||
visited = {_normalize(start_url)} # discover_links 含入口自身——预置去重
|
||
queue: deque[tuple[str, int]] = deque(
|
||
(u, 1) for u in discover_links(start_url, entry_html))
|
||
|
||
while queue and fetched < MAX_PAGES:
|
||
url, depth = queue.popleft()
|
||
norm = _normalize(url)
|
||
if norm in visited:
|
||
continue
|
||
visited.add(norm)
|
||
try:
|
||
resp = await safe_fetch.fetch(url, PAGE_TIMEOUT)
|
||
except FetchError:
|
||
warnings.append(f"子页抓取失败已跳过：{url}")
|
||
continue
|
||
html = _html_text(resp)
|
||
if html is None:
|
||
warnings.append(f"子页非 HTML 已跳过：{url}")
|
||
continue
|
||
clean = clean_html(html)
|
||
if is_thin(clean):
|
||
warnings.append(f"子页内容过薄已跳过：{url}")
|
||
continue
|
||
pages_text += "\n" + clean
|
||
htmls.append(html)
|
||
fetched += 1
|
||
if depth < MAX_DEPTH:
|
||
queue.extend((u, depth + 1) for u in discover_links(url, html))
|
||
|
||
# 3) 总文本 <1K（内容不足三触发之一）
|
||
if len(pages_text) < INSUFFICIENT_TOTAL_CHARS:
|
||
raise CrawlError(INSUFFICIENT_MESSAGE)
|
||
|
||
# 4) 主色：内联 style（全页 html 集）+ 首个外链 CSS（15s 追抓，失败
|
||
# 静默回退内联/默认色——样式表属增益项不陪葬）
|
||
css_text: str | None = None
|
||
css_url = _first_css_url(start_url, entry_html)
|
||
if css_url:
|
||
try:
|
||
css_resp = await safe_fetch.fetch(css_url, ASSET_TIMEOUT)
|
||
css_text = css_resp.text[:MAX_PAGE_CHARS]
|
||
except FetchError:
|
||
pass
|
||
combined_html = "\n".join(htmls)
|
||
color = extract_color(combined_html, css_text) or DEFAULT_COLOR
|
||
|
||
contacts = extract_contacts(combined_html)
|
||
|
||
logo_data_url = None
|
||
logo_url = extract_logo_url(combined_html)
|
||
if logo_url:
|
||
# src 可能是相对路径（WordPress uploads 常见）——对入口页绝对化
|
||
logo_data_url = await _fetch_logo(urljoin(start_url, logo_url),
|
||
warnings)
|
||
|
||
return {
|
||
"pages_text": pages_text,
|
||
"color": color,
|
||
"whatsapp": contacts["whatsapp"],
|
||
"email": contacts["email"],
|
||
"logo_data_url": logo_data_url,
|
||
"warnings": warnings,
|
||
"pages_fetched": fetched,
|
||
}
|