feat(site-import): 抓取编排与内存任务表
crawler：BFS ≤10 页（深度 ≤2）+ 入口双重门（非 html 零页/SPA 空壳）+ 薄子页跳过 + CSS/logo 15s 追抓（MIME+魔数双验 ≤2MB → base64 data URL） + 总文本 <1K 内容不足 + 主色默认 #066fc0 回退；logo 相对路径对入口绝对化。 task：(slug, task_id) 联合索引内存表 + 30 分钟惰性清理 + 强引用防 GC + 全平台并发 ≤2（复活同占名额）+ 频控 3 次/15min（新建与 retry 同计， SlidingWindowCounter 同款独立实例）+ 7 分钟总超时兜底（生成段超时标 generate 可 retry=generate 复活不重抓）+ 内容不足三触发失败文案映射。 conftest 增 autouse 复位夹具（任务表/频控单例跨测试隔离）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,191 @@
|
||||
"""site_import 抓取编排（spec §3.1/§3.3 的 crawl 段）。
|
||||
|
||||
BFS 选页 ≤10（深度 ≤2）→ safe_fetch 逐页 → extract 清洗抽取 →
|
||||
CSS/logo 追抓（15s 超时）→ 组装 crawl_result dict 供生成器。
|
||||
|
||||
入口页双重门（spec §3.1/§6）：非 text/html → 零页内容不足；清洗后
|
||||
<200 字符 → SPA 空壳（子页过薄只跳过不失败——JS 渲染的产品子页常见，
|
||||
整站不该陪葬）。单页 >500KB 截断解析（clean_html 未闭合块二遍清洗兜底）。
|
||||
discover_links 会回吐入口自身（href="#"/"/"）——visited 预置 start URL
|
||||
去重，防浪费 ≤10 页名额（Task 3 备忘）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
from collections import deque
|
||||
from urllib.parse import urljoin, urlsplit, urlunsplit
|
||||
|
||||
from app.platform.site_import import safe_fetch
|
||||
from app.platform.site_import.extract import (
|
||||
clean_html, discover_links, extract_color, extract_contacts,
|
||||
extract_logo_url, is_thin,
|
||||
)
|
||||
from app.platform.site_import.safe_fetch import FetchError
|
||||
|
||||
MAX_PAGES = 10
|
||||
MAX_DEPTH = 2
|
||||
DEFAULT_COLOR = "#066fc0" # extract_color None 的函数层不兜底原则（Task 3 备忘）
|
||||
PAGE_TIMEOUT = 10.0
|
||||
ASSET_TIMEOUT = 15.0
|
||||
MAX_PAGE_CHARS = 500 * 1024 # 单页 500KB 上限（截断解析）
|
||||
LOGO_MAX_BYTES = 2 * 1024 * 1024
|
||||
LOGO_OK_MIME = {"image/png", "image/jpeg", "image/webp"}
|
||||
INSUFFICIENT_TOTAL_CHARS = 1000
|
||||
|
||||
SPA_MESSAGE = "该网站需要浏览器渲染才能看到内容，暂不支持自动导入——请在配置中心手动配置"
|
||||
INSUFFICIENT_MESSAGE = "网站内容不足以生成配置，请在配置中心手动配置"
|
||||
|
||||
# 首个外链样式表发现（link 标签内 rel 含 stylesheet → 提取 href）
|
||||
_RE_LINK_TAG = re.compile(r"<link\b[^>]*>", re.IGNORECASE)
|
||||
_RE_REL_STYLESHEET = re.compile(r"""rel\s*=\s*["']?\s*stylesheet\b""",
|
||||
re.IGNORECASE)
|
||||
_RE_HREF_IN_TAG = re.compile(r"""href\s*=\s*["']([^"']+)["']""",
|
||||
re.IGNORECASE)
|
||||
|
||||
|
||||
class CrawlError(Exception):
|
||||
"""抓取阶段不可恢复失败（SPA 空壳/内容不足）——phase=crawl，不可 retry。"""
|
||||
|
||||
|
||||
def _sniff_image(data: bytes) -> str | None:
|
||||
"""魔数指纹（与 Content-Type 双验——防伪造头）。"""
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
|
||||
def _normalize(url: str) -> str:
|
||||
"""同源同径去重键（与 discover_links 的清洗口径对齐：去查询去锚点）。"""
|
||||
parts = urlsplit(url)
|
||||
return urlunsplit((parts.scheme.lower(), parts.netloc.lower(),
|
||||
parts.path, "", ""))
|
||||
|
||||
|
||||
def _html_text(resp) -> str | None:
|
||||
"""Content-Type 非 text/html → None（调用方跳页/判零页）；500KB 截断。"""
|
||||
ct = resp.headers.get("content-type", "").lower()
|
||||
if "text/html" not in ct:
|
||||
return None
|
||||
return resp.text[:MAX_PAGE_CHARS]
|
||||
|
||||
|
||||
def _first_css_url(base_url: str, html_text: str) -> str | None:
|
||||
for m in _RE_LINK_TAG.finditer(html_text):
|
||||
tag = m.group(0)
|
||||
if _RE_REL_STYLESHEET.search(tag):
|
||||
hm = _RE_HREF_IN_TAG.search(tag)
|
||||
if hm:
|
||||
return urljoin(base_url, hm.group(1).strip())
|
||||
return None
|
||||
|
||||
|
||||
async def _fetch_logo(url: str, warnings: list[str]) -> str | None:
|
||||
"""logo 下载校验（spec §3.1）：MIME 白名单 + 魔数双验 + ≤2MB →
|
||||
base64 data URL；不合格 → None + warning 注记（审核页无头像预览，
|
||||
保存链路头像项空置——零新后端端点）。"""
|
||||
try:
|
||||
resp = await safe_fetch.fetch(url, ASSET_TIMEOUT)
|
||||
except FetchError:
|
||||
warnings.append("logo 下载失败，已跳过头像")
|
||||
return None
|
||||
data = resp.content
|
||||
if len(data) > LOGO_MAX_BYTES:
|
||||
warnings.append("logo 超过 2MB 上限，已跳过头像")
|
||||
return None
|
||||
mime = resp.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
sniffed = _sniff_image(data)
|
||||
if mime not in LOGO_OK_MIME or sniffed is None or sniffed != mime:
|
||||
warnings.append("logo 为不支持的格式，已跳过头像")
|
||||
return None
|
||||
return f"data:{sniffed};base64," + base64.b64encode(data).decode("ascii")
|
||||
|
||||
|
||||
async def crawl(start_url: str) -> dict:
|
||||
"""整站抓取编排，返回 crawl_result dict。
|
||||
|
||||
FetchError 不在此包装——交任务层 fail（白话化 + phase 归档）。
|
||||
"""
|
||||
warnings: list[str] = []
|
||||
|
||||
# 1) 入口页：非 text/html → 零页（内容不足三触发之一）
|
||||
entry = await safe_fetch.fetch(start_url, PAGE_TIMEOUT)
|
||||
entry_html = _html_text(entry)
|
||||
if entry_html is None:
|
||||
raise CrawlError(INSUFFICIENT_MESSAGE)
|
||||
entry_clean = clean_html(entry_html)
|
||||
if is_thin(entry_clean):
|
||||
raise CrawlError(SPA_MESSAGE)
|
||||
|
||||
# 2) BFS：入口 depth 0 → 子页 ≤MAX_DEPTH；关键词优先序即抓取优先级
|
||||
pages_text = entry_clean
|
||||
htmls = [entry_html]
|
||||
fetched = 1
|
||||
visited = {_normalize(start_url)} # discover_links 含入口自身——预置去重
|
||||
queue: deque[tuple[str, int]] = deque(
|
||||
(u, 1) for u in discover_links(start_url, entry_html))
|
||||
|
||||
while queue and fetched < MAX_PAGES:
|
||||
url, depth = queue.popleft()
|
||||
norm = _normalize(url)
|
||||
if norm in visited:
|
||||
continue
|
||||
visited.add(norm)
|
||||
try:
|
||||
resp = await safe_fetch.fetch(url, PAGE_TIMEOUT)
|
||||
except FetchError:
|
||||
warnings.append(f"子页抓取失败已跳过：{url}")
|
||||
continue
|
||||
html = _html_text(resp)
|
||||
if html is None:
|
||||
warnings.append(f"子页非 HTML 已跳过：{url}")
|
||||
continue
|
||||
clean = clean_html(html)
|
||||
if is_thin(clean):
|
||||
warnings.append(f"子页内容过薄已跳过：{url}")
|
||||
continue
|
||||
pages_text += "\n" + clean
|
||||
htmls.append(html)
|
||||
fetched += 1
|
||||
if depth < MAX_DEPTH:
|
||||
queue.extend((u, depth + 1) for u in discover_links(url, html))
|
||||
|
||||
# 3) 总文本 <1K（内容不足三触发之一）
|
||||
if len(pages_text) < INSUFFICIENT_TOTAL_CHARS:
|
||||
raise CrawlError(INSUFFICIENT_MESSAGE)
|
||||
|
||||
# 4) 主色：内联 style（全页 html 集）+ 首个外链 CSS（15s 追抓，失败
|
||||
# 静默回退内联/默认色——样式表属增益项不陪葬）
|
||||
css_text: str | None = None
|
||||
css_url = _first_css_url(start_url, entry_html)
|
||||
if css_url:
|
||||
try:
|
||||
css_resp = await safe_fetch.fetch(css_url, ASSET_TIMEOUT)
|
||||
css_text = css_resp.text[:MAX_PAGE_CHARS]
|
||||
except FetchError:
|
||||
pass
|
||||
combined_html = "\n".join(htmls)
|
||||
color = extract_color(combined_html, css_text) or DEFAULT_COLOR
|
||||
|
||||
contacts = extract_contacts(combined_html)
|
||||
|
||||
logo_data_url = None
|
||||
logo_url = extract_logo_url(combined_html)
|
||||
if logo_url:
|
||||
# src 可能是相对路径（WordPress uploads 常见）——对入口页绝对化
|
||||
logo_data_url = await _fetch_logo(urljoin(start_url, logo_url),
|
||||
warnings)
|
||||
|
||||
return {
|
||||
"pages_text": pages_text,
|
||||
"color": color,
|
||||
"whatsapp": contacts["whatsapp"],
|
||||
"email": contacts["email"],
|
||||
"logo_data_url": logo_data_url,
|
||||
"warnings": warnings,
|
||||
"pages_fetched": fetched,
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
"""site_import 内存任务表（spec §3.3）——单进程 asyncio，(slug, task_id) 索引。
|
||||
|
||||
内存铁律：30 分钟惰性清理（查询/新建时顺带扫）；强引用防 GC（裸
|
||||
create_task 会被回收——asyncio 官方坑，模块级 dict 存 task 对象）；
|
||||
全平台并发 ≤2（复活任务占用同一名额）；频控 3 次/15 分钟（新建与
|
||||
retry 同计）。异常类型：ImportRateLimited→429 / ConcurrentLimit→409 /
|
||||
LookupError→404 / ValueError→409——API 层（Task 7）映射。
|
||||
|
||||
7 分钟总超时兜底：超时落在生成段（抓取产物已在）标 failed_phase=
|
||||
"generate"——retry=generate 是 spec §3.3 的预期恢复路径（不重抓）；
|
||||
落在抓取段则无产物，重开任务。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
|
||||
from app.platform.site_import import crawler, generator
|
||||
from app.platform.site_import.crawler import CrawlError, INSUFFICIENT_MESSAGE
|
||||
from app.platform.site_import.generator import GenerateError
|
||||
|
||||
TASK_TTL = 30 * 60
|
||||
TOTAL_TIMEOUT = 7 * 60
|
||||
MAX_CONCURRENT = 2
|
||||
IMPORT_WINDOW = 15 * 60
|
||||
IMPORT_MAX = 3
|
||||
|
||||
|
||||
class ImportRateLimited(Exception):
|
||||
"""频控超限（15 分钟 3 次）——API 层映射 429。"""
|
||||
|
||||
|
||||
class ConcurrentLimit(Exception):
|
||||
"""全平台并发上限（≤2，复活任务同占名额）——API 层映射 409。"""
|
||||
|
||||
|
||||
class _SlidingWindow:
|
||||
"""(kind, slug) 键滑动窗计数——config_routes.SlidingWindowCounter 同款
|
||||
风格的独立实例（勿共用 email/push 测试端点的预算）。同款语义：超限后
|
||||
allow 守卫先于一切预检；计数只落在真正发起的这次（校验失败不消耗）。"""
|
||||
|
||||
def __init__(self, window_seconds: float = IMPORT_WINDOW,
|
||||
max_calls: int = IMPORT_MAX) -> None:
|
||||
self.window_seconds = window_seconds
|
||||
self.max_calls = max_calls
|
||||
self._calls: dict[tuple[str, str], list[float]] = defaultdict(list)
|
||||
|
||||
def allow(self, key: tuple[str, str]) -> bool:
|
||||
self._prune(key, time.time())
|
||||
return len(self._calls.get(key, ())) < self.max_calls
|
||||
|
||||
def record(self, key: tuple[str, str]) -> None:
|
||||
now = time.time()
|
||||
self._prune(key, now)
|
||||
self._calls[key].append(now)
|
||||
|
||||
def _prune(self, key: tuple[str, str], now: float) -> None:
|
||||
calls = self._calls.get(key)
|
||||
if calls is None:
|
||||
return
|
||||
kept = [t for t in calls if t > now - self.window_seconds]
|
||||
if kept:
|
||||
self._calls[key] = kept
|
||||
else:
|
||||
self._calls.pop(key, None)
|
||||
|
||||
|
||||
# 模块级单例：导入频控（键 ("import", slug)，全平台各租户独立预算）
|
||||
_import_limiter = _SlidingWindow()
|
||||
|
||||
# 内存任务表：(slug, task_id) 联合索引——跨租户持他人 id 查询即 miss
|
||||
_tasks: dict[tuple[str, str], "ImportTask"] = {}
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
class TaskState:
|
||||
PENDING = "pending"
|
||||
CRAWLING = "crawling"
|
||||
GENERATING = "generating"
|
||||
DONE = "done"
|
||||
FAILED = "failed"
|
||||
|
||||
ACTIVE = (PENDING, CRAWLING, GENERATING)
|
||||
|
||||
|
||||
class ImportTask:
|
||||
"""一条导入任务：状态机 + 阶段产物 + 强引用的后台 aio task。
|
||||
|
||||
failed_phase："generate" 可 retry=generate 复活（复用 crawl_result，
|
||||
不重抓）；"crawl"（SPA/内容不足/抓取失败）与 None（超时落在抓取段/
|
||||
未知异常）不可复活——重开任务。
|
||||
"""
|
||||
|
||||
def __init__(self, slug: str, task_id: str, url: str | None) -> None:
|
||||
self.slug = slug
|
||||
self.task_id = task_id
|
||||
self.url = url
|
||||
self.state = TaskState.PENDING
|
||||
self.phase: str | None = None # 运行中阶段（crawling/generating）
|
||||
self.pages_fetched = 0
|
||||
self.draft: dict | None = None
|
||||
self.error: str | None = None
|
||||
self.created_at = time.time()
|
||||
self.finished_at: float | None = None
|
||||
self.failed_phase: str | None = None
|
||||
self.crawl_result: dict | None = None # retry=generate 复用
|
||||
self.need_crawl = True
|
||||
self._aio_task: asyncio.Task | None = None # 强引用防 GC
|
||||
|
||||
def fail(self, message: str, *, phase: str | None) -> None:
|
||||
self.state = TaskState.FAILED
|
||||
self.error = message
|
||||
self.failed_phase = phase
|
||||
self.phase = None
|
||||
self.finished_at = time.time()
|
||||
|
||||
def _finish(self, draft: dict) -> None:
|
||||
self.draft = draft
|
||||
self.state = TaskState.DONE
|
||||
self.phase = None
|
||||
self.finished_at = time.time()
|
||||
|
||||
|
||||
def _prune_expired(now: float | None = None) -> None:
|
||||
"""30 分钟惰性清理（查询/新建顺带扫）。运行中任务防御性取消——
|
||||
7 分钟总超时下正常不会发生（30 >> 7），仅兜异常态。"""
|
||||
now = time.time() if now is None else now
|
||||
deadline = now - TASK_TTL
|
||||
for key in [k for k, t in _tasks.items() if t.created_at < deadline]:
|
||||
stale = _tasks.pop(key)
|
||||
if stale._aio_task is not None and not stale._aio_task.done():
|
||||
stale._aio_task.cancel()
|
||||
|
||||
|
||||
def _count_active() -> int:
|
||||
return sum(1 for t in _tasks.values() if t.state in TaskState.ACTIVE)
|
||||
|
||||
|
||||
def active_count() -> int:
|
||||
"""全平台进行中任务数（pending/crawling/generating）。"""
|
||||
_prune_expired()
|
||||
return _count_active()
|
||||
|
||||
|
||||
def get(slug: str, task_id: str) -> ImportTask | None:
|
||||
"""(slug, task_id) 联合索引查询 + 惰性清理（跨租户 → None → 404）。"""
|
||||
_prune_expired()
|
||||
return _tasks.get((slug, task_id))
|
||||
|
||||
|
||||
async def start(slug: str, url: str | None, *,
|
||||
retry_task_id: str | None = None) -> str:
|
||||
"""新建或复活（retry=generate）导入任务，返回 task_id。
|
||||
|
||||
频控守卫最先（超限后 429 优先于一切预检）；record 只落在真正发起的
|
||||
这次——LookupError/ValueError/ConcurrentLimit 不消耗预算。
|
||||
"""
|
||||
_prune_expired()
|
||||
key = ("import", slug)
|
||||
if not _import_limiter.allow(key):
|
||||
raise ImportRateLimited(
|
||||
f"导入太频繁，{IMPORT_WINDOW // 60} 分钟内最多 {IMPORT_MAX} 次，"
|
||||
"稍后再试")
|
||||
async with _lock:
|
||||
if retry_task_id is not None:
|
||||
target = _tasks.get((slug, retry_task_id))
|
||||
if target is None:
|
||||
raise LookupError(
|
||||
f"task {retry_task_id!r} not found for tenant {slug!r}")
|
||||
if (target.state != TaskState.FAILED
|
||||
or target.failed_phase != "generate"):
|
||||
raise ValueError("任务当前状态不可重试"
|
||||
"（仅 generate 阶段失败可复活）")
|
||||
# 并发名额对新建与 retry 复活同样生效（spec §3.3）
|
||||
if _count_active() >= MAX_CONCURRENT:
|
||||
raise ConcurrentLimit("已有导入任务进行中，稍后再试")
|
||||
if retry_task_id is not None:
|
||||
# 复活同 task_id：复用已抓产物，不重抓
|
||||
target.state = TaskState.PENDING
|
||||
target.error = None
|
||||
target.failed_phase = None
|
||||
target.finished_at = None
|
||||
target.need_crawl = False
|
||||
else:
|
||||
target = ImportTask(slug, uuid.uuid4().hex, url)
|
||||
_tasks[(slug, target.task_id)] = target
|
||||
_import_limiter.record(key)
|
||||
target._aio_task = asyncio.create_task(_pipeline(target))
|
||||
return target.task_id
|
||||
|
||||
|
||||
async def _pipeline(task_obj: ImportTask) -> None:
|
||||
"""后台管线：7 分钟总超时兜底 + 异常归档为白话失败。"""
|
||||
try:
|
||||
await asyncio.wait_for(_run(task_obj), TOTAL_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
task_obj.fail(
|
||||
"导入超时，请重试",
|
||||
phase="generate" if task_obj.crawl_result is not None else None)
|
||||
except CrawlError as exc:
|
||||
task_obj.fail(str(exc), phase="crawl") # SPA/内容不足——不可 retry
|
||||
except GenerateError as exc:
|
||||
task_obj.fail(str(exc), phase="generate") # retry=generate 复活
|
||||
except Exception as exc: # 防御：未知异常不裸抛（任务表对象完整性优先）
|
||||
task_obj.fail(f"导入失败：{exc}", phase=None)
|
||||
|
||||
|
||||
async def _run(task_obj: ImportTask) -> None:
|
||||
if task_obj.need_crawl:
|
||||
task_obj.state = TaskState.CRAWLING
|
||||
task_obj.phase = "crawling"
|
||||
task_obj.crawl_result = await crawler.crawl(task_obj.url)
|
||||
task_obj.pages_fetched = int(
|
||||
task_obj.crawl_result.get("pages_fetched") or 0)
|
||||
task_obj.state = TaskState.GENERATING
|
||||
task_obj.phase = "generating"
|
||||
provider = generator.make_provider(None) # resolver 三级链（平台设置/env）
|
||||
# source_text 传 analyze 实际消费的截断后文本（24K 同款——Task 5 备忘：
|
||||
# 否则截断段数字会被 build 的数字校验误剔除）
|
||||
source = ((task_obj.crawl_result or {}).get("pages_text")
|
||||
or "")[:generator.TOTAL_INPUT_CAP]
|
||||
analysis = await generator.analyze(provider, source)
|
||||
if not (analysis.get("product_lines") or []):
|
||||
# 内容不足三触发之一（调用 1 产品线为空）——同 crawl 类不可复活
|
||||
task_obj.fail(INSUFFICIENT_MESSAGE, phase="crawl")
|
||||
return
|
||||
gen = await generator.generate(provider, analysis)
|
||||
task_obj._finish(
|
||||
generator.build(gen, analysis, task_obj.crawl_result, source))
|
||||
|
||||
|
||||
def _reset_for_test() -> None:
|
||||
"""测试隔离（conftest autouse 前后各跑）：任务表清空（运行中 aio
|
||||
任务防御性取消）+ 频控窗计数/参数复位。"""
|
||||
for t in _tasks.values():
|
||||
if t._aio_task is not None and not t._aio_task.done():
|
||||
t._aio_task.cancel()
|
||||
_tasks.clear()
|
||||
_import_limiter._calls.clear()
|
||||
_import_limiter.window_seconds = IMPORT_WINDOW
|
||||
_import_limiter.max_calls = IMPORT_MAX
|
||||
Reference in New Issue
Block a user