diff --git a/app/platform/site_import/extract.py b/app/platform/site_import/extract.py index 080abe5..0d8ce07 100644 --- a/app/platform/site_import/extract.py +++ b/app/platform/site_import/extract.py @@ -19,12 +19,18 @@ _RE_BLOCK = re.compile( r"<(script|style|nav|footer|header)\b[^>]*>.*?\1\s*>", re.IGNORECASE | re.DOTALL, ) +# EOF 未闭合块(>500KB 截断的 fetch 可能切在 script/style 中间)——配对清理后兜底 +_RE_BLOCK_UNCLOSED = re.compile( + r"<(script|style|nav|footer|header)\b[^>]*>.*$", + re.IGNORECASE | re.DOTALL, +) _RE_COMMENT = re.compile(r"", re.DOTALL) _RE_TAG = re.compile(r"<[^>]+>") _RE_WS = re.compile(r"\s+") # —— extract_color:只看内联 style 属性 + 外部 css_text,不扫全文(避免误收 URL 等) —— -_RE_STYLE_ATTR = re.compile(r"""style\s*=\s*["']([^"']*)["']""", re.IGNORECASE) +# \sstyle 防 data-style 等复合属性名误命中(与 \ssrc/\shref 同款守卫) +_RE_STYLE_ATTR = re.compile(r"""\sstyle\s*=\s*["']([^"']*)["']""", re.IGNORECASE) # 6 位优先、3 位展开;负向前瞻防截取更长 hex(如 #aabbccdd)的前缀 _RE_HEX = re.compile(r"#([0-9a-fA-F]{6}|[0-9a-fA-F]{3})(?![0-9a-fA-F])") @@ -61,8 +67,13 @@ _RESIDUE_PATTERNS: list[tuple[re.Pattern[str], str]] = [ def clean_html(html_text: str) -> str: - """去 script/style/nav/footer/header/注释,剩余标签转文本,空白折叠,>8192 截断。""" + """去 script/style/nav/footer/header/注释,剩余标签转文本,空白折叠,>8192 截断。 + + 先删配对块、再兜底删 EOF 未闭合块——顺序不可倒(未闭合模式会吞掉 + 配对块之后的全部文本)。 + """ t = _RE_BLOCK.sub(" ", html_text) + t = _RE_BLOCK_UNCLOSED.sub(" ", t) t = _RE_COMMENT.sub(" ", t) t = _RE_TAG.sub(" ", t) t = _html.unescape(t) diff --git a/tests/test_site_import_extract.py b/tests/test_site_import_extract.py index e08adaf..2a1d304 100644 --- a/tests/test_site_import_extract.py +++ b/tests/test_site_import_extract.py @@ -29,6 +29,7 @@ FIXTURE_WP = """ times on every order of building products.
keep this text