fix(site-import): extract 清洗未闭合块/style 守卫/logo 日期钉位

评审收口三项:clean_html 配对块清理后兜底删 EOF 未闭合块(>500KB
截断防 script/style 泄漏误判 is_thin,顺序钉死配对先行);style 属性
正则补 \s 守卫(data-style 不参与配色计数);FIXTURE_WP 旧日期 logo
前置使日期优先真被甄别(首现即胜的伪绿不再成立)。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-10-20 19:06:57 +08:00
parent c44aad6abd
commit 74c2915f25
2 changed files with 27 additions and 3 deletions
+13 -2
View File
@@ -19,12 +19,18 @@ _RE_BLOCK = re.compile(
r"<(script|style|nav|footer|header)\b[^>]*>.*?</\1\s*>",
re.IGNORECASE | re.DOTALL,
)
# EOF 未闭合块(>500KB 截断的 fetch 可能切在 script/style 中间)——配对清理后兜底
_RE_BLOCK_UNCLOSED = re.compile(
r"<(script|style|nav|footer|header)\b[^>]*>.*$",
re.IGNORECASE | re.DOTALL,
)
_RE_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
_RE_TAG = re.compile(r"<[^>]+>")
_RE_WS = re.compile(r"\s+")
# —— extract_color:只看内联 style 属性 + 外部 css_text,不扫全文(避免误收 URL 等) ——
_RE_STYLE_ATTR = re.compile(r"""style\s*=\s*["']([^"']*)["']""", re.IGNORECASE)
# \sstyle 防 data-style 等复合属性名误命中(与 \ssrc/\shref 同款守卫)
_RE_STYLE_ATTR = re.compile(r"""\sstyle\s*=\s*["']([^"']*)["']""", re.IGNORECASE)
# 6 位优先、3 位展开;负向前瞻防截取更长 hex(如 #aabbccdd)的前缀
_RE_HEX = re.compile(r"#([0-9a-fA-F]{6}|[0-9a-fA-F]{3})(?![0-9a-fA-F])")
@@ -61,8 +67,13 @@ _RESIDUE_PATTERNS: list[tuple[re.Pattern[str], str]] = [
def clean_html(html_text: str) -> str:
"""去 script/style/nav/footer/header/注释,剩余标签转文本,空白折叠,>8192 截断。"""
"""去 script/style/nav/footer/header/注释,剩余标签转文本,空白折叠,>8192 截断。
先删配对块、再兜底删 EOF 未闭合块——顺序不可倒(未闭合模式会吞掉
配对块之后的全部文本)。
"""
t = _RE_BLOCK.sub(" ", html_text)
t = _RE_BLOCK_UNCLOSED.sub(" ", t)
t = _RE_COMMENT.sub(" ", t)
t = _RE_TAG.sub(" ", t)
t = _html.unescape(t)
+14 -1
View File
@@ -29,6 +29,7 @@ FIXTURE_WP = """<!DOCTYPE html>
times on every order of building products.</p>
<div class="hero" style="background:#ffffff">
<img src="https://ex.com/wp-content/uploads/2026/01/hero-banner.jpg" alt="hero">
<img src="https://ex.com/wp-content/uploads/2020/02/logo-old.png" alt="old logo">
<a href="/"><img src="https://ex.com/wp-content/uploads/2026/03/logo-new.png" alt="logo"></a>
</div>
<section style="background-color:#ffffff">
@@ -41,7 +42,6 @@ FIXTURE_WP = """<!DOCTYPE html>
<a href="https://wa.me/8618603858592">WhatsApp us</a> |
<a href="mailto:sales@hesuu.com">Email sales</a> |
<a href="https://blog.other.com/building-tips">External blog</a>
<img src="https://ex.com/wp-content/uploads/2020/02/logo-old.png" alt="old logo">
</section>
<section class="cards" style="background:#ffffff">
<div class="card" style="background:#ffffff;border-top:4px solid #FCCA03">
@@ -96,6 +96,9 @@ FIXTURE_CROSS = """<html><body>
<a href="javascript:void(0)">JS</a>
</body></html>"""
# >500KB 抓取截断可能切在 script 中间——EOF 未闭合块不得泄漏进清洗文本
FIXTURE_UNCLOSED = '<html><body><p>keep this text</p><script>var j = "JAVAJUNK";'
class TestCleanHtml:
def test_去script_style_nav_footer_header_comment(self):
@@ -108,6 +111,11 @@ class TestCleanHtml:
t = clean_html(FIXTURE_9K)
assert len(t) <= 8192
def test_EOF未闭合script不泄漏(self):
t = clean_html(FIXTURE_UNCLOSED)
assert "JAVAJUNK" not in t
assert "keep this text" in t
class TestExtractColor:
def test_排除白黑灰取最高频(self):
@@ -125,6 +133,11 @@ class TestExtractColor:
html = '<span style="color:#fc3">x</span><b style="color:#fc3">y</b>'
assert extract_color(html, None) == "#ffcc33"
def test_data_style不参与计数(self):
# #123456 为可配色——若 data-style 被误收则返回它而非 None
html = '<div data-style="color:#123456">x</div>'
assert extract_color(html, None) is None
class TestExtractLogoUrl:
def test_取日期新者(self):