feat(site-import): HTML 清洗与抽取纯函数
spec §3.1 Task 3：clean_html（去 script/style/nav/footer/header/注释 + 空白折叠 + 8K 截断）、extract_color（内联 style+CSS 合并计数，白黑灰 排除，平票首现）、extract_logo_url（/YYYY/MM/ 日期新者胜出）、 extract_contacts（wa.me 归一 E164 + mailto）、find_residue（模板残留 清单）、discover_links（同源绝对化去锚去询，关键词稳定排前）、is_thin。 零新依赖（stdlib regex），纯函数无 IO，15 测试钉位。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,173 @@
|
||||
"""site_import HTML 清洗与抽取纯函数（spec §3.1）——fixture 内联，覆盖即标准。"""
|
||||
from app.platform.site_import.extract import (
|
||||
clean_html,
|
||||
discover_links,
|
||||
extract_color,
|
||||
extract_contacts,
|
||||
extract_logo_url,
|
||||
find_residue,
|
||||
is_thin,
|
||||
)
|
||||
|
||||
# WordPress 形态首页：script/style/nav/footer/header/注释各一段（埋独特词），
|
||||
# 内联 #FCCA03×3 / #ffffff×5 / #888888×2，logo 双版本（日期新者应胜出），
|
||||
# wa.me / mailto 联系方式，页脚模板残留三件套，同源产品/关于/联系页链接。
|
||||
FIXTURE_WP = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head><title>Hesuu Building Materials</title></head>
|
||||
<body>
|
||||
<header class="site-header"><div class="topbar">HEADJUNK promo bar text</div></header>
|
||||
<nav class="main-menu"><ul><li>NAVJUNK menu item one</li><li>NAVJUNK menu item two</li></ul></nav>
|
||||
<!-- COMMENTJUNK: builder leftover note -->
|
||||
<style>.theme-font{font-family:sans-serif}.theme-box{padding:1em}</style>
|
||||
<script>window.__track = function(){ alert("SCRIPTJUNK"); };</script>
|
||||
<main>
|
||||
<h1 style="border-bottom:3px solid #FCCA03">Welcome to Hesuu Building Materials</h1>
|
||||
<p>We are a leading supplier of construction products with over fifteen years
|
||||
of export experience. Our catalogue covers steel pipes, fittings, and panels
|
||||
for wholesalers and distributors worldwide, with stable quality and fast lead
|
||||
times on every order of building products.</p>
|
||||
<div class="hero" style="background:#ffffff">
|
||||
<img src="https://ex.com/wp-content/uploads/2026/01/hero-banner.jpg" alt="hero">
|
||||
<a href="/"><img src="https://ex.com/wp-content/uploads/2026/03/logo-new.png" alt="logo"></a>
|
||||
</div>
|
||||
<section style="background-color:#ffffff">
|
||||
<span style="color:#FCCA03">Featured collections</span>
|
||||
<p style="color:#888888">Mild steel pipes DN800 in stock.</p>
|
||||
<a href="/products/">All products</a> |
|
||||
<a href="https://ex.com/products/page/2/">Next page</a> |
|
||||
<a href="/about/">About us</a> |
|
||||
<a href="/contact/">Contact us</a> |
|
||||
<a href="https://wa.me/8618603858592">WhatsApp us</a> |
|
||||
<a href="mailto:sales@hesuu.com">Email sales</a> |
|
||||
<a href="https://blog.other.com/building-tips">External blog</a>
|
||||
<img src="https://ex.com/wp-content/uploads/2020/02/logo-old.png" alt="old logo">
|
||||
</section>
|
||||
<section class="cards" style="background:#ffffff">
|
||||
<div class="card" style="background:#ffffff;border-top:4px solid #FCCA03">
|
||||
<h2 style="color:#888888">Card title two</h2>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
<footer class="site-footer">
|
||||
<p>FOOTJUNK widget area</p>
|
||||
<p>Call us: 1-555-644-5566 © Zemez</p>
|
||||
<h3>Add Your Heading Text Here</h3>
|
||||
<span style="color:#ffffff">white on white</span>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# 内联只剩 #ffffff（白系被排除）——CSS 文件的 #6b9ebd×4 应胜出
|
||||
FIXTURE_WP_ONLY_WHITE = """<html><body>
|
||||
<main>
|
||||
<h1 style="color:#ffffff">Plain white headline</h1>
|
||||
<div style="background:#ffffff">White box with some words.</div>
|
||||
</main>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# 只有灰系——全被排除则 None
|
||||
FIXTURE_ONLY_GRAY = """<html><body>
|
||||
<p style="color:#888888">Only gray text here.</p>
|
||||
<span style="border-color:#888888">gray border</span>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# CSS 文件内容：#6b9ebd×4 + #ffffff×1
|
||||
FIXTURE_CSS = (
|
||||
".btn{color:#6b9ebd}\n"
|
||||
".btn-primary{color:#6b9ebd}\n"
|
||||
".link{color:#6b9ebd}\n"
|
||||
".badge{color:#6b9ebd}\n"
|
||||
".wrap{background:#ffffff}\n"
|
||||
)
|
||||
|
||||
FIXTURE_THIN = "<html><body><div></div></body></html>"
|
||||
|
||||
# 8K 截断用 9000 字符正文版本（>8192，避免测试文件巨大）
|
||||
FIXTURE_9K = "<html><body>" + "x" * 9000 + "</body></html>"
|
||||
|
||||
# 跨源 / 非 http 链接剔除
|
||||
FIXTURE_CROSS = """<html><body>
|
||||
<a href="/about/">About</a>
|
||||
<a href="https://evil.example.com/phish">Evil</a>
|
||||
<a href="javascript:void(0)">JS</a>
|
||||
</body></html>"""
|
||||
|
||||
|
||||
class TestCleanHtml:
|
||||
def test_去script_style_nav_footer_header_comment(self):
|
||||
t = clean_html(FIXTURE_WP)
|
||||
for junk in ["SCRIPTJUNK", "STYLEJUNK", "NAVJUNK", "FOOTJUNK", "HEADJUNK", "COMMENTJUNK"]:
|
||||
assert junk not in t
|
||||
assert "products" in t # 正文保留
|
||||
|
||||
def test_8K字符截断(self):
|
||||
t = clean_html(FIXTURE_9K)
|
||||
assert len(t) <= 8192
|
||||
|
||||
|
||||
class TestExtractColor:
|
||||
def test_排除白黑灰取最高频(self):
|
||||
# 内联 #FCCA03×3 应胜出（#ffffff×5 与 #888888×2 被排除）
|
||||
assert extract_color(FIXTURE_WP, None) == "#fcca03"
|
||||
|
||||
def test_CSS文件参与统计(self):
|
||||
# 内联只有 #ffffff；CSS 里 #6b9ebd×4 → 应取 #6b9ebd
|
||||
assert extract_color(FIXTURE_WP_ONLY_WHITE, FIXTURE_CSS) == "#6b9ebd"
|
||||
|
||||
def test_全被排除则None(self):
|
||||
assert extract_color(FIXTURE_ONLY_GRAY, None) is None
|
||||
|
||||
def test_三位hex展开计数(self):
|
||||
html = '<span style="color:#fc3">x</span><b style="color:#fc3">y</b>'
|
||||
assert extract_color(html, None) == "#ffcc33"
|
||||
|
||||
|
||||
class TestExtractLogoUrl:
|
||||
def test_取日期新者(self):
|
||||
assert extract_logo_url(FIXTURE_WP) == (
|
||||
"https://ex.com/wp-content/uploads/2026/03/logo-new.png")
|
||||
|
||||
def test_无logo返回None(self):
|
||||
assert extract_logo_url(FIXTURE_THIN) is None
|
||||
|
||||
|
||||
class TestExtractContacts:
|
||||
def test_wa_me归一E164(self):
|
||||
c = extract_contacts(FIXTURE_WP)
|
||||
assert c["whatsapp"] == "+8618603858592"
|
||||
assert c["email"] == "sales@hesuu.com"
|
||||
|
||||
def test_无联系方式返回空(self):
|
||||
assert extract_contacts(FIXTURE_THIN) == {"whatsapp": None, "email": None}
|
||||
|
||||
|
||||
class TestFindResidue:
|
||||
def test_常见模板残留命中(self):
|
||||
hits = find_residue("page text", FIXTURE_WP)
|
||||
assert any("555" in h for h in hits)
|
||||
assert any("Zemez" in h for h in hits)
|
||||
assert any("Heading" in h for h in hits)
|
||||
|
||||
|
||||
class TestDiscoverLinks:
|
||||
def test_同源产品关于联系页发现(self):
|
||||
links = discover_links("https://ex.com/", FIXTURE_WP)
|
||||
assert "https://ex.com/products/" in links
|
||||
assert "https://ex.com/products/page/2/" in links
|
||||
assert "https://ex.com/about/" in links
|
||||
|
||||
def test_跨源链接剔除(self):
|
||||
links = discover_links("https://ex.com/", FIXTURE_CROSS)
|
||||
assert links == ["https://ex.com/about/"]
|
||||
|
||||
|
||||
class TestIsThin:
|
||||
def test_空壳为True(self):
|
||||
assert is_thin(clean_html(FIXTURE_THIN)) is True
|
||||
|
||||
def test_正常页为False(self):
|
||||
assert is_thin(clean_html(FIXTURE_WP)) is False
|
||||
Reference in New Issue
Block a user