"""site_import HTML 清洗与抽取纯函数(spec §3.1)——fixture 内联,覆盖即标准。"""
from app.platform.site_import.extract import (
clean_html,
discover_links,
extract_color,
extract_contacts,
extract_logo_url,
find_residue,
is_thin,
)
# WordPress 形态首页:script/style/nav/footer/header/注释各一段(埋独特词),
# 内联 #FCCA03×3 / #ffffff×5 / #888888×2,logo 双版本(日期新者应胜出),
# wa.me / mailto 联系方式,页脚模板残留三件套,同源产品/关于/联系页链接。
FIXTURE_WP = """
Hesuu Building Materials
NAVJUNK menu item one NAVJUNK menu item two
Welcome to Hesuu Building Materials
We are a leading supplier of construction products with over fifteen years
of export experience. Our catalogue covers steel pipes, fittings, and panels
for wholesalers and distributors worldwide, with stable quality and fast lead
times on every order of building products.
"""
# 内联只剩 #ffffff(白系被排除)——CSS 文件的 #6b9ebd×4 应胜出
FIXTURE_WP_ONLY_WHITE = """
Plain white headline
White box with some words.
"""
# 只有灰系——全被排除则 None
FIXTURE_ONLY_GRAY = """
Only gray text here.
gray border
"""
# CSS 文件内容:#6b9ebd×4 + #ffffff×1
FIXTURE_CSS = (
".btn{color:#6b9ebd}\n"
".btn-primary{color:#6b9ebd}\n"
".link{color:#6b9ebd}\n"
".badge{color:#6b9ebd}\n"
".wrap{background:#ffffff}\n"
)
FIXTURE_THIN = "
"
# 8K 截断用 9000 字符正文版本(>8192,避免测试文件巨大)
FIXTURE_9K = "" + "x" * 9000 + ""
# 跨源 / 非 http 链接剔除
FIXTURE_CROSS = """
About
Evil
JS
"""
class TestCleanHtml:
def test_去script_style_nav_footer_header_comment(self):
t = clean_html(FIXTURE_WP)
for junk in ["SCRIPTJUNK", "STYLEJUNK", "NAVJUNK", "FOOTJUNK", "HEADJUNK", "COMMENTJUNK"]:
assert junk not in t
assert "products" in t # 正文保留
def test_8K字符截断(self):
t = clean_html(FIXTURE_9K)
assert len(t) <= 8192
class TestExtractColor:
def test_排除白黑灰取最高频(self):
# 内联 #FCCA03×3 应胜出(#ffffff×5 与 #888888×2 被排除)
assert extract_color(FIXTURE_WP, None) == "#fcca03"
def test_CSS文件参与统计(self):
# 内联只有 #ffffff;CSS 里 #6b9ebd×4 → 应取 #6b9ebd
assert extract_color(FIXTURE_WP_ONLY_WHITE, FIXTURE_CSS) == "#6b9ebd"
def test_全被排除则None(self):
assert extract_color(FIXTURE_ONLY_GRAY, None) is None
def test_三位hex展开计数(self):
html = 'x y '
assert extract_color(html, None) == "#ffcc33"
class TestExtractLogoUrl:
def test_取日期新者(self):
assert extract_logo_url(FIXTURE_WP) == (
"https://ex.com/wp-content/uploads/2026/03/logo-new.png")
def test_无logo返回None(self):
assert extract_logo_url(FIXTURE_THIN) is None
class TestExtractContacts:
def test_wa_me归一E164(self):
c = extract_contacts(FIXTURE_WP)
assert c["whatsapp"] == "+8618603858592"
assert c["email"] == "sales@hesuu.com"
def test_无联系方式返回空(self):
assert extract_contacts(FIXTURE_THIN) == {"whatsapp": None, "email": None}
class TestFindResidue:
def test_常见模板残留命中(self):
hits = find_residue("page text", FIXTURE_WP)
assert any("555" in h for h in hits)
assert any("Zemez" in h for h in hits)
assert any("Heading" in h for h in hits)
class TestDiscoverLinks:
def test_同源产品关于联系页发现(self):
links = discover_links("https://ex.com/", FIXTURE_WP)
assert "https://ex.com/products/" in links
assert "https://ex.com/products/page/2/" in links
assert "https://ex.com/about/" in links
def test_跨源链接剔除(self):
links = discover_links("https://ex.com/", FIXTURE_CROSS)
assert links == ["https://ex.com/about/"]
class TestIsThin:
def test_空壳为True(self):
assert is_thin(clean_html(FIXTURE_THIN)) is True
def test_正常页为False(self):
assert is_thin(clean_html(FIXTURE_WP)) is False