"""site_import HTML 清洗与抽取纯函数(spec §3.1)——fixture 内联,覆盖即标准。""" from app.platform.site_import.extract import ( clean_html, discover_links, extract_color, extract_contacts, extract_logo_url, find_residue, is_thin, ) # WordPress 形态首页:script/style/nav/footer/header/注释各一段(埋独特词), # 内联 #FCCA03×3 / #ffffff×5 / #888888×2,logo 双版本(日期新者应胜出), # wa.me / mailto 联系方式,页脚模板残留三件套,同源产品/关于/联系页链接。 FIXTURE_WP = """ Hesuu Building Materials

Welcome to Hesuu Building Materials

We are a leading supplier of construction products with over fifteen years of export experience. Our catalogue covers steel pipes, fittings, and panels for wholesalers and distributors worldwide, with stable quality and fast lead times on every order of building products.

hero logo
Featured collections

Mild steel pipes DN800 in stock.

All products | Next page | About us | Contact us | WhatsApp us | Email sales | External blog old logo

Card title two

""" # 内联只剩 #ffffff(白系被排除)——CSS 文件的 #6b9ebd×4 应胜出 FIXTURE_WP_ONLY_WHITE = """

Plain white headline

White box with some words.
""" # 只有灰系——全被排除则 None FIXTURE_ONLY_GRAY = """

Only gray text here.

gray border """ # CSS 文件内容:#6b9ebd×4 + #ffffff×1 FIXTURE_CSS = ( ".btn{color:#6b9ebd}\n" ".btn-primary{color:#6b9ebd}\n" ".link{color:#6b9ebd}\n" ".badge{color:#6b9ebd}\n" ".wrap{background:#ffffff}\n" ) FIXTURE_THIN = "
" # 8K 截断用 9000 字符正文版本(>8192,避免测试文件巨大) FIXTURE_9K = "" + "x" * 9000 + "" # 跨源 / 非 http 链接剔除 FIXTURE_CROSS = """ About Evil JS """ class TestCleanHtml: def test_去script_style_nav_footer_header_comment(self): t = clean_html(FIXTURE_WP) for junk in ["SCRIPTJUNK", "STYLEJUNK", "NAVJUNK", "FOOTJUNK", "HEADJUNK", "COMMENTJUNK"]: assert junk not in t assert "products" in t # 正文保留 def test_8K字符截断(self): t = clean_html(FIXTURE_9K) assert len(t) <= 8192 class TestExtractColor: def test_排除白黑灰取最高频(self): # 内联 #FCCA03×3 应胜出(#ffffff×5 与 #888888×2 被排除) assert extract_color(FIXTURE_WP, None) == "#fcca03" def test_CSS文件参与统计(self): # 内联只有 #ffffff;CSS 里 #6b9ebd×4 → 应取 #6b9ebd assert extract_color(FIXTURE_WP_ONLY_WHITE, FIXTURE_CSS) == "#6b9ebd" def test_全被排除则None(self): assert extract_color(FIXTURE_ONLY_GRAY, None) is None def test_三位hex展开计数(self): html = 'xy' assert extract_color(html, None) == "#ffcc33" class TestExtractLogoUrl: def test_取日期新者(self): assert extract_logo_url(FIXTURE_WP) == ( "https://ex.com/wp-content/uploads/2026/03/logo-new.png") def test_无logo返回None(self): assert extract_logo_url(FIXTURE_THIN) is None class TestExtractContacts: def test_wa_me归一E164(self): c = extract_contacts(FIXTURE_WP) assert c["whatsapp"] == "+8618603858592" assert c["email"] == "sales@hesuu.com" def test_无联系方式返回空(self): assert extract_contacts(FIXTURE_THIN) == {"whatsapp": None, "email": None} class TestFindResidue: def test_常见模板残留命中(self): hits = find_residue("page text", FIXTURE_WP) assert any("555" in h for h in hits) assert any("Zemez" in h for h in hits) assert any("Heading" in h for h in hits) class TestDiscoverLinks: def test_同源产品关于联系页发现(self): links = discover_links("https://ex.com/", FIXTURE_WP) assert "https://ex.com/products/" in links assert "https://ex.com/products/page/2/" in links assert "https://ex.com/about/" in links def test_跨源链接剔除(self): links = discover_links("https://ex.com/", FIXTURE_CROSS) assert links == ["https://ex.com/about/"] class TestIsThin: def test_空壳为True(self): assert is_thin(clean_html(FIXTURE_THIN)) is True def test_正常页为False(self): assert is_thin(clean_html(FIXTURE_WP)) is False