ai-kefu-admin/tests/test_site_import_extract.py

"""site_import HTML 清洗与抽取纯函数（spec §3.1）——fixture 内联，覆盖即标准。"""
from app.platform.site_import.extract import (
    clean_html,
    discover_links,
    extract_color,
    extract_contacts,
    extract_logo_url,
    find_residue,
    is_thin,
)

# WordPress 形态首页：script/style/nav/footer/header/注释各一段（埋独特词），
# 内联 #FCCA03×3 / #ffffff×5 / #888888×2，logo 双版本（日期新者应胜出），
# wa.me / mailto 联系方式，页脚模板残留三件套，同源产品/关于/联系页链接。
FIXTURE_WP = """<!DOCTYPE html>
<html lang="en">
<head><title>Hesuu Building Materials</title></head>
<body>
<header class="site-header"><div class="topbar">HEADJUNK promo bar text</div></header>
<nav class="main-menu"><ul><li>NAVJUNK menu item one</li><li>NAVJUNK menu item two</li></ul></nav>
<!-- COMMENTJUNK: builder leftover note -->
<style>.theme-font{font-family:sans-serif}.theme-box{padding:1em}</style>
<script>window.__track = function(){ alert("SCRIPTJUNK"); };</script>
<main>
  <h1 style="border-bottom:3px solid #FCCA03">Welcome to Hesuu Building Materials</h1>
  <p>We are a leading supplier of construction products with over fifteen years
  of export experience. Our catalogue covers steel pipes, fittings, and panels
  for wholesalers and distributors worldwide, with stable quality and fast lead
  times on every order of building products.</p>
  <div class="hero" style="background:#ffffff">
    <img src="https://ex.com/wp-content/uploads/2026/01/hero-banner.jpg" alt="hero">
    <img src="https://ex.com/wp-content/uploads/2020/02/logo-old.png" alt="old logo">
    <a href="/"><img src="https://ex.com/wp-content/uploads/2026/03/logo-new.png" alt="logo"></a>
  </div>
  <section style="background-color:#ffffff">
    <span style="color:#FCCA03">Featured collections</span>
    <p style="color:#888888">Mild steel pipes DN800 in stock.</p>
    <a href="/products/">All products</a> |
    <a href="https://ex.com/products/page/2/">Next page</a> |
    <a href="/about/">About us</a> |
    <a href="/contact/">Contact us</a> |
    <a href="https://wa.me/8618603858592">WhatsApp us</a> |
    <a href="mailto:sales@hesuu.com">Email sales</a> |
    <a href="https://blog.other.com/building-tips">External blog</a>
  </section>
  <section class="cards" style="background:#ffffff">
    <div class="card" style="background:#ffffff;border-top:4px solid #FCCA03">
      <h2 style="color:#888888">Card title two</h2>
    </div>
  </section>
</main>
<footer class="site-footer">
  <p>FOOTJUNK widget area</p>
  <p>Call us: 1-555-644-5566 &copy; Zemez</p>
  <h3>Add Your Heading Text Here</h3>
  <span style="color:#ffffff">white on white</span>
</footer>
</body>
</html>
"""

# 内联只剩 #ffffff（白系被排除）——CSS 文件的 #6b9ebd×4 应胜出
FIXTURE_WP_ONLY_WHITE = """<html><body>
<main>
  <h1 style="color:#ffffff">Plain white headline</h1>
  <div style="background:#ffffff">White box with some words.</div>
</main>
</body></html>
"""

# 只有灰系——全被排除则 None
FIXTURE_ONLY_GRAY = """<html><body>
<p style="color:#888888">Only gray text here.</p>
<span style="border-color:#888888">gray border</span>
</body></html>
"""

# CSS 文件内容：#6b9ebd×4 + #ffffff×1
FIXTURE_CSS = (
    ".btn{color:#6b9ebd}\n"
    ".btn-primary{color:#6b9ebd}\n"
    ".link{color:#6b9ebd}\n"
    ".badge{color:#6b9ebd}\n"
    ".wrap{background:#ffffff}\n"
)

FIXTURE_THIN = "<html><body><div></div></body></html>"

# 8K 截断用 9000 字符正文版本（>8192，避免测试文件巨大）
FIXTURE_9K = "<html><body>" + "x" * 9000 + "</body></html>"

# 跨源 / 非 http 链接剔除
FIXTURE_CROSS = """<html><body>
<a href="/about/">About</a>
<a href="https://evil.example.com/phish">Evil</a>
<a href="javascript:void(0)">JS</a>
</body></html>"""

# >500KB 抓取截断可能切在 script 中间——EOF 未闭合块不得泄漏进清洗文本
FIXTURE_UNCLOSED = '<html><body><p>keep this text</p><script>var j = "JAVAJUNK";'


class TestCleanHtml:
    def test_去script_style_nav_footer_header_comment(self):
        t = clean_html(FIXTURE_WP)
        for junk in ["SCRIPTJUNK", "STYLEJUNK", "NAVJUNK", "FOOTJUNK", "HEADJUNK", "COMMENTJUNK"]:
            assert junk not in t
        assert "products" in t  # 正文保留

    def test_8K字符截断(self):
        t = clean_html(FIXTURE_9K)
        assert len(t) <= 8192

    def test_EOF未闭合script不泄漏(self):
        t = clean_html(FIXTURE_UNCLOSED)
        assert "JAVAJUNK" not in t
        assert "keep this text" in t


class TestExtractColor:
    def test_排除白黑灰取最高频(self):
        # 内联 #FCCA03×3 应胜出（#ffffff×5 与 #888888×2 被排除）
        assert extract_color(FIXTURE_WP, None) == "#fcca03"

    def test_CSS文件参与统计(self):
        # 内联只有 #ffffff；CSS 里 #6b9ebd×4 → 应取 #6b9ebd
        assert extract_color(FIXTURE_WP_ONLY_WHITE, FIXTURE_CSS) == "#6b9ebd"

    def test_全被排除则None(self):
        assert extract_color(FIXTURE_ONLY_GRAY, None) is None

    def test_三位hex展开计数(self):
        html = '<span style="color:#fc3">x</span><b style="color:#fc3">y</b>'
        assert extract_color(html, None) == "#ffcc33"

    def test_data_style不参与计数(self):
        # #123456 为可配色——若 data-style 被误收则返回它而非 None
        html = '<div data-style="color:#123456">x</div>'
        assert extract_color(html, None) is None


class TestExtractLogoUrl:
    def test_取日期新者(self):
        assert extract_logo_url(FIXTURE_WP) == (
            "https://ex.com/wp-content/uploads/2026/03/logo-new.png")

    def test_无logo返回None(self):
        assert extract_logo_url(FIXTURE_THIN) is None


class TestExtractContacts:
    def test_wa_me归一E164(self):
        c = extract_contacts(FIXTURE_WP)
        assert c["whatsapp"] == "+8618603858592"
        assert c["email"] == "sales@hesuu.com"

    def test_无联系方式返回空(self):
        assert extract_contacts(FIXTURE_THIN) == {"whatsapp": None, "email": None}


class TestFindResidue:
    def test_常见模板残留命中(self):
        hits = find_residue("page text", FIXTURE_WP)
        assert any("555" in h for h in hits)
        assert any("Zemez" in h for h in hits)
        assert any("Heading" in h for h in hits)


class TestDiscoverLinks:
    def test_同源产品关于联系页发现(self):
        links = discover_links("https://ex.com/", FIXTURE_WP)
        assert "https://ex.com/products/" in links
        assert "https://ex.com/products/page/2/" in links
        assert "https://ex.com/about/" in links

    def test_跨源链接剔除(self):
        links = discover_links("https://ex.com/", FIXTURE_CROSS)
        assert links == ["https://ex.com/about/"]


class TestIsThin:
    def test_空壳为True(self):
        assert is_thin(clean_html(FIXTURE_THIN)) is True

    def test_正常页为False(self):
        assert is_thin(clean_html(FIXTURE_WP)) is False