yunke/yunke-server/tests/test_scrape_whatsapp_accounts.py

"""Tests for the WhatsApp staff-account scraper (M3).

Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.

The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal for page-number endpoints):

* page 1 -> 2 records, ``totalCount=4`` (rows under ``data.overviews``)
* page 2 -> 2 records, ``totalCount=4``
* page 3 -> empty ``overviews`` array (no more rows)

The scraper must yield exactly the 4 records (normalized to ``WhatsappAccount``
snake_case columns + ``raw_json`` + the embedded staff pair), make exactly 3
calls, and stop without a 4th. The empty-page stop is the primary signal;
``totalCount=4`` is only a defensive over-fetch ceiling that must NOT trigger
early (we yield exactly 4 == total, so the strict-``>`` ceiling never fires).
"""

from __future__ import annotations

import json as _json

import httpx
import respx

from app.scraper.whatsapp import scrape_whatsapp_accounts
from app.session.yunke_client import YunkeClient
from tests.cassettes.wa_accounts_fixture import (
    DEFAULT_PAGE_SIZE,
    TOTAL,
    WA_ACCOUNT_RECORDS,
    make_wa_accounts_response,
)

BASE = "https://crm2.yunkecn.com"
WA_ACCOUNTS_URL = f"{BASE}/qqweb/WhatsAppUser/businessSalesList"

USERID = "a" * 32
COMPANY = "8m0atd"


def _primed_client() -> YunkeClient:
    """A YunkeClient already in a logged-in state (login flow skipped)."""
    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)


# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + embedded staff + raw_json
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_accounts_yields_records_and_paginates():
    """Two full pages + a terminating empty page -> 4 records, 3 calls."""
    page1 = make_wa_accounts_response(
        WA_ACCOUNT_RECORDS[:2], total_count=TOTAL, page=1, page_size=DEFAULT_PAGE_SIZE
    )
    page2 = make_wa_accounts_response(
        WA_ACCOUNT_RECORDS[2:], total_count=TOTAL, page=2, page_size=DEFAULT_PAGE_SIZE
    )
    page3 = make_wa_accounts_response(
        [], total_count=TOTAL, page=3, page_size=DEFAULT_PAGE_SIZE
    )

    route = respx.post(WA_ACCOUNTS_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
            httpx.Response(200, json=page3),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r
            async for r in scrape_whatsapp_accounts(client, page_size=DEFAULT_PAGE_SIZE)
        ]
    finally:
        await client.aclose()

    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
    assert route.call_count == 3

    # Exactly the 4 fixture records (not duplicated, not dropped).
    assert len(results) == 4
    assert {r["uid"] for r in results} == {rec["uid"] for rec in WA_ACCOUNT_RECORDS}

    # Field mapping for each record (compare against the source record).
    for result, record in zip(results, WA_ACCOUNT_RECORDS, strict=True):
        # Identity PK.
        assert result["uid"] == record["uid"]

        # Embedded staff pair (the owning employee).
        assert result["user_id"] == record["userId"]
        assert result["user_name"] == record["userName"]

        # Naming + phones.
        assert result["nick_name"] == record["nickName"]
        assert result["phone"] == record["phone"]
        assert result["sale_phone"] == record["salePhone"]
        assert result["sales_bind_phone"] == record["salesBindPhone"]

        # Counts.
        assert result["contacts_num"] == record["contactsNum"]
        assert result["group_num"] == record["groupNum"]
        assert result["talker_count"] == record["talkerCount"]
        assert result["friend_add_num"] == record["friendAddNum"]

        # Status / timestamps.
        assert result["is_on_job"] == record["isOnJob"]
        assert result["last_update"] == record["lastUpdate"]

        # Version carried verbatim; raw_json preserves the original record.
        assert result["version"] == record["version"]
        assert result["raw_json"] == record


# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_accounts_sends_pagination_body():
    """Outgoing request body carries pageNum/pageSize; advances per page."""
    page1 = make_wa_accounts_response(
        WA_ACCOUNT_RECORDS[:1], total_count=TOTAL, page=1, page_size=1
    )
    page2 = make_wa_accounts_response([], total_count=TOTAL, page=2, page_size=1)
    route = respx.post(WA_ACCOUNTS_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_whatsapp_accounts(client, page_size=1)]
    finally:
        await client.aclose()

    # One record on page 1, empty page 2 stops the loop.
    assert len(results) == 1
    assert route.call_count == 2

    # First request: pageNum=1.
    body1 = _json.loads(route.calls[0].request.content.decode())
    assert body1["pageNum"] == 1
    assert body1["pageSize"] == 1

    # Second request: pageNum advanced to 2.
    body2 = _json.loads(route.calls[1].request.content.decode())
    assert body2["pageNum"] == 2
    assert body2["pageSize"] == 1


# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_accounts_stops_on_empty_first_page():
    """If the very first page is empty, scraper stops without over-fetching."""
    empty = make_wa_accounts_response([], total_count=0, page=1)
    route = respx.post(WA_ACCOUNTS_URL).mock(
        return_value=httpx.Response(200, json=empty)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_whatsapp_accounts(client)]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1


# ---------------------------------------------------------------------------
# Envelope: rows are read from data.overviews (NOT data.data)
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_accounts_reads_overviews_envelope():
    """Rows under ``data.overviews`` are picked up (not the WeChat ``data.data``)."""
    # A response that ALSO carries an empty ``data.data`` list must not confuse
    # the scraper — it must read ``overviews`` and yield the record.
    page = make_wa_accounts_response(
        WA_ACCOUNT_RECORDS[:1], total_count=1, page=1, page_size=1
    )
    page["data"]["data"] = []  # WeChat-style key present but empty
    respx.post(WA_ACCOUNTS_URL).mock(return_value=httpx.Response(200, json=page))
    # Serve an empty page next so the paginator stops.
    respx.post(WA_ACCOUNTS_URL).mock(
        side_effect=[
            httpx.Response(200, json=page),
            httpx.Response(
                200,
                json=make_wa_accounts_response([], total_count=1, page=2, page_size=1),
            ),
        ]
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_whatsapp_accounts(client, page_size=1)]
    finally:
        await client.aclose()

    assert len(results) == 1
    assert results[0]["uid"] == WA_ACCOUNT_RECORDS[0]["uid"]