yunke/yunke-server/tests/test_scrape_whatsapp_friends.py

"""Tests for the WhatsApp contact-list scraper (M3).

Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.

The fixture ships 3 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal):

* page 1 -> 2 records, ``totalCount=3`` (rows under ``data.overviews``)
* page 2 -> 1 record, ``totalCount=3``
* page 3 -> empty ``overviews`` array (no more rows)

The scraper must yield exactly the 3 records normalized to the ``Customer``
column shape with ``channel="whatsapp"``, ``friend_id`` <- ``friendUid`` and
``owner_account_id`` <- ``uid``.
"""

from __future__ import annotations

import json as _json

import httpx
import respx

from app.scraper.whatsapp import scrape_whatsapp_friends
from app.session.yunke_client import YunkeClient
from tests.cassettes.wa_friends_fixture import (
    DEFAULT_PAGE_SIZE,
    TOTAL,
    WA_FRIEND_RECORDS,
    make_wa_friends_response,
)

BASE = "https://crm2.yunkecn.com"
WA_FRIENDS_URL = f"{BASE}/qqweb/WhatsAppUser/businessFriendList"

USERID = "a" * 32
COMPANY = "8m0atd"


def _primed_client() -> YunkeClient:
    """A YunkeClient already in a logged-in state (login flow skipped)."""
    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)


# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + channel/ownership + raw_json
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_friends_yields_normalized_records_and_paginates():
    """Two full pages + a terminating empty page -> 3 records, 3 calls."""
    page1 = make_wa_friends_response(
        WA_FRIEND_RECORDS[:2], total_count=TOTAL, page=1, page_size=DEFAULT_PAGE_SIZE
    )
    page2 = make_wa_friends_response(
        WA_FRIEND_RECORDS[2:], total_count=TOTAL, page=2, page_size=DEFAULT_PAGE_SIZE
    )
    page3 = make_wa_friends_response(
        [], total_count=TOTAL, page=3, page_size=DEFAULT_PAGE_SIZE
    )

    route = respx.post(WA_FRIENDS_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
            httpx.Response(200, json=page3),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r async for r in scrape_whatsapp_friends(client, page_size=DEFAULT_PAGE_SIZE)
        ]
    finally:
        await client.aclose()

    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
    assert route.call_count == 3

    # Exactly the 3 fixture records (not duplicated, not dropped).
    assert len(results) == 3
    assert {r["friend_id"] for r in results} == {
        rec["friendUid"] for rec in WA_FRIEND_RECORDS
    }

    # Field mapping for each record (compare against the source record).
    for result, record in zip(results, WA_FRIEND_RECORDS, strict=True):
        # Identity / ownership / channel.
        assert result["friend_id"] == record["friendUid"]
        assert result["channel"] == "whatsapp"
        assert result["owner_account_id"] == record["uid"]
        assert result["user_id"] == record["userId"]

        # Naming / phone.
        assert result["friend_nickname"] == record["nickName"]
        assert result["phone"] == record["phone"]
        assert result["sale_name"] == record["salesNickName"]

        # Counts / timestamps.
        assert result["message_count"] == record["messageCount"]
        assert result["last_chat_time"] == record["lastUpdate"]

        # Version carried verbatim; raw_json preserves the original record.
        assert result["version"] == record["version"]
        assert result["raw_json"] == record


# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_friends_sends_pagination_body():
    """Outgoing request body carries pageNum/pageSize; advances per page."""
    page1 = make_wa_friends_response(
        WA_FRIEND_RECORDS[:1], total_count=TOTAL, page=1, page_size=1
    )
    page2 = make_wa_friends_response([], total_count=TOTAL, page=2, page_size=1)
    route = respx.post(WA_FRIENDS_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r
            async for r in scrape_whatsapp_friends(
                client, department_id="ROOT_DEPT", page_size=1
            )
        ]
    finally:
        await client.aclose()

    assert len(results) == 1
    assert route.call_count == 2

    body1 = _json.loads(route.calls[0].request.content.decode())
    # businessFriendList uses `page` (not pageNum) + departmentId + type.
    assert body1["page"] == 1
    assert body1["pageSize"] == 1
    assert body1["departmentId"] == "ROOT_DEPT"
    assert body1["type"] == "0"

    body2 = _json.loads(route.calls[1].request.content.decode())
    assert body2["page"] == 2
    assert body2["pageSize"] == 1


# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_whatsapp_friends_stops_on_empty_first_page():
    """If the very first page is empty, scraper stops without over-fetching."""
    empty = make_wa_friends_response([], total_count=0, page=1)
    route = respx.post(WA_FRIENDS_URL).mock(
        return_value=httpx.Response(200, json=empty)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_whatsapp_friends(client)]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1