yunke/yunke-server/tests/test_scrape_wechat_accounts.py

"""Tests for the WeChat staff-account scraper (Task 2.1).

Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.

The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal for page-number endpoints):

* page 1 -> 2 records, ``total=4``
* page 2 -> 2 records, ``total=4``
* page 3 -> empty ``data`` array (no more rows)

The scraper must yield exactly the 4 records (normalized to ``WechatAccount``
snake_case columns + ``raw_json``), make exactly 3 calls, and stop without a
4th. The empty-page stop is the primary signal; ``total=4`` is only a
defensive over-fetch ceiling that must NOT trigger early (we yield exactly
4 == total, so the strict-``>`` ceiling never fires here).
"""

from __future__ import annotations

import json as _json

import httpx
import respx

from app.scraper.wechat import scrape_wechat_accounts
from app.session.yunke_client import YunkeClient
from tests.cassettes.wxUser_fixture import (
    DEFAULT_PAGE_SIZE,
    TOTAL,
    WXUSER_RECORDS,
    make_wxuser_list_response,
)

BASE = "https://crm2.yunkecn.com"
WXUSER_LIST_URL = f"{BASE}/wechatweb/wxUser/list"

USERID = "a" * 32
COMPANY = "8m0atd"


def _primed_client() -> YunkeClient:
    """A YunkeClient already in a logged-in state (login flow skipped)."""
    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)


# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + raw_json
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_wechat_accounts_yields_records_and_paginates():
    """Two full pages + a terminating empty page -> 4 records, 3 calls."""
    page1 = make_wxuser_list_response(
        WXUSER_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
    )
    page2 = make_wxuser_list_response(
        WXUSER_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
    )
    page3 = make_wxuser_list_response(
        [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
    )

    route = respx.post(WXUSER_LIST_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
            httpx.Response(200, json=page3),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r
            async for r in scrape_wechat_accounts(
                client, page_size=DEFAULT_PAGE_SIZE
            )
        ]
    finally:
        await client.aclose()

    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
    assert route.call_count == 3

    # Exactly the 4 fixture records (not duplicated, not dropped).
    assert len(results) == 4
    assert {r["wx_id"] for r in results} == {
        rec["wechatId"] for rec in WXUSER_RECORDS
    }

    # Field mapping for each record (compare against the source record).
    for result, record in zip(results, WXUSER_RECORDS, strict=True):
        # Identity.
        assert result["wx_id"] == record["wechatId"]

        # Naming.
        assert result["nick_name"] == record["nickName"]
        assert result["alias"] == record["alias"]

        # Phones.
        assert result["phone"] == record["phone"]

        # Embedded staff fields (the owning employee).
        assert result["user_id"] == record["userId"]
        assert result["user_name"] == record["userName"]

        # Counts.
        assert result["contacts_num"] == record["contactsNum"]
        assert result["group_num"] == record["groupNum"]
        assert result["send_num"] == record["sendNum"]
        assert result["receive_num"] == record["receiveNum"]

        # Status / sizing / timestamps.
        assert result["is_on_job"] == record["isOnJob"]
        assert result["db_size"] == record["dbSize"]
        assert result["online_status"] == record["onlineStatus"]
        assert result["last_update"] == record["lastUpdate"]

        # Version carried verbatim; raw_json preserves the original record.
        assert result["version"] == record["version"]
        assert result["raw_json"] == record


# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_wechat_accounts_sends_pagination_body():
    """Outgoing request body carries pageNum/pageSize; advances per page."""
    page1 = make_wxuser_list_response(
        WXUSER_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
    )
    page2 = make_wxuser_list_response(
        [], total=TOTAL, page_num=2, page_size=1
    )
    route = respx.post(WXUSER_LIST_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r async for r in scrape_wechat_accounts(client, page_size=1)
        ]
    finally:
        await client.aclose()

    # One record on page 1, empty page 2 stops the loop.
    assert len(results) == 1
    assert route.call_count == 2

    # First request: pageNum=1.
    body1 = _json.loads(route.calls[0].request.content.decode())
    assert body1["pageNum"] == 1
    assert body1["pageSize"] == 1

    # Second request: pageNum advanced to 2.
    body2 = _json.loads(route.calls[1].request.content.decode())
    assert body2["pageNum"] == 2
    assert body2["pageSize"] == 1


# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_wechat_accounts_stops_on_empty_first_page():
    """If the very first page is empty, scraper stops without over-fetching."""
    empty = make_wxuser_list_response([], total=0, page_num=1)
    route = respx.post(WXUSER_LIST_URL).mock(
        return_value=httpx.Response(200, json=empty)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_wechat_accounts(client)]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1