yunke/yunke-server/tests/test_scrape_wechat_accounts.py

"""Tests for the WeChat staff-account scraper (Task 2.1).

Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.

The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the documented primary stop signal for page-number endpoints):

* page 1 -> 2 records, ``total=4``
* page 2 -> 2 records, ``total=4``
* page 3 -> empty ``data`` array (no more rows)

The scraper must yield exactly the 4 records (normalized to ``WechatAccount``
snake_case columns + ``raw_json``), make exactly 3 calls, and stop without a
4th. The empty-page stop is the primary signal; ``total=4`` is only a
defensive over-fetch ceiling that must NOT trigger early (we yield exactly
4 == total, so the strict-``>`` ceiling never fires here).

The REQUEST body is form-urlencoded (NOT JSON) and must carry the required
``isOnJob`` / ``departmentId`` / ``startdate`` / ``enddate`` filters; the
"send pagination body" test asserts those keys land on the wire.
"""

from __future__ import annotations

from urllib.parse import parse_qs

import httpx
import respx

from app.scraper.base import FORM_CONTENT_TYPE
from app.scraper.wechat import scrape_wechat_accounts
from app.session.yunke_client import YunkeClient
from tests.cassettes.wxUser_fixture import (
    DEFAULT_PAGE_SIZE,
    TOTAL,
    WXUSER_RECORDS,
    make_wxuser_list_response,
)

BASE = "https://crm2.yunkecn.com"
WXUSER_LIST_URL = f"{BASE}/wechatweb/wxUser/list"

USERID = "a" * 32
COMPANY = "8m0atd"

# The company-root department id used as the required ``departmentId`` filter.
# Mirrors the shape of the real root dept id (32 hex chars) without hardcoding
# the live value in the scraper itself.
ROOT_DEPARTMENT_ID = "0E9C4B803E134C7687CBA824886319DA"


def _primed_client() -> YunkeClient:
    """A YunkeClient already in a logged-in state (login flow skipped)."""
    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)


def _parse_form(request: httpx.Request) -> dict[str, str]:
    """Decode a form-urlencoded request body into a flat ``{key: value}`` dict.

    ``parse_qs`` returns lists per key; the wxUser body has no repeated keys,
    so we collapse each to its single value for ergonomic assertions.
    ``keep_blank_values=True`` so the always-empty ``userId`` / ``salesname``
    filters (sent as ``userId=``) survive decoding and can be asserted on.
    """
    body = request.content.decode()
    parsed = parse_qs(body, keep_blank_values=True)
    return {k: v[0] for k, v in parsed.items()}


# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + raw_json
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_wechat_accounts_yields_records_and_paginates():
    """Two full pages + a terminating empty page -> 4 records, 3 calls."""
    page1 = make_wxuser_list_response(
        WXUSER_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
    )
    page2 = make_wxuser_list_response(
        WXUSER_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
    )
    page3 = make_wxuser_list_response(
        [], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
    )

    route = respx.post(WXUSER_LIST_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
            httpx.Response(200, json=page3),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r
            async for r in scrape_wechat_accounts(
                client,
                department_id=ROOT_DEPARTMENT_ID,
                page_size=DEFAULT_PAGE_SIZE,
            )
        ]
    finally:
        await client.aclose()

    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
    assert route.call_count == 3

    # The required form filters ride on every request (the live endpoint
    # answers code -90000 without them).
    for call in route.calls:
        body = _parse_form(call.request)
        assert body["departmentId"] == ROOT_DEPARTMENT_ID
        assert body["isOnJob"] == "1"
        assert body["startdate"]
        assert body["enddate"]
        assert call.request.headers["content-type"] == FORM_CONTENT_TYPE

    # Exactly the 4 fixture records (not duplicated, not dropped).
    assert len(results) == 4
    assert {r["wx_id"] for r in results} == {
        rec["wechatId"] for rec in WXUSER_RECORDS
    }

    # Field mapping for each record (compare against the source record).
    for result, record in zip(results, WXUSER_RECORDS, strict=True):
        # Identity.
        assert result["wx_id"] == record["wechatId"]

        # Naming.
        assert result["nick_name"] == record["nickName"]
        assert result["alias"] == record["alias"]

        # Phones.
        assert result["phone"] == record["phone"]

        # Embedded staff fields (the owning employee).
        assert result["user_id"] == record["userId"]
        assert result["user_name"] == record["userName"]

        # Counts.
        assert result["contacts_num"] == record["contactsNum"]
        assert result["group_num"] == record["groupNum"]
        assert result["send_num"] == record["sendNum"]
        assert result["receive_num"] == record["receiveNum"]

        # Status / sizing / timestamps.
        assert result["is_on_job"] == record["isOnJob"]
        assert result["db_size"] == record["dbSize"]
        assert result["online_status"] == record["onlineStatus"]
        assert result["last_update"] == record["lastUpdate"]

        # Version carried verbatim; raw_json preserves the original record.
        assert result["version"] == record["version"]
        assert result["raw_json"] == record


# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize advanced correctly
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_wechat_accounts_sends_pagination_body():
    """Outgoing form body carries pageNum/pageSize + the required filters; the
    page number advances per request."""
    page1 = make_wxuser_list_response(
        WXUSER_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
    )
    page2 = make_wxuser_list_response(
        [], total=TOTAL, page_num=2, page_size=1
    )
    route = respx.post(WXUSER_LIST_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r
            async for r in scrape_wechat_accounts(
                client,
                department_id=ROOT_DEPARTMENT_ID,
                page_size=1,
                start="2026-07-01",
                end="2026-07-08",
            )
        ]
    finally:
        await client.aclose()

    # One record on page 1, empty page 2 stops the loop.
    assert len(results) == 1
    assert route.call_count == 2

    # Every request is form-urlencoded (NOT JSON) and carries the required
    # filters the live endpoint demands (isOnJob / departmentId / startdate /
    # enddate), plus the always-empty free-text filters from the real client.
    for call in route.calls:
        assert call.request.headers["content-type"] == FORM_CONTENT_TYPE
        body = _parse_form(call.request)
        assert body["isOnJob"] == "1"
        assert body["departmentId"] == ROOT_DEPARTMENT_ID
        assert body["startdate"] == "2026-07-01"
        assert body["enddate"] == "2026-07-08"
        assert body["userId"] == ""
        assert body["salesname"] == ""
        assert body["weChatNickName"] == ""

    # pageNum/pageSize advance correctly across the form body.
    body1 = _parse_form(route.calls[0].request)
    assert body1["pageNum"] == "1"
    assert body1["pageSize"] == "1"

    body2 = _parse_form(route.calls[1].request)
    assert body2["pageNum"] == "2"
    assert body2["pageSize"] == "1"


# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_wechat_accounts_stops_on_empty_first_page():
    """If the very first page is empty, scraper stops without over-fetching."""
    empty = make_wxuser_list_response([], total=0, page_num=1)
    route = respx.post(WXUSER_LIST_URL).mock(
        return_value=httpx.Response(200, json=empty)
    )

    client = _primed_client()
    try:
        results = [
            r
            async for r in scrape_wechat_accounts(
                client, department_id=ROOT_DEPARTMENT_ID
            )
        ]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1