yunke/yunke-server/tests/test_scrape_departments.py

"""Tests for the department-tree scraper (Task 2.0).

Drives the real flatten + normalize logic through ``respx`` by wiring a real
``YunkeClient`` (primed with a fake session so the login flow is skipped)
against a mocked transport.

The dept/tree endpoint is a SINGLE request (no pagination). The scraper must:

* decode the envelope defensively (``data`` is the node list, OR an object
  wrapping the node list under ``children``),
* flatten BOTH a nested tree (each node carrying ``children``) AND a flat list
  of nodes (parents via ``pId``) into the same flat set of records,
* map ``id``/``pId``/``name`` to ``dept_id``/``parent_id``/``name``,
* build a materialized ``path`` by walking up the parent chain (names joined
  with ``/``; root nodes have ``path == name``),
* preserve the verbatim node under ``raw_json``.

The two-shape contract (nested vs flat) is asserted by serving the SAME four
nodes in both shapes and asserting the scraper yields the SAME result set in
both cases.
"""

from __future__ import annotations

import httpx
import respx

from app.scraper.wechat import scrape_departments
from app.session.yunke_client import YunkeClient
from tests.cassettes.dept_tree_fixture import (
    ALL_NODES,
    EAST_NODE,
    FLAT_TREE_NODES,
    NESTED_TREE_NODES,
    NORTH_NODE,
    ROOT_NODE,
    TEAMA_NODE,
    make_dept_tree_response,
)

BASE = "https://crm2.yunkecn.com"
DEPT_TREE_URL = f"{BASE}/usercenter/dept/tree"

USERID = "a" * 32
COMPANY = "8m0atd"


def _primed_client() -> YunkeClient:
    """A YunkeClient already in a logged-in state (login flow skipped)."""
    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)


def _expected_path(node: dict) -> str:
    """Compute the expected materialized path for a fixture node.

    Root (Sales) -> "Sales"; East -> "Sales/East"; TeamA -> "Sales/East/TeamA";
    North -> "Sales/North".
    """
    by_id = {n["id"]: n for n in ALL_NODES}
    parts: list[str] = []
    cur: dict | None = node
    while cur is not None:
        parts.append(cur["name"])
        pid = cur.get("pId")
        # Falsy / unknown pId terminates the walk (root reached).
        cur = by_id.get(pid) if pid else None
    return "/".join(reversed(parts))


# ---------------------------------------------------------------------------
# End-to-end: nested tree flattens correctly with paths reconstructed
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_departments_flattens_nested_tree_with_paths():
    """Nested tree -> flat list of 4 depts with correct parent_id + path."""
    payload = make_dept_tree_response(NESTED_TREE_NODES)
    route = respx.post(DEPT_TREE_URL).mock(
        return_value=httpx.Response(200, json=payload)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_departments(client)]
    finally:
        await client.aclose()

    # Single request (no pagination).
    assert route.call_count == 1

    # All four nodes yielded (order-independent).
    assert len(results) == 4
    by_dept_id = {r["dept_id"]: r for r in results}
    assert set(by_dept_id) == {n["id"] for n in ALL_NODES}

    # Field mapping + parent reconstruction for each node.
    for node in ALL_NODES:
        rec = by_dept_id[node["id"]]
        assert rec["dept_id"] == node["id"]
        assert rec["name"] == node["name"]
        # Root node's parent_id is preserved verbatim (pId="0") — the scraper
        # does not synthesize NULL; it carries pId through. Materialized-path
        # construction treats falsy/unknown pId as "stop walking".
        assert rec["parent_id"] == node["pId"]
        assert rec["path"] == _expected_path(node)
        # raw_json preserves the original (flattened) node — keys id/pId/name.
        assert rec["raw_json"]["id"] == node["id"]
        assert rec["raw_json"]["name"] == node["name"]

    # Spot-check the headline path expectations (defensive against path-helper
    # bugs masking themselves).
    assert by_dept_id[ROOT_NODE["id"]]["path"] == "Sales"
    assert by_dept_id[EAST_NODE["id"]]["path"] == "Sales/East"
    assert by_dept_id[TEAMA_NODE["id"]]["path"] == "Sales/East/TeamA"
    assert by_dept_id[NORTH_NODE["id"]]["path"] == "Sales/North"


# ---------------------------------------------------------------------------
# Flat list of nodes (parents via pId) yields the SAME result as nested
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_departments_flattens_flat_list_to_same_result():
    """Flat node list (no children arrays) flattens to the same 4 records."""
    payload = make_dept_tree_response(FLAT_TREE_NODES)
    respx.post(DEPT_TREE_URL).mock(return_value=httpx.Response(200, json=payload))

    client = _primed_client()
    try:
        results = [r async for r in scrape_departments(client)]
    finally:
        await client.aclose()

    assert len(results) == 4
    by_dept_id = {r["dept_id"]: r for r in results}
    for node in ALL_NODES:
        rec = by_dept_id[node["id"]]
        assert rec["parent_id"] == node["pId"]
        assert rec["path"] == _expected_path(node)


# ---------------------------------------------------------------------------
# Defensive envelope: data is an OBJECT wrapping nodes under `children`
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_departments_unwraps_data_object_children():
    """When ``data`` is an object with ``children``, scraper unwraps it."""
    payload = make_dept_tree_response(
        FLAT_TREE_NODES, nested_in_data_object=True
    )
    respx.post(DEPT_TREE_URL).mock(return_value=httpx.Response(200, json=payload))

    client = _primed_client()
    try:
        results = [r async for r in scrape_departments(client)]
    finally:
        await client.aclose()

    assert len(results) == 4
    assert {r["dept_id"] for r in results} == {n["id"] for n in ALL_NODES}


# ---------------------------------------------------------------------------
# Empty tree -> no yields, single call (no infinite loop on missing data)
# ---------------------------------------------------------------------------


@respx.mock
async def test_scrape_departments_empty_tree_yields_nothing():
    """Empty data array -> no yields, exactly one call."""
    payload = make_dept_tree_response([])
    route = respx.post(DEPT_TREE_URL).mock(
        return_value=httpx.Response(200, json=payload)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_departments(client)]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1