yunke/yunke-server/tests/test_scrape_calls.py

"""Tests for the phone-call + SMS scrapers (M4, ``app/scraper/call.py``).

Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.

Two endpoint suites:

* ``POST /pc/callLog/list`` (phone calls) — envelope
  ``data.data`` / ``data.totalCount`` / ``data.pageCount``. Maps to
  :class:`PhoneCall` with ``channel="phone_call"``. ``incomingCall`` polarity
  is ``1=incoming / 0=outgoing`` and is stored DIRECTLY (no inversion — unlike
  the voice-call channel's ``isSend``).
* ``POST /pc/customer/smsLogsList`` (SMS) — envelope ``data.data`` /
  ``data.dataCount`` (note: ``dataCount``, NOT ``totalCount``), no
  ``pageCount``. Maps to :class:`Sms` with ``channel="sms"`` (the model
  default).

Both fixtures ship multiple pages so the tests exercise pagination across the
empty-page stop signal end-to-end, and assert the full field mapping incl.
the two distinct datetime string formats (``YYYY-MM-DD HH:MM:SS`` for calls,
``YYYY-MM-DD HH:MM`` for SMS) parsed to ms-epoch.
"""

from __future__ import annotations

import datetime as dt
import json as _json

import httpx
import respx

from app.scraper.call import scrape_phone_calls, scrape_sms
from app.session.yunke_client import YunkeClient
from tests.cassettes.callLog_fixture import (
    CALL_RECORDS,
    TOTAL_COUNT,
    make_call_log_response,
)
from tests.cassettes.callLog_fixture import (
    DEFAULT_PAGE_SIZE as CALL_PAGE_SIZE,
)
from tests.cassettes.smsLogs_fixture import (
    DATA_COUNT,
    SMS_RECORDS,
    make_sms_logs_response,
)
from tests.cassettes.smsLogs_fixture import (
    DEFAULT_PAGE_SIZE as SMS_PAGE_SIZE,
)

BASE = "https://crm2.yunkecn.com"
CALL_LOG_URL = f"{BASE}/pc/callLog/list"
SMS_LOG_URL = f"{BASE}/pc/customer/smsLogsList"

USERID = "a" * 32
COMPANY = "8m0atd"


def _primed_client() -> YunkeClient:
    """A YunkeClient already in a logged-in state (login flow skipped)."""
    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)


def _ms_seconds(date_str: str) -> int:
    """Expected ms-epoch for a ``YYYY-MM-DD HH:MM:SS`` string parsed as UTC.

    Mirrors the callLog scraper's documented parsing rule so the assertion is
    an independent recomputation (not a self-reference).
    """
    return int(
        dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
        .replace(tzinfo=dt.UTC)
        .timestamp()
        * 1000
    )


def _ms_minutes(date_str: str) -> int:
    """Expected ms-epoch for a ``YYYY-MM-DD HH:MM`` string parsed as UTC.

    Mirrors the SMS scraper's documented parsing rule (seconds default to 00).
    """
    return int(
        dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M")
        .replace(tzinfo=dt.UTC)
        .timestamp()
        * 1000
    )


# ===========================================================================
# Phone calls — POST /pc/callLog/list
# ===========================================================================


@respx.mock
async def test_scrape_phone_calls_yields_records_and_paginates():
    """Two full pages + a terminating empty page -> 4 records, 3 calls."""
    page1 = make_call_log_response(
        CALL_RECORDS[:2], total_count=TOTAL_COUNT, page_num=1, page_size=CALL_PAGE_SIZE
    )
    page2 = make_call_log_response(
        CALL_RECORDS[2:], total_count=TOTAL_COUNT, page_num=2, page_size=CALL_PAGE_SIZE
    )
    page3 = make_call_log_response(
        [], total_count=TOTAL_COUNT, page_num=3, page_size=CALL_PAGE_SIZE
    )

    route = respx.post(CALL_LOG_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
            httpx.Response(200, json=page3),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r async for r in scrape_phone_calls(client, page_size=CALL_PAGE_SIZE)
        ]
    finally:
        await client.aclose()

    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
    assert route.call_count == 3

    # Exactly the 4 fixture records (not duplicated, not dropped).
    assert len(results) == 4
    assert {r["call_id"] for r in results} == {rec["id"] for rec in CALL_RECORDS}

    # Field mapping for each record (compare against the source record).
    for result, record in zip(results, CALL_RECORDS, strict=True):
        # Identity + channel.
        assert result["call_id"] == record["id"]
        assert result["channel"] == "phone_call"

        # Staff side.
        assert result["user_id"] == record["userId"]
        assert result["user_id_name"] == record["userIdName"]
        assert result["department_id"] == record["departmentId"]

        # Call metadata.
        assert result["call_number"] == record["callNumber"]
        assert result["sim_phone"] == record["simPhone"]
        assert result["sim_number"] == record["simNumber"]
        assert result["imei"] == record["imei"]
        assert result["plan_customer_name"] == record["planCustomerName"]

        # Recording URL (may be None).
        assert result["record_file"] == record["recordFile"]

        # Status + duration + start time coercions.
        assert result["call_status"] == record["callStatus"]
        # callSeconds preferred over callDuration string.
        assert result["call_duration"] == record["callSeconds"]
        assert result["start_call_time"] == _ms_seconds(record["startCallTime"])

        # incomingCall polarity: 1=incoming/0=outgoing — stored DIRECTLY.
        assert result["incoming_call"] == record["incomingCall"]

        # is_upload (bool field) carried; download_count + qc_status carried.
        assert result["is_upload"] == record["isUpload"]
        assert result["download_count"] == record["downloadCount"]
        assert result["qc_status"] == record["qcStatus"]
        assert result["ai_asr"] == record["aiAsr"]

        # Version carried verbatim; raw_json preserves the original record.
        assert result["version"] == record["version"]
        assert result["raw_json"] == record


@respx.mock
async def test_scrape_phone_calls_null_record_file_yields_none():
    """A record with ``recordFile=null`` -> ``record_file`` None."""
    rec = {
        "id": "pc_null",
        "startCallTime": "2026-01-15 10:00:00",
        "callSeconds": 0,
        "callStatus": 0,
        "incomingCall": 1,  # incoming
        "userId": "staff_user_001",
        "recordFile": None,
    }
    page1 = make_call_log_response([rec], total_count=1, page_num=1, page_size=CALL_PAGE_SIZE)
    page2 = make_call_log_response([], total_count=1, page_num=2, page_size=CALL_PAGE_SIZE)
    respx.post(CALL_LOG_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [
            r async for r in scrape_phone_calls(client, page_size=CALL_PAGE_SIZE)
        ]
    finally:
        await client.aclose()

    assert len(results) == 1
    assert results[0]["record_file"] is None
    assert results[0]["call_status"] == 0
    # incoming (incomingCall=1) -> incoming_call=1 (no inversion).
    assert results[0]["incoming_call"] == 1


@respx.mock
async def test_scrape_phone_calls_sends_pagination_body():
    """Outgoing request body carries pageNum/pageSize; advances per page."""
    page1 = make_call_log_response(
        CALL_RECORDS[:1], total_count=TOTAL_COUNT, page_num=1, page_size=1
    )
    page2 = make_call_log_response(
        [], total_count=TOTAL_COUNT, page_num=2, page_size=1
    )
    route = respx.post(CALL_LOG_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_phone_calls(client, page_size=1)]
    finally:
        await client.aclose()

    # One record on page 1, empty page 2 stops the loop.
    assert len(results) == 1
    assert route.call_count == 2

    body1 = _json.loads(route.calls[0].request.content.decode())
    assert body1["pageNum"] == 1
    assert body1["pageSize"] == 1

    body2 = _json.loads(route.calls[1].request.content.decode())
    assert body2["pageNum"] == 2
    assert body2["pageSize"] == 1


@respx.mock
async def test_scrape_phone_calls_stops_on_empty_first_page():
    """If the very first page is empty, scraper stops without over-fetching."""
    empty = make_call_log_response([], total_count=0, page_num=1)
    route = respx.post(CALL_LOG_URL).mock(
        return_value=httpx.Response(200, json=empty)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_phone_calls(client)]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1


# ===========================================================================
# SMS — POST /pc/customer/smsLogsList
# ===========================================================================


@respx.mock
async def test_scrape_sms_yields_records_and_paginates():
    """Two pages (2 + 1 records) + a terminating empty page -> 3 records."""
    page1 = make_sms_logs_response(SMS_RECORDS[:2], data_count=DATA_COUNT)
    page2 = make_sms_logs_response(SMS_RECORDS[2:], data_count=DATA_COUNT)
    page3 = make_sms_logs_response([], data_count=DATA_COUNT)

    route = respx.post(SMS_LOG_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
            httpx.Response(200, json=page3),
        ]
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_sms(client, page_size=SMS_PAGE_SIZE)]
    finally:
        await client.aclose()

    # Pagination: exactly 3 calls (page1, page2, empty-page stop).
    assert route.call_count == 3

    # Exactly the 3 fixture records.
    assert len(results) == 3
    assert {r["sms_id"] for r in results} == {rec["id"] for rec in SMS_RECORDS}

    # Field mapping for each record (compare against the source record).
    for result, record in zip(results, SMS_RECORDS, strict=True):
        # Identity + channel (the model's default 'sms').
        assert result["sms_id"] == record["id"]
        assert result["channel"] == "sms"

        # Phone + text.
        assert result["customer_phone"] == record["customerPhone"]
        assert result["content"] == record["sms"]
        # Empty preview string -> None (the scraper normalizes falsy -> None).
        expected_preview = record["smsSub"] or None
        assert result["content_preview"] == expected_preview

        # Parties.
        assert result["sender"] == record["sender"]
        assert result["direction"] == record["direction"]
        assert result["user_id"] == record["userId"]
        assert result["sender_department"] == record["senderDepartment"]
        assert result["sender_department_id"] == record["senderDepartmentId"]
        assert result["receive_name"] == record["receiveName"]

        # SIM + source.
        assert result["sim_phone"] == record["simPhone"]
        assert result["sim_number"] == record["simNumber"]
        assert result["source"] == record["source"]

        # createTime "YYYY-MM-DD HH:MM" -> ms epoch (seconds default :00).
        assert result["create_time"] == _ms_minutes(record["createTime"])

        # raw_json preserves the original record.
        assert result["raw_json"] == record


@respx.mock
async def test_scrape_sms_empty_preview_yields_none_content_preview():
    """An empty ``smsSub`` string -> ``content_preview`` None (full text kept)."""
    rec = {
        "id": "sms_empty_preview",
        "customerPhone": "8613800000099",
        "sms": "好的，收到。",
        "smsSub": "",
        "sender": "8613800000099",
        "direction": 0,
        "userId": "staff_user_001",
        "createTime": "2026-01-16 09:00",
    }
    page1 = make_sms_logs_response([rec], data_count=1)
    page2 = make_sms_logs_response([], data_count=1)
    respx.post(SMS_LOG_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_sms(client, page_size=SMS_PAGE_SIZE)]
    finally:
        await client.aclose()

    assert len(results) == 1
    assert results[0]["content"] == "好的，收到。"
    assert results[0]["content_preview"] is None


@respx.mock
async def test_scrape_sms_sends_pagination_body():
    """Outgoing request body carries pageNum/pageSize; advances per page."""
    page1 = make_sms_logs_response(SMS_RECORDS[:1], data_count=DATA_COUNT)
    page2 = make_sms_logs_response([], data_count=DATA_COUNT)
    route = respx.post(SMS_LOG_URL).mock(
        side_effect=[
            httpx.Response(200, json=page1),
            httpx.Response(200, json=page2),
        ]
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_sms(client, page_size=1)]
    finally:
        await client.aclose()

    assert len(results) == 1
    assert route.call_count == 2

    body1 = _json.loads(route.calls[0].request.content.decode())
    assert body1["pageNum"] == 1
    assert body1["pageSize"] == 1

    body2 = _json.loads(route.calls[1].request.content.decode())
    assert body2["pageNum"] == 2
    assert body2["pageSize"] == 1


@respx.mock
async def test_scrape_sms_stops_on_empty_first_page():
    """If the very first page is empty, scraper stops without over-fetching."""
    empty = make_sms_logs_response([], data_count=0)
    route = respx.post(SMS_LOG_URL).mock(
        return_value=httpx.Response(200, json=empty)
    )

    client = _primed_client()
    try:
        results = [r async for r in scrape_sms(client)]
    finally:
        await client.aclose()

    assert results == []
    assert route.call_count == 1