浅念
79b89bc919
实测 queryWeChatVoiceList 必须带 callSource+departmentId+isRoom+startTime+endTime， 否则 200 但 data 空（0 数据）。WA callSource=4 / 微信 callSource=0，宽窗抓全量。 sync_*_voice_calls 照 sync_wechat_accounts 模式解析 _find_root_department_id。 Co-Authored-By: Claude <noreply@anthropic.com>
215 lines
8.0 KiB
Python
215 lines
8.0 KiB
Python
"""Tests for the WhatsApp voice-call scraper (Plan 4a Task 1).
|
|
|
|
Drives the real page-number pagination + normalization logic through
|
|
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so
|
|
the login flow is skipped) against a mocked transport.
|
|
|
|
The WA voice endpoint reuses the **unified** ``queryWeChatVoiceList``
|
|
interface (the same one the WeChat voice scraper hits) with the
|
|
WA-specific ``callSource="4"`` flag — same envelope (``data.data`` /
|
|
``data.total``), same record schema, just a different ``channel`` label
|
|
(``"wa_voice_call"``). The fixture ships 3 records across 2 pages of size
|
|
2, plus a terminating empty page (the page-number paginator's primary
|
|
stop signal):
|
|
|
|
* page 1 -> 2 records, ``total=3``
|
|
* page 2 -> 1 record, ``total=3``
|
|
* page 3 -> empty ``data`` array (no more rows)
|
|
|
|
The scraper must:
|
|
|
|
* make exactly 3 calls and stop without a 4th,
|
|
* send ``callSource=4`` + advancing ``pageNum``/``pageSize`` in the body,
|
|
* yield exactly the 3 records normalized to the ``PhoneCall`` column
|
|
shape with ``channel="wa_voice_call"``,
|
|
* preserve the WA-specific fields (``weChatId`` / ``callFileId`` /
|
|
``nextId`` / ``isRoom`` / ``from``) via ``raw_json`` (no PhoneCall
|
|
columns exist for them).
|
|
|
|
Field mapping asserted end-to-end (mirrors the WeChat voice test,
|
|
``test_scrape_wechat_voice_calls``):
|
|
|
|
* ``id`` -> ``call_id`` (PK)
|
|
* ``talker`` -> ``call_number`` (the customer WA id)
|
|
* ``talkerNickName`` -> ``plan_customer_name``
|
|
* ``ossFileName``-> ``record_file`` (may be null)
|
|
* ``startTime`` -> ``start_call_time`` (``YYYY-MM-DD HH:MM:SS`` -> ms epoch)
|
|
* ``duration`` -> ``call_duration`` (string seconds coerced to int)
|
|
* ``isSend`` -> ``incoming_call`` with INVERTED polarity (0=incoming
|
|
-> ``incoming_call=1``; 1=outgoing -> ``incoming_call=0``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import json as _json
|
|
|
|
import httpx
|
|
import respx
|
|
|
|
from app.scraper.whatsapp import scrape_wa_voice_calls
|
|
from app.session.yunke_client import YunkeClient
|
|
from tests.cassettes.wa_voice_fixture import (
|
|
DEFAULT_PAGE_SIZE,
|
|
TOTAL,
|
|
WA_VOICE_RECORDS,
|
|
make_wa_voice_list_response,
|
|
)
|
|
|
|
BASE = "https://crm2.yunkecn.com"
|
|
VOICE_URL = f"{BASE}/wechatweb/callRecord/queryWeChatVoiceList"
|
|
|
|
USERID = "a" * 32
|
|
COMPANY = "8m0atd"
|
|
|
|
|
|
def _primed_client() -> YunkeClient:
|
|
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
|
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
|
|
|
|
|
def _ms(date_str: str) -> int:
|
|
"""Expected ms-epoch for a ``YYYY-MM-DD HH:MM:SS`` string parsed as UTC.
|
|
|
|
Mirrors the scraper's documented parsing rule so the assertion is an
|
|
independent recomputation (not a self-reference).
|
|
"""
|
|
return int(
|
|
dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
|
|
.replace(tzinfo=dt.UTC)
|
|
.timestamp()
|
|
* 1000
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# End-to-end: pagination + body + field mapping + raw_json
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wa_voice_calls_yields_records_and_paginates():
|
|
"""Two pages + a terminating empty page -> 3 records, 3 calls.
|
|
|
|
Body must carry the full filter set the live endpoint REQUIRES
|
|
(``callSource=4`` + ``departmentId`` + ``isRoom`` + ``startTime`` +
|
|
``endTime`` — omit any and the endpoint returns 200 with empty
|
|
``data.data``, 0 rows) plus the standard ``pageNum``/``pageSize``
|
|
pagination pair. Each yielded record is normalized to the ``PhoneCall``
|
|
shape with ``channel="wa_voice_call"`` and the WA-specific fields
|
|
preserved via ``raw_json``.
|
|
"""
|
|
page1 = make_wa_voice_list_response(
|
|
WA_VOICE_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_wa_voice_list_response(
|
|
WA_VOICE_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page3 = make_wa_voice_list_response(
|
|
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
|
|
route = respx.post(VOICE_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
httpx.Response(200, json=page3),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wa_voice_calls(
|
|
client, department_id="DEPT_TEST", page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
|
|
assert route.call_count == 3
|
|
|
|
# Exactly the 3 fixture records (not duplicated, not dropped).
|
|
assert len(results) == len(WA_VOICE_RECORDS)
|
|
assert {r["call_id"] for r in results} == {rec["id"] for rec in WA_VOICE_RECORDS}
|
|
|
|
# First request body: full required filter set the live endpoint demands.
|
|
# callSource=4 (WA) + departmentId + isRoom="0" + wide time window +
|
|
# empty userId/sort/ids (the live endpoint accepts these blank).
|
|
body1 = _json.loads(route.calls[0].request.content.decode())
|
|
assert body1["callSource"] == "4"
|
|
assert body1["departmentId"] == "DEPT_TEST"
|
|
assert body1["isRoom"] == "0"
|
|
assert body1["startTime"] == "2000-01-01 00:00:00"
|
|
assert body1["endTime"] == "2099-12-31 23:59:59"
|
|
assert body1["pageNum"] == 1
|
|
assert body1["pageSize"] == DEFAULT_PAGE_SIZE
|
|
# The blank-but-required keys are present (the live endpoint expects
|
|
# them in the body even when empty).
|
|
assert body1["userId"] == ""
|
|
assert body1["sort"] == ""
|
|
assert body1["ids"] == []
|
|
|
|
# Second page advances pageNum and keeps the same filter set verbatim.
|
|
body2 = _json.loads(route.calls[1].request.content.decode())
|
|
assert body2["pageNum"] == 2
|
|
assert body2["pageSize"] == DEFAULT_PAGE_SIZE
|
|
assert body2["callSource"] == "4"
|
|
assert body2["departmentId"] == "DEPT_TEST"
|
|
|
|
# Field mapping for each record (compare against the source record).
|
|
for result, record in zip(results, WA_VOICE_RECORDS, strict=True):
|
|
# Identity + channel.
|
|
assert result["call_id"] == record["id"]
|
|
assert result["channel"] == "wa_voice_call"
|
|
|
|
# Staff side.
|
|
assert result["user_id"] == record["userId"]
|
|
assert result["user_id_name"] == record["userName"]
|
|
|
|
# Customer side: talker -> call_number, talkerNickName -> plan_customer_name.
|
|
assert result["call_number"] == record["talker"]
|
|
assert result["plan_customer_name"] == record["talkerNickName"]
|
|
|
|
# Recording URL (may be None).
|
|
assert result["record_file"] == record["ossFileName"]
|
|
|
|
# Status + duration + start time coercions.
|
|
assert result["call_status"] == record["callStatus"]
|
|
assert result["call_duration"] == int(record["duration"])
|
|
assert result["start_call_time"] == _ms(record["startTime"])
|
|
|
|
# incoming_call polarity: isSend 0=incoming -> 1; 1=outgoing -> 0.
|
|
assert result["incoming_call"] == 1 - record["isSend"]
|
|
|
|
# Version carried verbatim; raw_json preserves the original record
|
|
# (so WA-specific fields like weChatId / callFileId / nextId / isRoom
|
|
# / from survive end-to-end despite having no PhoneCall column).
|
|
assert result["version"] == record["version"]
|
|
assert result["raw_json"] == record
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Empty first page -> no yields, single call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wa_voice_calls_stops_on_empty_first_page():
|
|
"""If the very first page is empty, scraper stops without over-fetching."""
|
|
empty = make_wa_voice_list_response([], total=0, page_num=1)
|
|
route = respx.post(VOICE_URL).mock(
|
|
return_value=httpx.Response(200, json=empty)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [r async for r in scrape_wa_voice_calls(client)]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert results == []
|
|
assert route.call_count == 1
|