feat: phone-call + sms scrapers
M4 (/pc/ CRM module): mirror phone-call + SMS endpoints. Scraper (app/scraper/call.py): - scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call') Envelope data.data / data.totalCount / data.pageCount; page-number pagination. incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike voice-call isSend); callSeconds preferred over legacy callDuration string; startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch. - scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms') Envelope data.data / data.dataCount (NOT totalCount), no pageCount; relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch; empty smsSub normalized to None content_preview. Shared helper: extracted parse_datetime_to_ms(value, formats) to app/scraper/parsers.py and refactored _parse_voice_start_time to delegate (DRY across all three call/SMS datetime sources). Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id / sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion, matching start_call_time's existing treatment. Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert + SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types 'phone_call' and 'sms'. Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures exercising pagination + full field mapping), repo idempotency + coercion tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers. ruff clean. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,210 @@
|
||||
"""Fixture payloads for ``POST /pc/callLog/list`` (M4 phone-call scraper).
|
||||
|
||||
These records mirror the real field names confirmed by Phase-0 + the data
|
||||
dictionary §4 (电话记录 phone_call): ``id, startCallTime(YYYY-MM-DD HH:MM:SS),
|
||||
callDuration("M'SS\"")/callSeconds(int秒), callStatus(0未接/2接通),
|
||||
incomingCall(0呼出/1呼入), userId/userIdName(员工), departmentId, callNumber,
|
||||
simPhone, simNumber, imei, recordFile(录音OSS URL, may be null), isUpload,
|
||||
downloadCount, aiAsr, qcStatus, planCustomerName, version``. Live ``totalCount``
|
||||
is 6047 across ``pageCount=303`` pages.
|
||||
|
||||
The envelope differs from the wechat ``data.data`` / ``data.total`` shape: the
|
||||
row list sits under ``data.data`` (same) but the declared row count is
|
||||
``data.totalCount`` (NOT ``data.total``) and the page count is ``data.pageCount``::
|
||||
|
||||
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
||||
"data":{"data":[...], "page":.., "pageCount":.., "totalCount":..,
|
||||
"phoneHidden":false, "hideRecordPlay":false}}
|
||||
|
||||
The cassette ships four records across two pages so a test can drive the
|
||||
page-number paginator across the empty-page stop signal:
|
||||
|
||||
* page 1 -> 2 of the 4 fixture records, ``totalCount=4``
|
||||
* page 2 -> the other 2 records, ``totalCount=4``
|
||||
* (page 3 -> empty ``data`` array, served inline by the test)
|
||||
|
||||
Records deliberately exercise the documented mapping edges:
|
||||
|
||||
* ``pc_2001`` — outgoing connected call (``incomingCall=0``; ``callStatus=2``;
|
||||
``callSeconds=151``; non-null ``recordFile``; full SIM/IMEI metadata).
|
||||
* ``pc_2002`` — incoming no-answer call with NO recording (``incomingCall=1``;
|
||||
``callStatus=0``; ``recordFile=null``).
|
||||
* ``pc_2003`` — incoming connected call with a recording + AI ASR + QC data.
|
||||
* ``pc_2004`` — outgoing connected call (exercises the other polarity pair).
|
||||
|
||||
NOTE on ``incomingCall`` polarity: this API's ``incomingCall`` is
|
||||
``1=incoming / 0=outgoing`` — the SAME polarity as ``PhoneCall.incoming_call``
|
||||
(no inversion needed, unlike the voice-call channel's ``isSend``).
|
||||
"""
|
||||
|
||||
# The module name mirrors the upstream endpoint name (``callLog``); the M4 spec
|
||||
# uses this naming family.
|
||||
# ruff: noqa: N999
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Capture-shaped phone-call records (4 total across 2 pages).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CALL_RECORDS: list[dict[str, Any]] = [
|
||||
{
|
||||
# Outgoing connected call with a recording.
|
||||
"id": "pc_2001",
|
||||
"startCallTime": "2026-01-15 10:30:00", # -> ms epoch UTC
|
||||
"callSeconds": 151, # preferred over callDuration string
|
||||
"callDuration": "2'31\"", # legacy "M'SS\"" string; scraper prefers callSeconds
|
||||
"callStatus": 2, # 2=接通 (connected)
|
||||
"incomingCall": 0, # 0=呼出 (outgoing) -> incoming_call=0
|
||||
"userId": "staff_user_001",
|
||||
"userIdName": "Alice Wang",
|
||||
"departmentId": "dept_sales_east",
|
||||
"callNumber": "8613800000001",
|
||||
"simPhone": "8613800000111",
|
||||
"simNumber": "8986000112345678901",
|
||||
"imei": "356938035643801",
|
||||
"recordFile": (
|
||||
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/phone-call/"
|
||||
"8m0atd/imei001/1700000000000_1700000151000.mp3"
|
||||
),
|
||||
"isUpload": 1,
|
||||
"downloadCount": 3,
|
||||
"aiAsr": "您好，请问您方便了解一下我们的产品吗？",
|
||||
"qcStatus": 1,
|
||||
"planCustomerName": "VIP Customer A",
|
||||
"version": "v1.20260731.pc2001",
|
||||
},
|
||||
{
|
||||
# Incoming no-answer call, no recording.
|
||||
"id": "pc_2002",
|
||||
"startCallTime": "2026-01-15 11:00:00",
|
||||
"callSeconds": 0,
|
||||
"callDuration": "0'0\"",
|
||||
"callStatus": 0, # 0=未接 (no answer)
|
||||
"incomingCall": 1, # 1=呼入 (incoming) -> incoming_call=1
|
||||
"userId": "staff_user_001",
|
||||
"userIdName": "Alice Wang",
|
||||
"departmentId": "dept_sales_east",
|
||||
"callNumber": "8613800000002",
|
||||
"simPhone": "8613800000111",
|
||||
"simNumber": "8986000112345678901",
|
||||
"imei": "356938035643801",
|
||||
"recordFile": None, # no recording -> record_file None
|
||||
"isUpload": 0,
|
||||
"downloadCount": 0,
|
||||
"aiAsr": None,
|
||||
"qcStatus": 0,
|
||||
"planCustomerName": "Lead B",
|
||||
"version": "v1.20260731.pc2002",
|
||||
},
|
||||
{
|
||||
# Incoming connected call with recording + AI ASR text.
|
||||
"id": "pc_2003",
|
||||
"startCallTime": "2026-01-15 12:00:00",
|
||||
"callSeconds": 45,
|
||||
"callDuration": "0'45\"",
|
||||
"callStatus": 2, # connected
|
||||
"incomingCall": 1, # incoming -> incoming_call=1
|
||||
"userId": "staff_user_002",
|
||||
"userIdName": "Bob Chen",
|
||||
"departmentId": "dept_sales_west",
|
||||
"callNumber": "8613800000003",
|
||||
"simPhone": "8613800000222",
|
||||
"simNumber": "8986000223456789012",
|
||||
"imei": "356938035643802",
|
||||
"recordFile": (
|
||||
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/phone-call/"
|
||||
"8m0atd/imei002/1700000003000_1700000045000.mp3"
|
||||
),
|
||||
"isUpload": 1,
|
||||
"downloadCount": 1,
|
||||
"aiAsr": "好的，我考虑一下。",
|
||||
"qcStatus": 2,
|
||||
"planCustomerName": "Customer C",
|
||||
"version": "v1.20260731.pc2003",
|
||||
},
|
||||
{
|
||||
# Outgoing connected call (the other polarity pair).
|
||||
"id": "pc_2004",
|
||||
"startCallTime": "2026-01-16 09:00:00",
|
||||
"callSeconds": 207,
|
||||
"callDuration": "3'27\"",
|
||||
"callStatus": 2, # connected
|
||||
"incomingCall": 0, # outgoing -> incoming_call=0
|
||||
"userId": "staff_user_001",
|
||||
"userIdName": "Alice Wang",
|
||||
"departmentId": "dept_sales_east",
|
||||
"callNumber": "8613800000004",
|
||||
"simPhone": "8613800000111",
|
||||
"simNumber": "8986000112345678901",
|
||||
"imei": "356938035643801",
|
||||
"recordFile": (
|
||||
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/phone-call/"
|
||||
"8m0atd/imei001/1700000005000_1700000212000.mp3"
|
||||
),
|
||||
"isUpload": 1,
|
||||
"downloadCount": 2,
|
||||
"aiAsr": None,
|
||||
"qcStatus": 0,
|
||||
"planCustomerName": "Customer D",
|
||||
"version": "v1.20260731.pc2004",
|
||||
},
|
||||
]
|
||||
|
||||
# Default page size for the fixture (and the scraper). totalCount=4 means the
|
||||
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
|
||||
TOTAL_COUNT = len(CALL_RECORDS)
|
||||
DEFAULT_PAGE_SIZE = 2
|
||||
# Mirrors the live capture's declared page count (purely cosmetic for the
|
||||
# fixture; the paginator relies on the empty-page stop signal, not pageCount).
|
||||
PAGE_COUNT = 2
|
||||
|
||||
|
||||
def make_call_log_response(
|
||||
records: list[dict[str, Any]],
|
||||
*,
|
||||
total_count: int,
|
||||
page_num: int = 1,
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
page_count: int | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a capture-shaped ``callLog/list`` response envelope.
|
||||
|
||||
Shape (row list under ``data.data``; row count on ``data.totalCount``;
|
||||
page count on ``data.pageCount``)::
|
||||
|
||||
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
||||
"data":{"data":[...], "page":.., "pageCount":.., "totalCount":..,
|
||||
"phoneHidden":false, "hideRecordPlay":false}}
|
||||
"""
|
||||
return {
|
||||
"message": "调用成功",
|
||||
"code": "10000",
|
||||
"success": True,
|
||||
"traceId": trace_id or f"trace-{page_num}",
|
||||
"data": {
|
||||
"data": list(records),
|
||||
"page": page_num,
|
||||
"pageCount": page_count if page_count is not None else page_count_for(page_size, total_count),
|
||||
"totalCount": total_count,
|
||||
"phoneHidden": False,
|
||||
"hideRecordPlay": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def page_count_for(page_size: int, total_count: int) -> int:
|
||||
"""Compute the declared page count for a fixture (ceil(total/size)).
|
||||
|
||||
Helper so the envelope's ``pageCount`` is internally consistent with the
|
||||
declared ``totalCount`` + page size without forcing each test to recompute
|
||||
it. The paginator does NOT consume ``pageCount`` (it relies on the
|
||||
empty-page stop signal), so this is cosmetic — but it keeps the fixture
|
||||
capture-shaped.
|
||||
"""
|
||||
if page_size <= 0:
|
||||
return 0
|
||||
return (total_count + page_size - 1) // page_size
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Fixture payloads for ``POST /pc/customer/smsLogsList`` (M4 SMS scraper).
|
||||
|
||||
These records mirror the real field names confirmed by Phase-0 + the data
|
||||
dictionary §4 (短信记录 sms): ``id, customerPhone, sms(全文)/smsSub(预览),
|
||||
sender, direction(0收/1发), userId, senderDepartment/senderDepartmentId,
|
||||
receiveName, createTime(YYYY-MM-DD HH:MM), simPhone/simNumber, source``. Live
|
||||
``dataCount`` is 53.
|
||||
|
||||
The envelope differs from the callLog + wechat shapes: the row list sits under
|
||||
``data.data`` (same) but the declared row count is ``data.dataCount`` (NOT
|
||||
``totalCount`` and NOT ``total``), and there is **no** ``pageCount``::
|
||||
|
||||
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
||||
"data":{"data":[...], "dataCount":.., "power":...}}
|
||||
|
||||
Because there is no ``pageCount``, the scraper relies solely on the empty-page
|
||||
stop signal (advance ``pageNum`` until a page comes back with an empty
|
||||
``data.data`` array).
|
||||
|
||||
The cassette ships three records across two pages so a test can drive the
|
||||
page-number paginator across the empty-page stop signal:
|
||||
|
||||
* page 1 -> 2 of the 3 fixture records, ``dataCount=3``
|
||||
* page 2 -> the remaining 1 record, ``dataCount=3``
|
||||
* (page 3 -> empty ``data`` array, served inline by the test)
|
||||
|
||||
Records deliberately exercise the documented mapping edges:
|
||||
|
||||
* ``sms_3001`` — incoming SMS (``direction=0`` 收) with full + preview text.
|
||||
* ``sms_3002`` — outgoing SMS (``direction=1`` 发) to a different customer.
|
||||
* ``sms_3003`` — incoming SMS with an empty preview (exercises the
|
||||
``content_preview`` None path; ``sms`` still carries the full text).
|
||||
|
||||
``createTime`` is a ``YYYY-MM-DD HH:MM`` datetime string (NO seconds — one
|
||||
format precision shorter than the callLog's ``startCallTime``); the scraper
|
||||
parses it to a ms-epoch int treating the missing seconds as ``:00``.
|
||||
"""
|
||||
|
||||
# The module name mirrors the upstream endpoint name (``smsLogsList`` ->
|
||||
# ``smsLogs``); the M4 spec uses this naming family.
|
||||
# ruff: noqa: N999
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Capture-shaped SMS records (3 total across 2 pages: page1=2, page2=1).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SMS_RECORDS: list[dict[str, Any]] = [
|
||||
{
|
||||
# Incoming SMS (0=收) with full + preview text.
|
||||
"id": "sms_3001",
|
||||
"customerPhone": "8613800000001",
|
||||
"sms": "您好，我在网上看到您的产品介绍，想了解一下价格和详情，方便回电吗？",
|
||||
"smsSub": "您好，我在网上看到您的产品介绍...",
|
||||
"sender": "8613800000001",
|
||||
"direction": 0, # 0=收 (incoming)
|
||||
"userId": "staff_user_001",
|
||||
"senderDepartment": "销售一部",
|
||||
"senderDepartmentId": "dept_sales_east",
|
||||
"receiveName": "Alice Wang",
|
||||
"createTime": "2026-01-15 10:30", # YYYY-MM-DD HH:MM (no seconds)
|
||||
"simPhone": "8613800000111",
|
||||
"simNumber": "8986000112345678901",
|
||||
"source": "card",
|
||||
},
|
||||
{
|
||||
# Outgoing SMS (1=发) to a different customer.
|
||||
"id": "sms_3002",
|
||||
"customerPhone": "8613800000002",
|
||||
"sms": "王先生您好，这是您咨询的订单详情，请查收附件。",
|
||||
"smsSub": "王先生您好，这是您咨询的订单详情...",
|
||||
"sender": "system",
|
||||
"direction": 1, # 1=发 (outgoing)
|
||||
"userId": "staff_user_002",
|
||||
"senderDepartment": "销售二部",
|
||||
"senderDepartmentId": "dept_sales_west",
|
||||
"receiveName": "Bob Chen",
|
||||
"createTime": "2026-01-15 11:00",
|
||||
"simPhone": "8613800000222",
|
||||
"simNumber": "8986000223456789012",
|
||||
"source": "manual",
|
||||
},
|
||||
{
|
||||
# Incoming SMS with an EMPTY preview (content_preview None path).
|
||||
"id": "sms_3003",
|
||||
"customerPhone": "8613800000003",
|
||||
"sms": "好的，收到。",
|
||||
"smsSub": "", # empty preview -> content_preview None
|
||||
"sender": "8613800000003",
|
||||
"direction": 0, # incoming
|
||||
"userId": "staff_user_001",
|
||||
"senderDepartment": "销售一部",
|
||||
"senderDepartmentId": "dept_sales_east",
|
||||
"receiveName": "Alice Wang",
|
||||
"createTime": "2026-01-16 09:00",
|
||||
"simPhone": "8613800000111",
|
||||
"simNumber": "8986000112345678901",
|
||||
"source": "reply",
|
||||
},
|
||||
]
|
||||
|
||||
# Default page size for the fixture (and the scraper). dataCount=3 with
|
||||
# page_size=2 fills page 1 (2 records) + page 2 (1 record) + a terminating
|
||||
# empty page.
|
||||
DATA_COUNT = len(SMS_RECORDS)
|
||||
DEFAULT_PAGE_SIZE = 2
|
||||
|
||||
|
||||
def make_sms_logs_response(
|
||||
records: list[dict[str, Any]],
|
||||
*,
|
||||
data_count: int,
|
||||
trace_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a capture-shaped ``smsLogsList`` response envelope.
|
||||
|
||||
Shape (row list under ``data.data``; row count on ``data.dataCount``;
|
||||
no ``pageCount`` / ``pageNum``)::
|
||||
|
||||
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
|
||||
"data":{"data":[...], "dataCount":.., "power":true}}
|
||||
"""
|
||||
return {
|
||||
"message": "调用成功",
|
||||
"code": "10000",
|
||||
"success": True,
|
||||
"traceId": trace_id or "trace-1",
|
||||
"data": {
|
||||
"data": list(records),
|
||||
"dataCount": data_count,
|
||||
"power": True,
|
||||
},
|
||||
}
|
||||
@@ -518,3 +518,164 @@ async def test_sync_all_wechat_runs_each_entity_then_messages(monkeypatch):
|
||||
assert all(r.status == "success" for r in results)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Phone calls + SMS — full re-scrape entities (M4)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
async def test_phone_calls_full_rescrape_upserts_all_records_sync_run_and_watermark(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Phone-call list has no `since` param: 'incremental' = re-scrape all and
|
||||
rely on upsert idempotency. The orchestrator upserts every record, records
|
||||
a success SyncRun (entity_type='phone_call'), and stamps the watermark's
|
||||
last_synced_at."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
call_recs = [
|
||||
{"call_id": "pc1", "channel": "phone_call", "version": "v1"},
|
||||
{"call_id": "pc2", "channel": "phone_call", "version": "v1"},
|
||||
]
|
||||
|
||||
async def fake_scrape_phone_calls(client):
|
||||
for r in call_recs:
|
||||
yield r
|
||||
|
||||
upserted: list[dict] = []
|
||||
|
||||
async def fake_upsert_phone_call(session, rec):
|
||||
upserted.append(rec)
|
||||
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "scrape_phone_calls", fake_scrape_phone_calls
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "upsert_phone_call", fake_upsert_phone_call
|
||||
)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
result = await orchestrator.sync_phone_calls(session, client=object())
|
||||
await session.commit()
|
||||
|
||||
assert result.status == "success"
|
||||
assert result.fetched == 2
|
||||
assert result.upserted == 2
|
||||
assert [r["call_id"] for r in upserted] == ["pc1", "pc2"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = (
|
||||
await session.execute(
|
||||
sa.select(SyncRun).where(SyncRun.entity_type == "phone_call")
|
||||
)
|
||||
).scalar_one()
|
||||
assert run.status == "success"
|
||||
assert run.records_fetched == 2
|
||||
assert run.records_upserted == 2
|
||||
assert run.error is None
|
||||
|
||||
state = (
|
||||
await session.execute(
|
||||
sa.select(SyncState).where(
|
||||
SyncState.entity_type == "phone_call"
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
assert state.last_synced_at is not None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_sms_full_rescrape_upserts_all_records_sync_run_and_watermark(
|
||||
monkeypatch,
|
||||
):
|
||||
"""SMS list has no `since` param: same full-re-scrape shape as phone calls.
|
||||
Records a success SyncRun (entity_type='sms') and stamps the watermark."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
sms_recs = [
|
||||
{"sms_id": "sms1", "channel": "sms", "content": "a"},
|
||||
{"sms_id": "sms2", "channel": "sms", "content": "b"},
|
||||
{"sms_id": "sms3", "channel": "sms", "content": "c"},
|
||||
]
|
||||
|
||||
async def fake_scrape_sms(client):
|
||||
for r in sms_recs:
|
||||
yield r
|
||||
|
||||
upserted: list[dict] = []
|
||||
|
||||
async def fake_upsert_sms(session, rec):
|
||||
upserted.append(rec)
|
||||
|
||||
monkeypatch.setattr(orchestrator, "scrape_sms", fake_scrape_sms)
|
||||
monkeypatch.setattr(orchestrator, "upsert_sms", fake_upsert_sms)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
result = await orchestrator.sync_sms(session, client=object())
|
||||
await session.commit()
|
||||
|
||||
assert result.status == "success"
|
||||
assert result.fetched == 3
|
||||
assert result.upserted == 3
|
||||
assert [r["sms_id"] for r in upserted] == ["sms1", "sms2", "sms3"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = (
|
||||
await session.execute(
|
||||
sa.select(SyncRun).where(SyncRun.entity_type == "sms")
|
||||
)
|
||||
).scalar_one()
|
||||
assert run.status == "success"
|
||||
assert run.records_fetched == 3
|
||||
assert run.records_upserted == 3
|
||||
assert run.error is None
|
||||
|
||||
state = (
|
||||
await session.execute(
|
||||
sa.select(SyncState).where(SyncState.entity_type == "sms")
|
||||
)
|
||||
).scalar_one()
|
||||
assert state.last_synced_at is not None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_failing_phone_call_scraper_records_failed_run(monkeypatch):
|
||||
"""Same failure policy as the wechat entities: a scraper that raises
|
||||
records a failed SyncRun (entity_type='phone_call') and returns a failed
|
||||
result without re-raising."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
async def fake_scrape_phone_calls(client):
|
||||
raise RuntimeError("callLog 500")
|
||||
yield {} # pragma: no cover - make this an async generator
|
||||
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "scrape_phone_calls", fake_scrape_phone_calls
|
||||
)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
result = await orchestrator.sync_phone_calls(session, client=object())
|
||||
await session.commit()
|
||||
|
||||
assert result.status == "failed"
|
||||
assert "callLog 500" in (result.error or "")
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = (
|
||||
await session.execute(
|
||||
sa.select(SyncRun).where(SyncRun.entity_type == "phone_call")
|
||||
)
|
||||
).scalar_one()
|
||||
assert run.status == "failed"
|
||||
assert "callLog 500" in (run.error or "")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
@@ -29,6 +29,7 @@ from app.db.models import (
|
||||
GroupMember,
|
||||
Message,
|
||||
PhoneCall,
|
||||
Sms,
|
||||
Staff,
|
||||
WechatAccount,
|
||||
WhatsappAccount,
|
||||
@@ -41,6 +42,8 @@ from app.storage.repos import (
|
||||
upsert_customers,
|
||||
upsert_department,
|
||||
upsert_messages,
|
||||
upsert_phone_call,
|
||||
upsert_sms,
|
||||
upsert_staff,
|
||||
upsert_staff_from_account,
|
||||
upsert_voice_call,
|
||||
@@ -1219,3 +1222,183 @@ async def test_upsert_messages_with_whatsapp_channel():
|
||||
assert fetched.is_send is False
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PhoneCall (phone_call) upsert (M4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_upsert_phone_call_is_idempotent_and_updates_record_file():
|
||||
"""Same call_id twice with different record_file/version -> 1 row, refreshed.
|
||||
|
||||
The phone-call channel (``channel="phone_call"``) shares the
|
||||
:class:`PhoneCall` table with the voice-call channel; the per-row
|
||||
``channel`` value disambiguates them.
|
||||
"""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec1 = {
|
||||
"call_id": "pc_dup",
|
||||
"channel": "phone_call",
|
||||
"record_file": "https://example.com/call_v1.mp3",
|
||||
"call_duration": 10,
|
||||
"call_status": 2,
|
||||
"incoming_call": 0,
|
||||
"version": "v1",
|
||||
}
|
||||
rec2 = {
|
||||
"call_id": "pc_dup",
|
||||
"channel": "phone_call",
|
||||
"record_file": "https://example.com/call_v2.mp3",
|
||||
"call_duration": 12,
|
||||
"call_status": 0,
|
||||
"incoming_call": 1,
|
||||
"version": "v2",
|
||||
}
|
||||
|
||||
await upsert_phone_call(session, rec1)
|
||||
await session.commit()
|
||||
await upsert_phone_call(session, rec2)
|
||||
await session.commit()
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(PhoneCall)
|
||||
.where(PhoneCall.call_id == "pc_dup")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].call_id == "pc_dup"
|
||||
assert rows[0].channel == "phone_call"
|
||||
assert rows[0].record_file == "https://example.com/call_v2.mp3"
|
||||
assert rows[0].call_duration == 12
|
||||
assert rows[0].call_status == 0
|
||||
assert rows[0].incoming_call == 1
|
||||
assert rows[0].version == "v2"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_phone_call_null_record_file_and_coerces_ints():
|
||||
"""null record_file -> None; numeric-str call_duration/start_call_time/
|
||||
call_status/incoming_call/qc_status coerced to int; is_upload -> bool."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"call_id": "pc_null",
|
||||
"channel": "phone_call",
|
||||
"record_file": None,
|
||||
"call_duration": "151", # str -> int
|
||||
"start_call_time": "1700000000123", # numeric str -> int (ms epoch)
|
||||
"call_status": "2", # str -> int
|
||||
"incoming_call": "1", # str -> int (1=incoming/0=outgoing)
|
||||
"qc_status": "1", # str -> int
|
||||
"is_upload": "1", # str -> bool
|
||||
"download_count": 3,
|
||||
}
|
||||
await upsert_phone_call(session, rec)
|
||||
await session.commit()
|
||||
|
||||
fetched = (
|
||||
await session.execute(
|
||||
sa.select(PhoneCall).where(PhoneCall.call_id == "pc_null")
|
||||
)
|
||||
).scalar_one()
|
||||
assert fetched.record_file is None
|
||||
assert fetched.call_duration == 151
|
||||
assert fetched.start_call_time == 1700000000123
|
||||
assert fetched.call_status == 2
|
||||
assert fetched.incoming_call == 1
|
||||
assert fetched.qc_status == 1
|
||||
assert fetched.is_upload is True
|
||||
assert fetched.download_count == 3
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sms upsert (M4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_upsert_sms_is_idempotent_and_updates_content():
|
||||
"""Same sms_id twice with different content/version -> 1 row, refreshed."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec1 = {
|
||||
"sms_id": "sms_dup",
|
||||
"channel": "sms",
|
||||
"customer_phone": "8613800000001",
|
||||
"content": "first message",
|
||||
"content_preview": "first...",
|
||||
"direction": 0,
|
||||
"create_time": 1_700_000_000_000,
|
||||
"version": "v1",
|
||||
}
|
||||
rec2 = {
|
||||
"sms_id": "sms_dup",
|
||||
"channel": "sms",
|
||||
"customer_phone": "8613800000001",
|
||||
"content": "updated message",
|
||||
"content_preview": "updated...",
|
||||
"direction": 0,
|
||||
"create_time": 1_700_000_000_000,
|
||||
"version": "v2",
|
||||
}
|
||||
|
||||
await upsert_sms(session, rec1)
|
||||
await session.commit()
|
||||
await upsert_sms(session, rec2)
|
||||
await session.commit()
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Sms)
|
||||
.where(Sms.sms_id == "sms_dup")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].sms_id == "sms_dup"
|
||||
assert rows[0].channel == "sms"
|
||||
assert rows[0].content == "updated message"
|
||||
assert rows[0].content_preview == "updated..."
|
||||
assert rows[0].version == "v2"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_sms_coerces_direction_and_create_time():
|
||||
"""direction numeric str -> int; create_time numeric str -> int (ms epoch)."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"sms_id": "sms_coerce",
|
||||
"channel": "sms",
|
||||
"direction": "1", # str -> int (1=发/outgoing)
|
||||
"create_time": "1700000000123", # numeric str -> int (ms epoch)
|
||||
"content": "hi",
|
||||
}
|
||||
await upsert_sms(session, rec)
|
||||
await session.commit()
|
||||
|
||||
fetched = (
|
||||
await session.execute(
|
||||
sa.select(Sms).where(Sms.sms_id == "sms_coerce")
|
||||
)
|
||||
).scalar_one()
|
||||
assert fetched.direction == 1
|
||||
assert fetched.create_time == 1700000000123
|
||||
assert fetched.content == "hi"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
"""Tests for the phone-call + SMS scrapers (M4, ``app/scraper/call.py``).
|
||||
|
||||
Drives the real page-number pagination + normalization logic through
|
||||
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
|
||||
login flow is skipped) against a mocked transport.
|
||||
|
||||
Two endpoint suites:
|
||||
|
||||
* ``POST /pc/callLog/list`` (phone calls) — envelope
|
||||
``data.data`` / ``data.totalCount`` / ``data.pageCount``. Maps to
|
||||
:class:`PhoneCall` with ``channel="phone_call"``. ``incomingCall`` polarity
|
||||
is ``1=incoming / 0=outgoing`` and is stored DIRECTLY (no inversion — unlike
|
||||
the voice-call channel's ``isSend``).
|
||||
* ``POST /pc/customer/smsLogsList`` (SMS) — envelope ``data.data`` /
|
||||
``data.dataCount`` (note: ``dataCount``, NOT ``totalCount``), no
|
||||
``pageCount``. Maps to :class:`Sms` with ``channel="sms"`` (the model
|
||||
default).
|
||||
|
||||
Both fixtures ship multiple pages so the tests exercise pagination across the
|
||||
empty-page stop signal end-to-end, and assert the full field mapping incl.
|
||||
the two distinct datetime string formats (``YYYY-MM-DD HH:MM:SS`` for calls,
|
||||
``YYYY-MM-DD HH:MM`` for SMS) parsed to ms-epoch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json as _json
|
||||
|
||||
import httpx
|
||||
import respx
|
||||
|
||||
from app.scraper.call import scrape_phone_calls, scrape_sms
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from tests.cassettes.callLog_fixture import (
|
||||
CALL_RECORDS,
|
||||
TOTAL_COUNT,
|
||||
make_call_log_response,
|
||||
)
|
||||
from tests.cassettes.callLog_fixture import (
|
||||
DEFAULT_PAGE_SIZE as CALL_PAGE_SIZE,
|
||||
)
|
||||
from tests.cassettes.smsLogs_fixture import (
|
||||
DATA_COUNT,
|
||||
SMS_RECORDS,
|
||||
make_sms_logs_response,
|
||||
)
|
||||
from tests.cassettes.smsLogs_fixture import (
|
||||
DEFAULT_PAGE_SIZE as SMS_PAGE_SIZE,
|
||||
)
|
||||
|
||||
BASE = "https://crm2.yunkecn.com"
|
||||
CALL_LOG_URL = f"{BASE}/pc/callLog/list"
|
||||
SMS_LOG_URL = f"{BASE}/pc/customer/smsLogsList"
|
||||
|
||||
USERID = "a" * 32
|
||||
COMPANY = "8m0atd"
|
||||
|
||||
|
||||
def _primed_client() -> YunkeClient:
|
||||
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
||||
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
||||
|
||||
|
||||
def _ms_seconds(date_str: str) -> int:
|
||||
"""Expected ms-epoch for a ``YYYY-MM-DD HH:MM:SS`` string parsed as UTC.
|
||||
|
||||
Mirrors the callLog scraper's documented parsing rule so the assertion is
|
||||
an independent recomputation (not a self-reference).
|
||||
"""
|
||||
return int(
|
||||
dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
|
||||
.replace(tzinfo=dt.UTC)
|
||||
.timestamp()
|
||||
* 1000
|
||||
)
|
||||
|
||||
|
||||
def _ms_minutes(date_str: str) -> int:
|
||||
"""Expected ms-epoch for a ``YYYY-MM-DD HH:MM`` string parsed as UTC.
|
||||
|
||||
Mirrors the SMS scraper's documented parsing rule (seconds default to 00).
|
||||
"""
|
||||
return int(
|
||||
dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M")
|
||||
.replace(tzinfo=dt.UTC)
|
||||
.timestamp()
|
||||
* 1000
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Phone calls — POST /pc/callLog/list
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_phone_calls_yields_records_and_paginates():
|
||||
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
|
||||
page1 = make_call_log_response(
|
||||
CALL_RECORDS[:2], total_count=TOTAL_COUNT, page_num=1, page_size=CALL_PAGE_SIZE
|
||||
)
|
||||
page2 = make_call_log_response(
|
||||
CALL_RECORDS[2:], total_count=TOTAL_COUNT, page_num=2, page_size=CALL_PAGE_SIZE
|
||||
)
|
||||
page3 = make_call_log_response(
|
||||
[], total_count=TOTAL_COUNT, page_num=3, page_size=CALL_PAGE_SIZE
|
||||
)
|
||||
|
||||
route = respx.post(CALL_LOG_URL).mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=page1),
|
||||
httpx.Response(200, json=page2),
|
||||
httpx.Response(200, json=page3),
|
||||
]
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [
|
||||
r async for r in scrape_phone_calls(client, page_size=CALL_PAGE_SIZE)
|
||||
]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
|
||||
assert route.call_count == 3
|
||||
|
||||
# Exactly the 4 fixture records (not duplicated, not dropped).
|
||||
assert len(results) == 4
|
||||
assert {r["call_id"] for r in results} == {rec["id"] for rec in CALL_RECORDS}
|
||||
|
||||
# Field mapping for each record (compare against the source record).
|
||||
for result, record in zip(results, CALL_RECORDS, strict=True):
|
||||
# Identity + channel.
|
||||
assert result["call_id"] == record["id"]
|
||||
assert result["channel"] == "phone_call"
|
||||
|
||||
# Staff side.
|
||||
assert result["user_id"] == record["userId"]
|
||||
assert result["user_id_name"] == record["userIdName"]
|
||||
assert result["department_id"] == record["departmentId"]
|
||||
|
||||
# Call metadata.
|
||||
assert result["call_number"] == record["callNumber"]
|
||||
assert result["sim_phone"] == record["simPhone"]
|
||||
assert result["sim_number"] == record["simNumber"]
|
||||
assert result["imei"] == record["imei"]
|
||||
assert result["plan_customer_name"] == record["planCustomerName"]
|
||||
|
||||
# Recording URL (may be None).
|
||||
assert result["record_file"] == record["recordFile"]
|
||||
|
||||
# Status + duration + start time coercions.
|
||||
assert result["call_status"] == record["callStatus"]
|
||||
# callSeconds preferred over callDuration string.
|
||||
assert result["call_duration"] == record["callSeconds"]
|
||||
assert result["start_call_time"] == _ms_seconds(record["startCallTime"])
|
||||
|
||||
# incomingCall polarity: 1=incoming/0=outgoing — stored DIRECTLY.
|
||||
assert result["incoming_call"] == record["incomingCall"]
|
||||
|
||||
# is_upload (bool field) carried; download_count + qc_status carried.
|
||||
assert result["is_upload"] == record["isUpload"]
|
||||
assert result["download_count"] == record["downloadCount"]
|
||||
assert result["qc_status"] == record["qcStatus"]
|
||||
assert result["ai_asr"] == record["aiAsr"]
|
||||
|
||||
# Version carried verbatim; raw_json preserves the original record.
|
||||
assert result["version"] == record["version"]
|
||||
assert result["raw_json"] == record
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_phone_calls_null_record_file_yields_none():
|
||||
"""A record with ``recordFile=null`` -> ``record_file`` None."""
|
||||
rec = {
|
||||
"id": "pc_null",
|
||||
"startCallTime": "2026-01-15 10:00:00",
|
||||
"callSeconds": 0,
|
||||
"callStatus": 0,
|
||||
"incomingCall": 1, # incoming
|
||||
"userId": "staff_user_001",
|
||||
"recordFile": None,
|
||||
}
|
||||
page1 = make_call_log_response([rec], total_count=1, page_num=1, page_size=CALL_PAGE_SIZE)
|
||||
page2 = make_call_log_response([], total_count=1, page_num=2, page_size=CALL_PAGE_SIZE)
|
||||
respx.post(CALL_LOG_URL).mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=page1),
|
||||
httpx.Response(200, json=page2),
|
||||
]
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [
|
||||
r async for r in scrape_phone_calls(client, page_size=CALL_PAGE_SIZE)
|
||||
]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["record_file"] is None
|
||||
assert results[0]["call_status"] == 0
|
||||
# incoming (incomingCall=1) -> incoming_call=1 (no inversion).
|
||||
assert results[0]["incoming_call"] == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_phone_calls_sends_pagination_body():
|
||||
"""Outgoing request body carries pageNum/pageSize; advances per page."""
|
||||
page1 = make_call_log_response(
|
||||
CALL_RECORDS[:1], total_count=TOTAL_COUNT, page_num=1, page_size=1
|
||||
)
|
||||
page2 = make_call_log_response(
|
||||
[], total_count=TOTAL_COUNT, page_num=2, page_size=1
|
||||
)
|
||||
route = respx.post(CALL_LOG_URL).mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=page1),
|
||||
httpx.Response(200, json=page2),
|
||||
]
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [r async for r in scrape_phone_calls(client, page_size=1)]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
# One record on page 1, empty page 2 stops the loop.
|
||||
assert len(results) == 1
|
||||
assert route.call_count == 2
|
||||
|
||||
body1 = _json.loads(route.calls[0].request.content.decode())
|
||||
assert body1["pageNum"] == 1
|
||||
assert body1["pageSize"] == 1
|
||||
|
||||
body2 = _json.loads(route.calls[1].request.content.decode())
|
||||
assert body2["pageNum"] == 2
|
||||
assert body2["pageSize"] == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_phone_calls_stops_on_empty_first_page():
|
||||
"""If the very first page is empty, scraper stops without over-fetching."""
|
||||
empty = make_call_log_response([], total_count=0, page_num=1)
|
||||
route = respx.post(CALL_LOG_URL).mock(
|
||||
return_value=httpx.Response(200, json=empty)
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [r async for r in scrape_phone_calls(client)]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
assert results == []
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# SMS — POST /pc/customer/smsLogsList
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_sms_yields_records_and_paginates():
|
||||
"""Two pages (2 + 1 records) + a terminating empty page -> 3 records."""
|
||||
page1 = make_sms_logs_response(SMS_RECORDS[:2], data_count=DATA_COUNT)
|
||||
page2 = make_sms_logs_response(SMS_RECORDS[2:], data_count=DATA_COUNT)
|
||||
page3 = make_sms_logs_response([], data_count=DATA_COUNT)
|
||||
|
||||
route = respx.post(SMS_LOG_URL).mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=page1),
|
||||
httpx.Response(200, json=page2),
|
||||
httpx.Response(200, json=page3),
|
||||
]
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [r async for r in scrape_sms(client, page_size=SMS_PAGE_SIZE)]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
|
||||
assert route.call_count == 3
|
||||
|
||||
# Exactly the 3 fixture records.
|
||||
assert len(results) == 3
|
||||
assert {r["sms_id"] for r in results} == {rec["id"] for rec in SMS_RECORDS}
|
||||
|
||||
# Field mapping for each record (compare against the source record).
|
||||
for result, record in zip(results, SMS_RECORDS, strict=True):
|
||||
# Identity + channel (the model's default 'sms').
|
||||
assert result["sms_id"] == record["id"]
|
||||
assert result["channel"] == "sms"
|
||||
|
||||
# Phone + text.
|
||||
assert result["customer_phone"] == record["customerPhone"]
|
||||
assert result["content"] == record["sms"]
|
||||
# Empty preview string -> None (the scraper normalizes falsy -> None).
|
||||
expected_preview = record["smsSub"] or None
|
||||
assert result["content_preview"] == expected_preview
|
||||
|
||||
# Parties.
|
||||
assert result["sender"] == record["sender"]
|
||||
assert result["direction"] == record["direction"]
|
||||
assert result["user_id"] == record["userId"]
|
||||
assert result["sender_department"] == record["senderDepartment"]
|
||||
assert result["sender_department_id"] == record["senderDepartmentId"]
|
||||
assert result["receive_name"] == record["receiveName"]
|
||||
|
||||
# SIM + source.
|
||||
assert result["sim_phone"] == record["simPhone"]
|
||||
assert result["sim_number"] == record["simNumber"]
|
||||
assert result["source"] == record["source"]
|
||||
|
||||
# createTime "YYYY-MM-DD HH:MM" -> ms epoch (seconds default :00).
|
||||
assert result["create_time"] == _ms_minutes(record["createTime"])
|
||||
|
||||
# raw_json preserves the original record.
|
||||
assert result["raw_json"] == record
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_sms_empty_preview_yields_none_content_preview():
|
||||
"""An empty ``smsSub`` string -> ``content_preview`` None (full text kept)."""
|
||||
rec = {
|
||||
"id": "sms_empty_preview",
|
||||
"customerPhone": "8613800000099",
|
||||
"sms": "好的，收到。",
|
||||
"smsSub": "",
|
||||
"sender": "8613800000099",
|
||||
"direction": 0,
|
||||
"userId": "staff_user_001",
|
||||
"createTime": "2026-01-16 09:00",
|
||||
}
|
||||
page1 = make_sms_logs_response([rec], data_count=1)
|
||||
page2 = make_sms_logs_response([], data_count=1)
|
||||
respx.post(SMS_LOG_URL).mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=page1),
|
||||
httpx.Response(200, json=page2),
|
||||
]
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [r async for r in scrape_sms(client, page_size=SMS_PAGE_SIZE)]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["content"] == "好的，收到。"
|
||||
assert results[0]["content_preview"] is None
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_sms_sends_pagination_body():
|
||||
"""Outgoing request body carries pageNum/pageSize; advances per page."""
|
||||
page1 = make_sms_logs_response(SMS_RECORDS[:1], data_count=DATA_COUNT)
|
||||
page2 = make_sms_logs_response([], data_count=DATA_COUNT)
|
||||
route = respx.post(SMS_LOG_URL).mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=page1),
|
||||
httpx.Response(200, json=page2),
|
||||
]
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [r async for r in scrape_sms(client, page_size=1)]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
assert len(results) == 1
|
||||
assert route.call_count == 2
|
||||
|
||||
body1 = _json.loads(route.calls[0].request.content.decode())
|
||||
assert body1["pageNum"] == 1
|
||||
assert body1["pageSize"] == 1
|
||||
|
||||
body2 = _json.loads(route.calls[1].request.content.decode())
|
||||
assert body2["pageNum"] == 2
|
||||
assert body2["pageSize"] == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_scrape_sms_stops_on_empty_first_page():
|
||||
"""If the very first page is empty, scraper stops without over-fetching."""
|
||||
empty = make_sms_logs_response([], data_count=0)
|
||||
route = respx.post(SMS_LOG_URL).mock(
|
||||
return_value=httpx.Response(200, json=empty)
|
||||
)
|
||||
|
||||
client = _primed_client()
|
||||
try:
|
||||
results = [r async for r in scrape_sms(client)]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
assert results == []
|
||||
assert route.call_count == 1
|
||||
Reference in New Issue
Block a user