feat: phone-call + sms scrapers

M4 (/pc/ CRM module): mirror phone-call + SMS endpoints. Scraper (app/scraper/call.py): - scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call') Envelope data.data / data.totalCount / data.pageCount; page-number pagination. incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike voice-call isSend); callSeconds preferred over legacy callDuration string; startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch. - scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms') Envelope data.data / data.dataCount (NOT totalCount), no pageCount; relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch; empty smsSub normalized to None content_preview. Shared helper: extracted parse_datetime_to_ms(value, formats) to app/scraper/parsers.py and refactored _parse_voice_start_time to delegate (DRY across all three call/SMS datetime sources). Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id / sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion, matching start_call_time's existing treatment. Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert + SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types 'phone_call' and 'sms'. Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures exercising pagination + full field mapping), repo idempotency + coercion tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers. ruff clean. Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-01 15:59:25 +08:00
parent 8a22fd9ff8
commit 44d64dd237
10 changed files with 1647 additions and 42 deletions
@@ -0,0 +1,210 @@
+"""Fixture payloads for ``POST /pc/callLog/list`` (M4 phone-call scraper).
+
+These records mirror the real field names confirmed by Phase-0 + the data
+dictionary §4 (电话记录 phone_call): ``id, startCallTime(YYYY-MM-DD HH:MM:SS),
+callDuration("M'SS\"")/callSeconds(int秒), callStatus(0未接/2接通),
+incomingCall(0呼出/1呼入), userId/userIdName(员工), departmentId, callNumber,
+simPhone, simNumber, imei, recordFile(录音OSS URL, may be null), isUpload,
+downloadCount, aiAsr, qcStatus, planCustomerName, version``. Live ``totalCount``
+is 6047 across ``pageCount=303`` pages.
+
+The envelope differs from the wechat ``data.data`` / ``data.total`` shape: the
+row list sits under ``data.data`` (same) but the declared row count is
+``data.totalCount`` (NOT ``data.total``) and the page count is ``data.pageCount``::
+
+    {"message":"调用成功","code":"10000","success":true,"traceId":"...",
+     "data":{"data":[...], "page":.., "pageCount":.., "totalCount":..,
+             "phoneHidden":false, "hideRecordPlay":false}}
+
+The cassette ships four records across two pages so a test can drive the
+page-number paginator across the empty-page stop signal:
+
+* page 1 -> 2 of the 4 fixture records, ``totalCount=4``
+* page 2 -> the other 2 records, ``totalCount=4``
+* (page 3 -> empty ``data`` array, served inline by the test)
+
+Records deliberately exercise the documented mapping edges:
+
+* ``pc_2001`` — outgoing connected call (``incomingCall=0``; ``callStatus=2``;
+  ``callSeconds=151``; non-null ``recordFile``; full SIM/IMEI metadata).
+* ``pc_2002`` — incoming no-answer call with NO recording (``incomingCall=1``;
+  ``callStatus=0``; ``recordFile=null``).
+* ``pc_2003`` — incoming connected call with a recording + AI ASR + QC data.
+* ``pc_2004`` — outgoing connected call (exercises the other polarity pair).
+
+NOTE on ``incomingCall`` polarity: this API's ``incomingCall`` is
+``1=incoming / 0=outgoing`` — the SAME polarity as ``PhoneCall.incoming_call``
+(no inversion needed, unlike the voice-call channel's ``isSend``).
+"""
+
+# The module name mirrors the upstream endpoint name (``callLog``); the M4 spec
+# uses this naming family.
+# ruff: noqa: N999
+
+from __future__ import annotations
+
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Capture-shaped phone-call records (4 total across 2 pages).
+# ---------------------------------------------------------------------------
+
+CALL_RECORDS: list[dict[str, Any]] = [
+    {
+        # Outgoing connected call with a recording.
+        "id": "pc_2001",
+        "startCallTime": "2026-01-15 10:30:00",  # -> ms epoch UTC
+        "callSeconds": 151,  # preferred over callDuration string
+        "callDuration": "2'31\"",  # legacy "M'SS\"" string; scraper prefers callSeconds
+        "callStatus": 2,  # 2=接通 (connected)
+        "incomingCall": 0,  # 0=呼出 (outgoing) -> incoming_call=0
+        "userId": "staff_user_001",
+        "userIdName": "Alice Wang",
+        "departmentId": "dept_sales_east",
+        "callNumber": "8613800000001",
+        "simPhone": "8613800000111",
+        "simNumber": "8986000112345678901",
+        "imei": "356938035643801",
+        "recordFile": (
+            "https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/phone-call/"
+            "8m0atd/imei001/1700000000000_1700000151000.mp3"
+        ),
+        "isUpload": 1,
+        "downloadCount": 3,
+        "aiAsr": "您好，请问您方便了解一下我们的产品吗？",
+        "qcStatus": 1,
+        "planCustomerName": "VIP Customer A",
+        "version": "v1.20260731.pc2001",
+    },
+    {
+        # Incoming no-answer call, no recording.
+        "id": "pc_2002",
+        "startCallTime": "2026-01-15 11:00:00",
+        "callSeconds": 0,
+        "callDuration": "0'0\"",
+        "callStatus": 0,  # 0=未接 (no answer)
+        "incomingCall": 1,  # 1=呼入 (incoming) -> incoming_call=1
+        "userId": "staff_user_001",
+        "userIdName": "Alice Wang",
+        "departmentId": "dept_sales_east",
+        "callNumber": "8613800000002",
+        "simPhone": "8613800000111",
+        "simNumber": "8986000112345678901",
+        "imei": "356938035643801",
+        "recordFile": None,  # no recording -> record_file None
+        "isUpload": 0,
+        "downloadCount": 0,
+        "aiAsr": None,
+        "qcStatus": 0,
+        "planCustomerName": "Lead B",
+        "version": "v1.20260731.pc2002",
+    },
+    {
+        # Incoming connected call with recording + AI ASR text.
+        "id": "pc_2003",
+        "startCallTime": "2026-01-15 12:00:00",
+        "callSeconds": 45,
+        "callDuration": "0'45\"",
+        "callStatus": 2,  # connected
+        "incomingCall": 1,  # incoming -> incoming_call=1
+        "userId": "staff_user_002",
+        "userIdName": "Bob Chen",
+        "departmentId": "dept_sales_west",
+        "callNumber": "8613800000003",
+        "simPhone": "8613800000222",
+        "simNumber": "8986000223456789012",
+        "imei": "356938035643802",
+        "recordFile": (
+            "https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/phone-call/"
+            "8m0atd/imei002/1700000003000_1700000045000.mp3"
+        ),
+        "isUpload": 1,
+        "downloadCount": 1,
+        "aiAsr": "好的，我考虑一下。",
+        "qcStatus": 2,
+        "planCustomerName": "Customer C",
+        "version": "v1.20260731.pc2003",
+    },
+    {
+        # Outgoing connected call (the other polarity pair).
+        "id": "pc_2004",
+        "startCallTime": "2026-01-16 09:00:00",
+        "callSeconds": 207,
+        "callDuration": "3'27\"",
+        "callStatus": 2,  # connected
+        "incomingCall": 0,  # outgoing -> incoming_call=0
+        "userId": "staff_user_001",
+        "userIdName": "Alice Wang",
+        "departmentId": "dept_sales_east",
+        "callNumber": "8613800000004",
+        "simPhone": "8613800000111",
+        "simNumber": "8986000112345678901",
+        "imei": "356938035643801",
+        "recordFile": (
+            "https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/phone-call/"
+            "8m0atd/imei001/1700000005000_1700000212000.mp3"
+        ),
+        "isUpload": 1,
+        "downloadCount": 2,
+        "aiAsr": None,
+        "qcStatus": 0,
+        "planCustomerName": "Customer D",
+        "version": "v1.20260731.pc2004",
+    },
+]
+
+# Default page size for the fixture (and the scraper). totalCount=4 means the
+# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
+TOTAL_COUNT = len(CALL_RECORDS)
+DEFAULT_PAGE_SIZE = 2
+# Mirrors the live capture's declared page count (purely cosmetic for the
+# fixture; the paginator relies on the empty-page stop signal, not pageCount).
+PAGE_COUNT = 2
+
+
+def make_call_log_response(
+    records: list[dict[str, Any]],
+    *,
+    total_count: int,
+    page_num: int = 1,
+    page_size: int = DEFAULT_PAGE_SIZE,
+    page_count: int | None = None,
+    trace_id: str | None = None,
+) -> dict[str, Any]:
+    """Build a capture-shaped ``callLog/list`` response envelope.
+
+    Shape (row list under ``data.data``; row count on ``data.totalCount``;
+    page count on ``data.pageCount``)::
+
+        {"message":"调用成功","code":"10000","success":true,"traceId":"...",
+         "data":{"data":[...], "page":.., "pageCount":.., "totalCount":..,
+                 "phoneHidden":false, "hideRecordPlay":false}}
+    """
+    return {
+        "message": "调用成功",
+        "code": "10000",
+        "success": True,
+        "traceId": trace_id or f"trace-{page_num}",
+        "data": {
+            "data": list(records),
+            "page": page_num,
+            "pageCount": page_count if page_count is not None else page_count_for(page_size, total_count),
+            "totalCount": total_count,
+            "phoneHidden": False,
+            "hideRecordPlay": False,
+        },
+    }
+
+
+def page_count_for(page_size: int, total_count: int) -> int:
+    """Compute the declared page count for a fixture (ceil(total/size)).
+
+    Helper so the envelope's ``pageCount`` is internally consistent with the
+    declared ``totalCount`` + page size without forcing each test to recompute
+    it. The paginator does NOT consume ``pageCount`` (it relies on the
+    empty-page stop signal), so this is cosmetic — but it keeps the fixture
+    capture-shaped.
+    """
+    if page_size <= 0:
+        return 0
+    return (total_count + page_size - 1) // page_size