feat: phone-call + sms scrapers

M4 (/pc/ CRM module): mirror phone-call + SMS endpoints. Scraper (app/scraper/call.py): - scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call') Envelope data.data / data.totalCount / data.pageCount; page-number pagination. incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike voice-call isSend); callSeconds preferred over legacy callDuration string; startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch. - scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms') Envelope data.data / data.dataCount (NOT totalCount), no pageCount; relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch; empty smsSub normalized to None content_preview. Shared helper: extracted parse_datetime_to_ms(value, formats) to app/scraper/parsers.py and refactored _parse_voice_start_time to delegate (DRY across all three call/SMS datetime sources). Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id / sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion, matching start_call_time's existing treatment. Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert + SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types 'phone_call' and 'sms'. Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures exercising pagination + full field mapping), repo idempotency + coercion tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers. ruff clean. Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-01 15:59:25 +08:00
parent 8a22fd9ff8
commit 44d64dd237
10 changed files with 1647 additions and 42 deletions
@@ -29,6 +29,7 @@ from app.db.models import (
    GroupMember,
    Message,
    PhoneCall,
+    Sms,
    Staff,
    WechatAccount,
    WhatsappAccount,
@@ -41,6 +42,8 @@ from app.storage.repos import (
    upsert_customers,
    upsert_department,
    upsert_messages,
+    upsert_phone_call,
+    upsert_sms,
    upsert_staff,
    upsert_staff_from_account,
    upsert_voice_call,
@@ -1219,3 +1222,183 @@ async def test_upsert_messages_with_whatsapp_channel():
            assert fetched.is_send is False
    finally:
        await engine.dispose()
+
+
+# ---------------------------------------------------------------------------
+# PhoneCall (phone_call) upsert (M4)
+# ---------------------------------------------------------------------------
+
+
+async def test_upsert_phone_call_is_idempotent_and_updates_record_file():
+    """Same call_id twice with different record_file/version -> 1 row, refreshed.
+
+    The phone-call channel (``channel="phone_call"``) shares the
+    :class:`PhoneCall` table with the voice-call channel; the per-row
+    ``channel`` value disambiguates them.
+    """
+    engine = await _make_engine()
+    try:
+        SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+        async with SessionLocal() as session:
+            rec1 = {
+                "call_id": "pc_dup",
+                "channel": "phone_call",
+                "record_file": "https://example.com/call_v1.mp3",
+                "call_duration": 10,
+                "call_status": 2,
+                "incoming_call": 0,
+                "version": "v1",
+            }
+            rec2 = {
+                "call_id": "pc_dup",
+                "channel": "phone_call",
+                "record_file": "https://example.com/call_v2.mp3",
+                "call_duration": 12,
+                "call_status": 0,
+                "incoming_call": 1,
+                "version": "v2",
+            }
+
+            await upsert_phone_call(session, rec1)
+            await session.commit()
+            await upsert_phone_call(session, rec2)
+            await session.commit()
+
+            rows = (
+                await session.execute(
+                    sa.select(PhoneCall)
+                    .where(PhoneCall.call_id == "pc_dup")
+                    .execution_options(populate_existing=True)
+                )
+            ).scalars().all()
+            assert len(rows) == 1
+            assert rows[0].call_id == "pc_dup"
+            assert rows[0].channel == "phone_call"
+            assert rows[0].record_file == "https://example.com/call_v2.mp3"
+            assert rows[0].call_duration == 12
+            assert rows[0].call_status == 0
+            assert rows[0].incoming_call == 1
+            assert rows[0].version == "v2"
+    finally:
+        await engine.dispose()
+
+
+async def test_upsert_phone_call_null_record_file_and_coerces_ints():
+    """null record_file -> None; numeric-str call_duration/start_call_time/
+    call_status/incoming_call/qc_status coerced to int; is_upload -> bool."""
+    engine = await _make_engine()
+    try:
+        SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+        async with SessionLocal() as session:
+            rec = {
+                "call_id": "pc_null",
+                "channel": "phone_call",
+                "record_file": None,
+                "call_duration": "151",  # str -> int
+                "start_call_time": "1700000000123",  # numeric str -> int (ms epoch)
+                "call_status": "2",  # str -> int
+                "incoming_call": "1",  # str -> int (1=incoming/0=outgoing)
+                "qc_status": "1",  # str -> int
+                "is_upload": "1",  # str -> bool
+                "download_count": 3,
+            }
+            await upsert_phone_call(session, rec)
+            await session.commit()
+
+            fetched = (
+                await session.execute(
+                    sa.select(PhoneCall).where(PhoneCall.call_id == "pc_null")
+                )
+            ).scalar_one()
+            assert fetched.record_file is None
+            assert fetched.call_duration == 151
+            assert fetched.start_call_time == 1700000000123
+            assert fetched.call_status == 2
+            assert fetched.incoming_call == 1
+            assert fetched.qc_status == 1
+            assert fetched.is_upload is True
+            assert fetched.download_count == 3
+    finally:
+        await engine.dispose()
+
+
+# ---------------------------------------------------------------------------
+# Sms upsert (M4)
+# ---------------------------------------------------------------------------
+
+
+async def test_upsert_sms_is_idempotent_and_updates_content():
+    """Same sms_id twice with different content/version -> 1 row, refreshed."""
+    engine = await _make_engine()
+    try:
+        SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+        async with SessionLocal() as session:
+            rec1 = {
+                "sms_id": "sms_dup",
+                "channel": "sms",
+                "customer_phone": "8613800000001",
+                "content": "first message",
+                "content_preview": "first...",
+                "direction": 0,
+                "create_time": 1_700_000_000_000,
+                "version": "v1",
+            }
+            rec2 = {
+                "sms_id": "sms_dup",
+                "channel": "sms",
+                "customer_phone": "8613800000001",
+                "content": "updated message",
+                "content_preview": "updated...",
+                "direction": 0,
+                "create_time": 1_700_000_000_000,
+                "version": "v2",
+            }
+
+            await upsert_sms(session, rec1)
+            await session.commit()
+            await upsert_sms(session, rec2)
+            await session.commit()
+
+            rows = (
+                await session.execute(
+                    sa.select(Sms)
+                    .where(Sms.sms_id == "sms_dup")
+                    .execution_options(populate_existing=True)
+                )
+            ).scalars().all()
+            assert len(rows) == 1
+            assert rows[0].sms_id == "sms_dup"
+            assert rows[0].channel == "sms"
+            assert rows[0].content == "updated message"
+            assert rows[0].content_preview == "updated..."
+            assert rows[0].version == "v2"
+    finally:
+        await engine.dispose()
+
+
+async def test_upsert_sms_coerces_direction_and_create_time():
+    """direction numeric str -> int; create_time numeric str -> int (ms epoch)."""
+    engine = await _make_engine()
+    try:
+        SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+        async with SessionLocal() as session:
+            rec = {
+                "sms_id": "sms_coerce",
+                "channel": "sms",
+                "direction": "1",  # str -> int (1=发/outgoing)
+                "create_time": "1700000000123",  # numeric str -> int (ms epoch)
+                "content": "hi",
+            }
+            await upsert_sms(session, rec)
+            await session.commit()
+
+            fetched = (
+                await session.execute(
+                    sa.select(Sms).where(Sms.sms_id == "sms_coerce")
+                )
+            ).scalar_one()
+            assert fetched.direction == 1
+            assert fetched.create_time == 1700000000123
+            assert fetched.content == "hi"
+    finally:
+        await engine.dispose()