feat: phone-call + sms scrapers
M4 (/pc/ CRM module): mirror phone-call + SMS endpoints. Scraper (app/scraper/call.py): - scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call') Envelope data.data / data.totalCount / data.pageCount; page-number pagination. incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike voice-call isSend); callSeconds preferred over legacy callDuration string; startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch. - scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms') Envelope data.data / data.dataCount (NOT totalCount), no pageCount; relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch; empty smsSub normalized to None content_preview. Shared helper: extracted parse_datetime_to_ms(value, formats) to app/scraper/parsers.py and refactored _parse_voice_start_time to delegate (DRY across all three call/SMS datetime sources). Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id / sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion, matching start_call_time's existing treatment. Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert + SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types 'phone_call' and 'sms'. Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures exercising pagination + full field mapping), repo idempotency + coercion tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers. ruff clean. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -61,6 +61,7 @@ from app.db.models import (
|
||||
Media,
|
||||
Message,
|
||||
PhoneCall,
|
||||
Sms,
|
||||
Staff,
|
||||
WechatAccount,
|
||||
WhatsappAccount,
|
||||
@@ -73,6 +74,8 @@ __all__ = [
|
||||
"upsert_department",
|
||||
"upsert_media",
|
||||
"upsert_messages",
|
||||
"upsert_phone_call",
|
||||
"upsert_sms",
|
||||
"upsert_staff",
|
||||
"upsert_staff_from_account",
|
||||
"upsert_voice_call",
|
||||
@@ -104,6 +107,7 @@ _INT_FIELDS: frozenset[str] = frozenset(
|
||||
"call_duration",
|
||||
"start_call_time",
|
||||
"qc_status",
|
||||
"create_time",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -505,6 +509,42 @@ async def upsert_voice_call(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
await _upsert_one(session, PhoneCall, rec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PhoneCall (phone_call) + Sms upsert (M4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def upsert_phone_call(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
"""Idempotently upsert one :class:`PhoneCall` from a phone-call record (M4).
|
||||
|
||||
Same ``call_id`` twice -> one row, with non-PK fields refreshed from the
|
||||
later record. PK = ``call_id`` (the source ``id``). The phone-call channel
|
||||
(``channel="phone_call"``) shares the :class:`PhoneCall` table with the
|
||||
voice-call channel; the per-row ``channel`` value disambiguates them.
|
||||
|
||||
Applies the documented int coercions (``call_duration`` /
|
||||
``start_call_time`` / ``call_status`` / ``incoming_call`` / ``qc_status``
|
||||
-> int when they arrive as numeric strings; the phone-call scraper already
|
||||
emits ints, but a hand-built record may not). ``is_upload`` is a bool
|
||||
field and goes through :func:`_coerce_bool` like the other bool columns.
|
||||
"""
|
||||
await _upsert_one(session, PhoneCall, rec)
|
||||
|
||||
|
||||
async def upsert_sms(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
"""Idempotently upsert one :class:`Sms` (PK = ``sms_id``) (M4).
|
||||
|
||||
Same ``sms_id`` twice -> one row, with non-PK fields refreshed from the
|
||||
later record. Applies the documented int coercions (``direction`` /
|
||||
``create_time`` -> int when they arrive as numeric strings; the SMS
|
||||
scraper already emits ints, but a hand-built record may not). The
|
||||
``channel`` column defaults to ``"sms"`` at the model level; the scraper
|
||||
carries it explicitly so the upsert is independent of the column-default
|
||||
application order.
|
||||
"""
|
||||
await _upsert_one(session, Sms, rec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Media upsert (M2, Task 2.6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user