feat: phone-call + sms scrapers
M4 (/pc/ CRM module): mirror phone-call + SMS endpoints. Scraper (app/scraper/call.py): - scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call') Envelope data.data / data.totalCount / data.pageCount; page-number pagination. incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike voice-call isSend); callSeconds preferred over legacy callDuration string; startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch. - scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms') Envelope data.data / data.dataCount (NOT totalCount), no pageCount; relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch; empty smsSub normalized to None content_preview. Shared helper: extracted parse_datetime_to_ms(value, formats) to app/scraper/parsers.py and refactored _parse_voice_start_time to delegate (DRY across all three call/SMS datetime sources). Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id / sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion, matching start_call_time's existing treatment. Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert + SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types 'phone_call' and 'sms'. Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures exercising pagination + full field mapping), repo idempotency + coercion tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers. ruff clean. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -46,6 +46,7 @@ from typing import Any
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import SyncRun
|
||||
from app.scraper.call import scrape_phone_calls, scrape_sms
|
||||
from app.scraper.media import download_message_media
|
||||
from app.scraper.wechat import (
|
||||
scrape_departments,
|
||||
@@ -62,6 +63,8 @@ from app.storage.repos import (
|
||||
upsert_customer,
|
||||
upsert_department,
|
||||
upsert_messages,
|
||||
upsert_phone_call,
|
||||
upsert_sms,
|
||||
upsert_voice_call,
|
||||
upsert_wechat_account,
|
||||
)
|
||||
@@ -73,6 +76,8 @@ __all__ = [
|
||||
"SyncResult",
|
||||
"sync_all_wechat",
|
||||
"sync_departments",
|
||||
"sync_phone_calls",
|
||||
"sync_sms",
|
||||
"sync_wechat_accounts",
|
||||
"sync_wechat_friends",
|
||||
"sync_wechat_groups",
|
||||
@@ -93,6 +98,10 @@ ENTITY_WECHAT_FRIEND = "wechat_friend"
|
||||
ENTITY_WECHAT_GROUP = "wechat_group"
|
||||
ENTITY_WECHAT_VOICE_CALL = "wechat_voice_call"
|
||||
ENTITY_MESSAGE = "message"
|
||||
# M4 (/pc/ CRM module) — phone calls + SMS share no channel column with the
|
||||
# wechat-family entities; they track their own SyncRun / SyncState rows.
|
||||
ENTITY_PHONE_CALL = "phone_call"
|
||||
ENTITY_SMS = "sms"
|
||||
|
||||
CHANNEL_WECHAT = "wechat"
|
||||
|
||||
@@ -432,6 +441,54 @@ async def sync_wechat_voice_calls(
|
||||
)
|
||||
|
||||
|
||||
async def sync_phone_calls(
|
||||
session: AsyncSession, client: YunkeClient
|
||||
) -> SyncResult:
|
||||
"""Full re-scrape of all phone-call records (``/pc/callLog/list``) and
|
||||
idempotent upsert into :class:`PhoneCall` (``channel="phone_call"``).
|
||||
|
||||
Mirrors :func:`sync_wechat_voice_calls`: the callLog endpoint has no
|
||||
``since`` param, so "incremental" = re-scrape everything and rely on the
|
||||
repo's idempotent Core upsert (same ``call_id`` twice -> one refreshed
|
||||
row). Records a SyncRun row (``entity_type='phone_call'``) and stamps the
|
||||
watermark's ``last_synced_at`` (no cursor advance — there is no upstream
|
||||
window to close). A scraper failure is recorded as a failed SyncRun and a
|
||||
failed SyncResult is returned (no re-raise), so a phone-call scrape that
|
||||
500s does NOT abort a larger batch.
|
||||
"""
|
||||
return await _sync_full_rescrape(
|
||||
session,
|
||||
entity_type=ENTITY_PHONE_CALL,
|
||||
channel=None,
|
||||
scope_id=None,
|
||||
scrape=lambda: scrape_phone_calls(client),
|
||||
upsert=upsert_phone_call,
|
||||
)
|
||||
|
||||
|
||||
async def sync_sms(
|
||||
session: AsyncSession, client: YunkeClient
|
||||
) -> SyncResult:
|
||||
"""Full re-scrape of all SMS records (``/pc/customer/smsLogsList``) and
|
||||
idempotent upsert into :class:`Sms`.
|
||||
|
||||
Mirrors :func:`sync_phone_calls`: the smsLogsList endpoint has no
|
||||
``since`` param, so "incremental" = re-scrape everything and rely on the
|
||||
repo's idempotent Core upsert (same ``sms_id`` twice -> one refreshed
|
||||
row). Records a SyncRun row (``entity_type='sms'``) and stamps the
|
||||
watermark's ``last_synced_at``. A scraper failure is recorded as a failed
|
||||
SyncRun + SyncResult without re-raising.
|
||||
"""
|
||||
return await _sync_full_rescrape(
|
||||
session,
|
||||
entity_type=ENTITY_SMS,
|
||||
channel=None,
|
||||
scope_id=None,
|
||||
scrape=lambda: scrape_sms(client),
|
||||
upsert=upsert_sms,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coordinator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user