feat: phone-call + sms scrapers
M4 (/pc/ CRM module): mirror phone-call + SMS endpoints. Scraper (app/scraper/call.py): - scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call') Envelope data.data / data.totalCount / data.pageCount; page-number pagination. incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike voice-call isSend); callSeconds preferred over legacy callDuration string; startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch. - scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms') Envelope data.data / data.dataCount (NOT totalCount), no pageCount; relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch; empty smsSub normalized to None content_preview. Shared helper: extracted parse_datetime_to_ms(value, formats) to app/scraper/parsers.py and refactored _parse_voice_start_time to delegate (DRY across all three call/SMS datetime sources). Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id / sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion, matching start_call_time's existing treatment. Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert + SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types 'phone_call' and 'sms'. Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures exercising pagination + full field mapping), repo idempotency + coercion tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers. ruff clean. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -518,3 +518,164 @@ async def test_sync_all_wechat_runs_each_entity_then_messages(monkeypatch):
|
||||
assert all(r.status == "success" for r in results)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Phone calls + SMS — full re-scrape entities (M4)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
async def test_phone_calls_full_rescrape_upserts_all_records_sync_run_and_watermark(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Phone-call list has no `since` param: 'incremental' = re-scrape all and
|
||||
rely on upsert idempotency. The orchestrator upserts every record, records
|
||||
a success SyncRun (entity_type='phone_call'), and stamps the watermark's
|
||||
last_synced_at."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
call_recs = [
|
||||
{"call_id": "pc1", "channel": "phone_call", "version": "v1"},
|
||||
{"call_id": "pc2", "channel": "phone_call", "version": "v1"},
|
||||
]
|
||||
|
||||
async def fake_scrape_phone_calls(client):
|
||||
for r in call_recs:
|
||||
yield r
|
||||
|
||||
upserted: list[dict] = []
|
||||
|
||||
async def fake_upsert_phone_call(session, rec):
|
||||
upserted.append(rec)
|
||||
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "scrape_phone_calls", fake_scrape_phone_calls
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "upsert_phone_call", fake_upsert_phone_call
|
||||
)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
result = await orchestrator.sync_phone_calls(session, client=object())
|
||||
await session.commit()
|
||||
|
||||
assert result.status == "success"
|
||||
assert result.fetched == 2
|
||||
assert result.upserted == 2
|
||||
assert [r["call_id"] for r in upserted] == ["pc1", "pc2"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = (
|
||||
await session.execute(
|
||||
sa.select(SyncRun).where(SyncRun.entity_type == "phone_call")
|
||||
)
|
||||
).scalar_one()
|
||||
assert run.status == "success"
|
||||
assert run.records_fetched == 2
|
||||
assert run.records_upserted == 2
|
||||
assert run.error is None
|
||||
|
||||
state = (
|
||||
await session.execute(
|
||||
sa.select(SyncState).where(
|
||||
SyncState.entity_type == "phone_call"
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
assert state.last_synced_at is not None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_sms_full_rescrape_upserts_all_records_sync_run_and_watermark(
|
||||
monkeypatch,
|
||||
):
|
||||
"""SMS list has no `since` param: same full-re-scrape shape as phone calls.
|
||||
Records a success SyncRun (entity_type='sms') and stamps the watermark."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
sms_recs = [
|
||||
{"sms_id": "sms1", "channel": "sms", "content": "a"},
|
||||
{"sms_id": "sms2", "channel": "sms", "content": "b"},
|
||||
{"sms_id": "sms3", "channel": "sms", "content": "c"},
|
||||
]
|
||||
|
||||
async def fake_scrape_sms(client):
|
||||
for r in sms_recs:
|
||||
yield r
|
||||
|
||||
upserted: list[dict] = []
|
||||
|
||||
async def fake_upsert_sms(session, rec):
|
||||
upserted.append(rec)
|
||||
|
||||
monkeypatch.setattr(orchestrator, "scrape_sms", fake_scrape_sms)
|
||||
monkeypatch.setattr(orchestrator, "upsert_sms", fake_upsert_sms)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
result = await orchestrator.sync_sms(session, client=object())
|
||||
await session.commit()
|
||||
|
||||
assert result.status == "success"
|
||||
assert result.fetched == 3
|
||||
assert result.upserted == 3
|
||||
assert [r["sms_id"] for r in upserted] == ["sms1", "sms2", "sms3"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = (
|
||||
await session.execute(
|
||||
sa.select(SyncRun).where(SyncRun.entity_type == "sms")
|
||||
)
|
||||
).scalar_one()
|
||||
assert run.status == "success"
|
||||
assert run.records_fetched == 3
|
||||
assert run.records_upserted == 3
|
||||
assert run.error is None
|
||||
|
||||
state = (
|
||||
await session.execute(
|
||||
sa.select(SyncState).where(SyncState.entity_type == "sms")
|
||||
)
|
||||
).scalar_one()
|
||||
assert state.last_synced_at is not None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_failing_phone_call_scraper_records_failed_run(monkeypatch):
|
||||
"""Same failure policy as the wechat entities: a scraper that raises
|
||||
records a failed SyncRun (entity_type='phone_call') and returns a failed
|
||||
result without re-raising."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
async def fake_scrape_phone_calls(client):
|
||||
raise RuntimeError("callLog 500")
|
||||
yield {} # pragma: no cover - make this an async generator
|
||||
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "scrape_phone_calls", fake_scrape_phone_calls
|
||||
)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
result = await orchestrator.sync_phone_calls(session, client=object())
|
||||
await session.commit()
|
||||
|
||||
assert result.status == "failed"
|
||||
assert "callLog 500" in (result.error or "")
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = (
|
||||
await session.execute(
|
||||
sa.select(SyncRun).where(SyncRun.entity_type == "phone_call")
|
||||
)
|
||||
).scalar_one()
|
||||
assert run.status == "failed"
|
||||
assert "callLog 500" in (run.error or "")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
Reference in New Issue
Block a user