feat: phone-call + sms scrapers

M4 (/pc/ CRM module): mirror phone-call + SMS endpoints.

Scraper (app/scraper/call.py):
- scrape_phone_calls: POST /pc/callLog/list -> PhoneCall (channel='phone_call')
  Envelope data.data / data.totalCount / data.pageCount; page-number pagination.
  incomingCall (1=incoming/0=outgoing) stored DIRECTLY (no inversion, unlike
  voice-call isSend); callSeconds preferred over legacy callDuration string;
  startCallTime (YYYY-MM-DD HH:MM:SS) -> ms epoch.
- scrape_sms: POST /pc/customer/smsLogsList -> Sms (channel='sms')
  Envelope data.data / data.dataCount (NOT totalCount), no pageCount;
  relies on empty-page stop signal. createTime (YYYY-MM-DD HH:MM) -> ms epoch;
  empty smsSub normalized to None content_preview.

Shared helper: extracted parse_datetime_to_ms(value, formats) to
app/scraper/parsers.py and refactored _parse_voice_start_time to delegate
(DRY across all three call/SMS datetime sources).

Repos: upsert_phone_call + upsert_sms (idempotent Core upsert by call_id /
sms_id). Added create_time to _INT_FIELDS for defensive numeric-str coercion,
matching start_call_time's existing treatment.

Orchestrator: sync_phone_calls + sync_sms (full re-scrape + idempotent upsert
+ SyncRun log + watermark), mirroring sync_wechat_voice_calls. Entity types
'phone_call' and 'sms'.

Tests (+15, suite 211 -> 226): respx-mocked scraper tests (2-page fixtures
exercising pagination + full field mapping), repo idempotency + coercion
tests, orchestrator SyncRun/watermark tests with monkeypatched scrapers.
ruff clean.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 15:59:25 +08:00
parent 8a22fd9ff8
commit 44d64dd237
10 changed files with 1647 additions and 42 deletions
+161
View File
@@ -518,3 +518,164 @@ async def test_sync_all_wechat_runs_each_entity_then_messages(monkeypatch):
assert all(r.status == "success" for r in results)
finally:
await engine.dispose()
# ===========================================================================
# Phone calls + SMS — full re-scrape entities (M4)
# ===========================================================================
async def test_phone_calls_full_rescrape_upserts_all_records_sync_run_and_watermark(
monkeypatch,
):
"""Phone-call list has no `since` param: 'incremental' = re-scrape all and
rely on upsert idempotency. The orchestrator upserts every record, records
a success SyncRun (entity_type='phone_call'), and stamps the watermark's
last_synced_at."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
call_recs = [
{"call_id": "pc1", "channel": "phone_call", "version": "v1"},
{"call_id": "pc2", "channel": "phone_call", "version": "v1"},
]
async def fake_scrape_phone_calls(client):
for r in call_recs:
yield r
upserted: list[dict] = []
async def fake_upsert_phone_call(session, rec):
upserted.append(rec)
monkeypatch.setattr(
orchestrator, "scrape_phone_calls", fake_scrape_phone_calls
)
monkeypatch.setattr(
orchestrator, "upsert_phone_call", fake_upsert_phone_call
)
async with SessionLocal() as session:
result = await orchestrator.sync_phone_calls(session, client=object())
await session.commit()
assert result.status == "success"
assert result.fetched == 2
assert result.upserted == 2
assert [r["call_id"] for r in upserted] == ["pc1", "pc2"]
async with SessionLocal() as session:
run = (
await session.execute(
sa.select(SyncRun).where(SyncRun.entity_type == "phone_call")
)
).scalar_one()
assert run.status == "success"
assert run.records_fetched == 2
assert run.records_upserted == 2
assert run.error is None
state = (
await session.execute(
sa.select(SyncState).where(
SyncState.entity_type == "phone_call"
)
)
).scalar_one()
assert state.last_synced_at is not None
finally:
await engine.dispose()
async def test_sms_full_rescrape_upserts_all_records_sync_run_and_watermark(
monkeypatch,
):
"""SMS list has no `since` param: same full-re-scrape shape as phone calls.
Records a success SyncRun (entity_type='sms') and stamps the watermark."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
sms_recs = [
{"sms_id": "sms1", "channel": "sms", "content": "a"},
{"sms_id": "sms2", "channel": "sms", "content": "b"},
{"sms_id": "sms3", "channel": "sms", "content": "c"},
]
async def fake_scrape_sms(client):
for r in sms_recs:
yield r
upserted: list[dict] = []
async def fake_upsert_sms(session, rec):
upserted.append(rec)
monkeypatch.setattr(orchestrator, "scrape_sms", fake_scrape_sms)
monkeypatch.setattr(orchestrator, "upsert_sms", fake_upsert_sms)
async with SessionLocal() as session:
result = await orchestrator.sync_sms(session, client=object())
await session.commit()
assert result.status == "success"
assert result.fetched == 3
assert result.upserted == 3
assert [r["sms_id"] for r in upserted] == ["sms1", "sms2", "sms3"]
async with SessionLocal() as session:
run = (
await session.execute(
sa.select(SyncRun).where(SyncRun.entity_type == "sms")
)
).scalar_one()
assert run.status == "success"
assert run.records_fetched == 3
assert run.records_upserted == 3
assert run.error is None
state = (
await session.execute(
sa.select(SyncState).where(SyncState.entity_type == "sms")
)
).scalar_one()
assert state.last_synced_at is not None
finally:
await engine.dispose()
async def test_failing_phone_call_scraper_records_failed_run(monkeypatch):
"""Same failure policy as the wechat entities: a scraper that raises
records a failed SyncRun (entity_type='phone_call') and returns a failed
result without re-raising."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async def fake_scrape_phone_calls(client):
raise RuntimeError("callLog 500")
yield {} # pragma: no cover - make this an async generator
monkeypatch.setattr(
orchestrator, "scrape_phone_calls", fake_scrape_phone_calls
)
async with SessionLocal() as session:
result = await orchestrator.sync_phone_calls(session, client=object())
await session.commit()
assert result.status == "failed"
assert "callLog 500" in (result.error or "")
async with SessionLocal() as session:
run = (
await session.execute(
sa.select(SyncRun).where(SyncRun.entity_type == "phone_call")
)
).scalar_one()
assert run.status == "failed"
assert "callLog 500" in (run.error or "")
finally:
await engine.dispose()