feat: whatsapp account/friend/message scrapers
Mirrors the WeChat scraper patterns (M3) using the shared paginator +
parser + repo building blocks. Three new async generators yield
snake_case dicts + verbatim raw_json, ready for the channel-generic
upsert_customer / upsert_messages repos.
- app/scraper/whatsapp.py (new): scrape_whatsapp_accounts /
scrape_whatsapp_friends (page-number pagination over the
data.overviews / data.totalCount envelope) and scrape_whatsapp_messages
(backward-cursor pagination, JSON body {uid,friendUid,start,end} —
NOT form-encoded like WeChat msg/chats). Drops type==-2 dividers;
maps mediaName -> file_url as a deferred-download key.
- app/scraper/parsers.py: add normalize_whatsapp_type (0->text,
1->image, -2->divider, else unknown; never raises) and a shared
coerce_int_or_none scraper helper (mirrors repos._coerce_int so the
scraper emits type_raw ints without a DB dependency).
- app/storage/repos.py: add upsert_whatsapp_account (PK uid) which
also upserts the embedded Staff(user_id, name) via the existing
upsert_staff_from_account helper; WhatsappAccount imported.
Tests (respx mocks, no network): 3 cassette fixtures + 3 scraper test
modules (accounts/friends/messages), parsers + repos extended with WA
cases. Divider drop, backward-cursor advance (end == E1-1), JSON body
asserts, embedded-staff upsert, and channel-generic Customer/Message
upserts all covered.
Suite: 182 -> 211 passing. ruff clean.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -12,7 +12,11 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.scraper.parsers import normalize_wechat_type
|
||||
from app.scraper.parsers import (
|
||||
coerce_int_or_none,
|
||||
normalize_wechat_type,
|
||||
normalize_whatsapp_type,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -52,3 +56,62 @@ def test_normalize_wechat_type_never_raises():
|
||||
out = normalize_wechat_type(raw)
|
||||
assert isinstance(out, str)
|
||||
assert out == "unknown"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# WhatsApp message-type normalization (M3)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw,expected",
|
||||
[
|
||||
# Phase-0 confirmed WhatsApp codes (see data dictionary §3):
|
||||
(0, "text"),
|
||||
(1, "image"), # image / media
|
||||
(-2, "divider"), # date separator — NOT a real message (scraper drops it)
|
||||
# Unconfirmed / other media codes — pass through as "unknown", never dropped.
|
||||
(2, "unknown"),
|
||||
(3, "unknown"),
|
||||
(13, "unknown"),
|
||||
(100, "unknown"),
|
||||
# Stringified ints (the upstream API may send either form).
|
||||
("0", "text"),
|
||||
("1", "image"),
|
||||
("-2", "divider"),
|
||||
("2", "unknown"),
|
||||
],
|
||||
)
|
||||
def test_normalize_whatsapp_type_maps_known_and_unknown(raw, expected):
|
||||
assert normalize_whatsapp_type(raw) == expected
|
||||
|
||||
|
||||
def test_normalize_whatsapp_type_never_raises():
|
||||
"""Whatever we throw at it, it returns a string — never raises."""
|
||||
for raw in (None, "", "junk", "1.5", [], {}, object(), float("nan")):
|
||||
out = normalize_whatsapp_type(raw)
|
||||
assert isinstance(out, str)
|
||||
assert out == "unknown"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# coerce_int_or_none — shared scraper helper (M3)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def test_coerce_int_or_none_handles_documented_cases():
|
||||
"""``type_raw``-style fields: str -> int; None preserved; 0 preserved."""
|
||||
assert coerce_int_or_none(None) is None
|
||||
assert coerce_int_or_none(0) == 0
|
||||
assert coerce_int_or_none(2) == 2
|
||||
assert coerce_int_or_none(-2) == -2
|
||||
assert coerce_int_or_none("0") == 0
|
||||
assert coerce_int_or_none("2") == 2
|
||||
assert coerce_int_or_none("-2") == -2
|
||||
|
||||
|
||||
def test_coerce_int_or_none_returns_none_for_non_numeric_string():
|
||||
"""A non-numeric value (e.g. a label) -> None."""
|
||||
assert coerce_int_or_none("text") is None
|
||||
assert coerce_int_or_none("") is None
|
||||
assert coerce_int_or_none("unknown") is None
|
||||
|
||||
Reference in New Issue
Block a user