feat(scraper): WA 群 scraper(businessFriendList+type=1,待风控重置后实测)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-11 16:04:03 +08:00
parent 17ee5a1a6c
commit 0e1cb87156
4 changed files with 509 additions and 2 deletions
+66 -1
View File
@@ -69,6 +69,7 @@ from app.scraper.whatsapp import (
scrape_wa_voice_calls,
scrape_whatsapp_accounts,
scrape_whatsapp_friends,
scrape_whatsapp_groups,
scrape_whatsapp_messages,
)
from app.session.yunke_client import YunkeClient
@@ -100,14 +101,15 @@ __all__ = [
"sync_sms",
"sync_voice_clip_audio",
"sync_voice_clips",
"sync_wa_voice_calls",
"sync_wechat_accounts",
"sync_wechat_friends",
"sync_wechat_groups",
"sync_wechat_messages",
"sync_wechat_voice_calls",
"sync_wa_voice_calls",
"sync_whatsapp_accounts",
"sync_whatsapp_friends",
"sync_whatsapp_groups",
"sync_whatsapp_messages",
]
@@ -130,6 +132,12 @@ ENTITY_WA_VOICE_CALL = "wa_voice_call"
# org's department tree with WeChat (sync_departments seeds it).
ENTITY_WHATSAPP_ACCOUNT = "whatsapp_account"
ENTITY_WHATSAPP_FRIEND = "whatsapp_friend"
# WhatsApp chat-groups share the ``ChatGroup`` table with wechat groups (the
# ``channel`` column distinguishes them). The scraper wraps
# ``businessFriendList`` with the group-type switch (UNVERIFIED param — see
# :func:`app.scraper.whatsapp.scrape_whatsapp_groups`); records land with
# ``channel='whatsapp'``.
ENTITY_WHATSAPP_GROUP = "whatsapp_group"
ENTITY_MESSAGE = "message"
# M4 (/pc/ CRM module) — phone calls + SMS share no channel column with the
# wechat-family entities; they track their own SyncRun / SyncState rows.
@@ -703,6 +711,63 @@ async def sync_whatsapp_friends(
)
async def sync_whatsapp_groups(
session: AsyncSession,
client: YunkeClient,
*,
department_id: str | None = None,
) -> SyncResult:
"""Full re-scrape of all WhatsApp chat-groups and idempotent upsert.
Mirrors :func:`sync_wechat_groups` (the WA analogue) but with the WA
department-resolution path borrowed from :func:`sync_whatsapp_friends`:
the WA ``businessFriendList`` body REQUIRES ``departmentId`` (the
company-root dept id) or it returns empty. When omitted, the orchestrator
resolves it from the mirrored ``Department`` table via
:func:`_find_root_department_id` — which is why :func:`sync_departments`
must run first. If no root can be resolved, the scrape is SKIPPED with a
logged warning (returns a ``status='skipped'`` :class:`SyncResult` and
records no ``SyncRun``) so a missing org tree does not abort a multi-entity
batch.
Records land in the shared :class:`ChatGroup` table with
``channel='whatsapp'`` (stamped by :func:`_normalize_whatsapp_group`), so
:func:`upsert_chat_group` is reused directly — exactly the same pattern as
:func:`sync_wechat_groups`. The ``channel`` column distinguishes WA groups
from WeChat groups.
.. note::
UNVERIFIED (2026-08-11, risk-control): the WA group ``type="1"`` body
switch is inferred, not captured. See
:func:`app.scraper.whatsapp.scrape_whatsapp_groups`. If a post-risk-
control smoke returns 0 rows, the param name is the likely culprit.
"""
resolved = department_id
if resolved is None:
resolved = await _find_root_department_id(session)
if not resolved:
logger.warning(
"sync_whatsapp_groups: no root department_id resolved"
" (sync departments first?); skipping businessFriendList scrape"
)
return SyncResult(
entity=ENTITY_WHATSAPP_GROUP,
status="skipped",
fetched=0,
upserted=0,
error="no root department_id available",
)
return await _sync_full_rescrape(
session,
entity_type=ENTITY_WHATSAPP_GROUP,
channel=CHANNEL_WHATSAPP,
scope_id=None,
scrape=lambda: scrape_whatsapp_groups(client, department_id=resolved),
upsert=upsert_chat_group,
)
async def sync_whatsapp_messages(
session: AsyncSession,
client: YunkeClient,