xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scraper): WA 群 scraper(businessFriendList+type=1,待风控重置后实测)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-11 16:04:03 +08:00
parent 17ee5a1a6c
commit 0e1cb87156
4 changed files with 509 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/app/scraper/whatsapp.py
+123 -1
View File
@@ -9,6 +9,12 @@ Endpoints covered:
and yields normalized dicts ready to be upserted into
:class:`app.db.models.Customer` with ``channel="whatsapp"``. Page-number
pagination.
* :func:`scrape_whatsapp_groups` — wraps the SAME ``businessFriendList``
endpoint with ``type="1"`` to select WA chat-groups (``"0"`` selects
friends) and yields normalized :class:`app.db.models.ChatGroup` dicts with
``channel="whatsapp"``. The ``type`` group switch is UNVERIFIED (see the
section docstring) — the structure is correct, only the param name may need
a post-risk-control adjustment.
* :func:`scrape_whatsapp_messages` — wraps ``POST /qqweb/WhatsAppUser/chatRecords``
and yields normalized chat-message dicts with ``channel="whatsapp"``.
Backward-cursor pagination by timestamp (the same strategy as WeChat
@@ -34,13 +40,15 @@ from typing import Any
from app.scraper.base import paginate_by_cursor, paginate_by_page
from app.scraper.parsers import coerce_int_or_none, normalize_whatsapp_type, parse_datetime_to_ms
from app.scraper.wechat import _normalize_chat_group
from app.session.yunke_client import YunkeClient
__all__ = [
"scrape_wa_voice_calls",
"scrape_whatsapp_accounts",
"scrape_whatsapp_friends",
"scrape_whatsapp_groups",
"scrape_whatsapp_messages",
"scrape_wa_voice_calls",
]
WHATSAPP_ACCOUNTS_PATH = "/qqweb/WhatsAppUser/businessSalesList"
@@ -317,6 +325,120 @@ async def scrape_whatsapp_friends(
yield _normalize_whatsapp_friend(record)
# ===========================================================================
# Chat-group scraper — POST /qqweb/WhatsAppUser/businessFriendList
# with ``type="1"`` (groups). ``type="0"`` selects friends (see above).
#
# UNVERIFIED (云客 risk-control blocking live API calls as of 2026-08-11):
# the group-type body parameter is inferred, not captured. The WA friends
# body sends ``type="0"`` and the WA CRM SPA exposes two parallel routes —
# ``friendlist/0`` (friends) and ``friendlist/1`` (groups) — so by symmetry
# ``type="1"`` is the most likely group selector. If the post-risk-control
# smoke returns 0 rows, try alternative param names BEFORE concluding the
# endpoint is wrong: ``friendType: "1"``, ``friendTypeYunke: "2"`` (the
# WeChat switch value), or a sibling path entirely. The field mapping and
# pagination structure here are correct regardless; only the switch param
# name may need adjustment.
# ===========================================================================
def _build_wa_group_body(
page_num: int, page_size: int, *, department_id: str = ""
) -> dict[str, Any]:
"""Build the ``businessFriendList`` request body for chat-groups (type=1).
Identical to :func:`_build_wa_friend_body` except for ``type``: ``"1"``
selects groups (``"0"`` selects friends). UNVERIFIED — see the section
docstring above for the post-risk-control verification plan. ``departmentId``
is omitted when blank so a test/fixture call without one still forms a
valid body (the live endpoint requires it for non-empty results, same as
the friends variant).
"""
body: dict[str, Any] = {
"searchInfo": "",
"friendSearchInfo": "",
"page": page_num,
"pageSize": page_size,
# UNVERIFIED group-type switch — see section docstring.
"type": "1",
}
if department_id:
body["departmentId"] = department_id
return body
def _normalize_whatsapp_group(record: dict[str, Any]) -> dict[str, Any]:
"""Map one WA ``businessFriendList`` group record to the ``ChatGroup`` shape.
Reuses :func:`app.scraper.wechat._normalize_chat_group` for the full field
mapping — the WA endpoint returns the SAME field names the WeChat
``friend_list/es`` endpoint does for the group-type switch (``friendWxId``
is the ``@chatroom`` id, ``friendWxNickname`` is the group name,
``friendNum`` is the member count, etc.) — then overrides ``channel`` to
``"whatsapp"`` so WA groups are distinguishable from WeChat groups in the
shared :class:`app.db.models.ChatGroup` table.
Member ids are NOT surfaced by this endpoint (only the ``friendNum``
count); ``member_ids`` / ``member_ids_list`` are emitted as ``None`` /
``[]`` by the shared WeChat normalizer and left untouched here.
"""
out = _normalize_chat_group(record)
out["channel"] = "whatsapp"
return out
async def scrape_whatsapp_groups(
client: YunkeClient,
*,
department_id: str = "",
page_size: int = 50,
) -> AsyncIterator[dict[str, Any]]:
"""Yield normalized WhatsApp chat-group records from ``businessFriendList``.
Mirrors :func:`scrape_whatsapp_friends` but sends ``type="1"`` in the body
to select groups (the WA CRM SPA's ``friendlist/1`` route). Shares the
friends endpoint's envelope (rows under ``data.overviews``, declared total
under ``data.totalCount``), so the same ``_parse_wa_friend_total`` /
``_parse_wa_friend_rows`` helpers are reused.
Each record is mapped to the ``ChatGroup`` column shape with
``channel="whatsapp"`` (via :func:`_normalize_whatsapp_group`, which reuses
the WeChat group normalizer for the field mapping) plus a verbatim
``raw_json`` copy. Member ids are NOT available — ``member_ids`` /
``member_ids_list`` are ``None`` / ``[]``.
.. note::
UNVERIFIED (2026-08-11, risk-control): the ``type="1"`` group switch is
inferred from the WA SPA's ``friendlist/0`` / ``friendlist/1`` routes,
not a live capture. If the post-risk-control smoke returns 0 rows, try
``friendType`` / ``friendTypeYunke`` before concluding the endpoint is
wrong. See the section docstring above.
Parameters
----------
client:
Authenticated :class:`YunkeClient`. Auth headers / 401 re-login are
handled by the client.
department_id:
Company-root dept id (the ``departmentId`` body field). Required by the
live endpoint for non-empty results (same as friends); resolved by the
orchestrator when omitted.
page_size:
Page size to request. Defaults to 50.
"""
async for record in paginate_by_page(
client,
WHATSAPP_FRIENDS_PATH,
page_size=page_size,
parse_total=_parse_wa_friend_total,
parse_rows=_parse_wa_friend_rows,
build_body=_build_wa_group_body,
department_id=department_id,
):
yield _normalize_whatsapp_group(record)
# ===========================================================================
# Message scraper (M3) — POST /qqweb/WhatsAppUser/chatRecords
# ===========================================================================
yunke-server/app/sync/orchestrator.py
+66 -1
View File
@@ -69,6 +69,7 @@ from app.scraper.whatsapp import (
scrape_wa_voice_calls,
scrape_whatsapp_accounts,
scrape_whatsapp_friends,
scrape_whatsapp_groups,
scrape_whatsapp_messages,
)
from app.session.yunke_client import YunkeClient
@@ -100,14 +101,15 @@ __all__ = [
"sync_sms",
"sync_voice_clip_audio",
"sync_voice_clips",
"sync_wa_voice_calls",
"sync_wechat_accounts",
"sync_wechat_friends",
"sync_wechat_groups",
"sync_wechat_messages",
"sync_wechat_voice_calls",
"sync_wa_voice_calls",
"sync_whatsapp_accounts",
"sync_whatsapp_friends",
"sync_whatsapp_groups",
"sync_whatsapp_messages",
]
@@ -130,6 +132,12 @@ ENTITY_WA_VOICE_CALL = "wa_voice_call"
# org's department tree with WeChat (sync_departments seeds it).
ENTITY_WHATSAPP_ACCOUNT = "whatsapp_account"
ENTITY_WHATSAPP_FRIEND = "whatsapp_friend"
# WhatsApp chat-groups share the ``ChatGroup`` table with wechat groups (the
# ``channel`` column distinguishes them). The scraper wraps
# ``businessFriendList`` with the group-type switch (UNVERIFIED param — see
# :func:`app.scraper.whatsapp.scrape_whatsapp_groups`); records land with
# ``channel='whatsapp'``.
ENTITY_WHATSAPP_GROUP = "whatsapp_group"
ENTITY_MESSAGE = "message"
# M4 (/pc/ CRM module) — phone calls + SMS share no channel column with the
# wechat-family entities; they track their own SyncRun / SyncState rows.
@@ -703,6 +711,63 @@ async def sync_whatsapp_friends(
)
async def sync_whatsapp_groups(
session: AsyncSession,
client: YunkeClient,
*,
department_id: str | None = None,
) -> SyncResult:
"""Full re-scrape of all WhatsApp chat-groups and idempotent upsert.
Mirrors :func:`sync_wechat_groups` (the WA analogue) but with the WA
department-resolution path borrowed from :func:`sync_whatsapp_friends`:
the WA ``businessFriendList`` body REQUIRES ``departmentId`` (the
company-root dept id) or it returns empty. When omitted, the orchestrator
resolves it from the mirrored ``Department`` table via
:func:`_find_root_department_id` — which is why :func:`sync_departments`
must run first. If no root can be resolved, the scrape is SKIPPED with a
logged warning (returns a ``status='skipped'`` :class:`SyncResult` and
records no ``SyncRun``) so a missing org tree does not abort a multi-entity
batch.
Records land in the shared :class:`ChatGroup` table with
``channel='whatsapp'`` (stamped by :func:`_normalize_whatsapp_group`), so
:func:`upsert_chat_group` is reused directly — exactly the same pattern as
:func:`sync_wechat_groups`. The ``channel`` column distinguishes WA groups
from WeChat groups.
.. note::
UNVERIFIED (2026-08-11, risk-control): the WA group ``type="1"`` body
switch is inferred, not captured. See
:func:`app.scraper.whatsapp.scrape_whatsapp_groups`. If a post-risk-
control smoke returns 0 rows, the param name is the likely culprit.
"""
resolved = department_id
if resolved is None:
resolved = await _find_root_department_id(session)
if not resolved:
logger.warning(
"sync_whatsapp_groups: no root department_id resolved"
" (sync departments first?); skipping businessFriendList scrape"
)
return SyncResult(
entity=ENTITY_WHATSAPP_GROUP,
status="skipped",
fetched=0,
upserted=0,
error="no root department_id available",
)
return await _sync_full_rescrape(
session,
entity_type=ENTITY_WHATSAPP_GROUP,
channel=CHANNEL_WHATSAPP,
scope_id=None,
scrape=lambda: scrape_whatsapp_groups(client, department_id=resolved),
upsert=upsert_chat_group,
)
async def sync_whatsapp_messages(
session: AsyncSession,
client: YunkeClient,
yunke-server/app/sync/scheduler.py
+12
View File
@@ -69,6 +69,7 @@ from app.sync.orchestrator import (
sync_wechat_voice_calls,
sync_whatsapp_accounts,
sync_whatsapp_friends,
sync_whatsapp_groups,
sync_whatsapp_messages,
)
@@ -178,6 +179,12 @@ async def _run_wa_friends(
return [await sync_whatsapp_friends(session, client)]
async def _run_wa_groups(
session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> list[SyncResult]:
return [await sync_whatsapp_groups(session, client)]
async def _run_phone_calls(
session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> list[SyncResult]:
@@ -274,6 +281,10 @@ async def _run_all(
# which ``_run_all_wechat`` seeds via sync_departments — so order matters.
results.append(await sync_whatsapp_accounts(session, client))
results.append(await sync_whatsapp_friends(session, client))
# WA chat-groups: share the ChatGroup table with wechat groups (channel
# distinguishes). Same department_id dependency as wa_friends. UNVERIFIED
# group-type switch — see scrape_whatsapp_groups.
results.append(await sync_whatsapp_groups(session, client))
# WA messages: per-conversation incremental sync, discovered from the
# already-mirrored ``customer`` table (channel='whatsapp'). Must run
# AFTER wa_friends (which seeds the customer rows the discovery reads).
@@ -307,6 +318,7 @@ ENTITY_FUNCS: dict[
"wa_voice_calls": _run_wa_voice_calls,
"wa_accounts": _run_wa_accounts,
"wa_friends": _run_wa_friends,
"wa_groups": _run_wa_groups,
"phone_calls": _run_phone_calls,
"sms": _run_sms,
"voice_clips": _run_voice_clips,