feat: wechat group+member and talker scrapers

Add scrape_wechat_groups (POST /wechatweb/chatroom/getChatroomList) and
scrape_wechat_talkers (POST /wechatweb/wxFriends/talkerList), both reusing
the page-number paginator. Groups parse the semicolon-joined memberIds into
a member_ids_list so upsert_chat_group can mirror GroupMember rows
(composite PK channel/room_id/member_id) idempotently alongside the
ChatGroup row. Talkers are a sync-planning input (member_count>0 marks a
group conversation); no dedicated table.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 13:54:16 +08:00
parent 2b79e7a232
commit 2a74d3888d
7 changed files with 1163 additions and 4 deletions
+61 -1
View File
@@ -48,9 +48,18 @@ import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import DeclarativeBase
from app.db.models import Customer, Department, Message, Staff, WechatAccount
from app.db.models import (
ChatGroup,
Customer,
Department,
GroupMember,
Message,
Staff,
WechatAccount,
)
__all__ = [
"upsert_chat_group",
"upsert_customer",
"upsert_customers",
"upsert_department",
@@ -383,3 +392,54 @@ async def upsert_staff_from_account(session: AsyncSession, rec: dict[str, Any])
if key in rec:
staff_rec[key] = rec[key]
await upsert_staff(session, staff_rec)
# ---------------------------------------------------------------------------
# ChatGroup + GroupMember upsert (M2)
# ---------------------------------------------------------------------------
async def upsert_chat_group(session: AsyncSession, rec: dict[str, Any]) -> None:
"""Idempotently upsert one :class:`ChatGroup` AND its :class:`GroupMember` rows.
PK = ``room_id``. The group row is upserted first; then, for every id in
the record's ``member_ids_list`` (the scraper-parsed, empties-dropped
member wxid list), a :class:`GroupMember` row is upserted against the
composite PK ``(channel, room_id, member_id)``. Re-scraping a group
therefore refreshes the group row and re-asserts its members without
duplication (idempotent).
The ``member_ids_list`` key is the parsed list shape produced by
:func:`app.scraper.wechat._normalize_chat_group`; it is NOT a
``ChatGroup`` column, so it is dropped by :func:`_build_column_values`
when writing the group row (the raw semicolon-joined string is preserved
on ``ChatGroup.member_ids``).
Per-member ``nickname`` / ``avatar`` / ``display_name`` are left ``None``
here — Phase-0 does not yet resolve them (they need a separate per-member
API; see data dictionary §9.6). The repo writes a minimal member row keyed
by the composite PK so membership is captured now and enriched later.
"""
await _upsert_one(session, ChatGroup, rec)
channel = rec.get("channel") or "wechat"
room_id = rec.get("room_id")
# No room_id -> nothing to key member rows on (defensive; the scraper
# always sets room_id). Skip the member loop rather than raising so a
# malformed record still lands its (PK-less) group upsert attempt above.
if not room_id:
return
member_ids: list[Any] = list(rec.get("member_ids_list") or [])
for member_id in member_ids:
# Defensive: never insert an empty/falsy member id (would collide on
# an empty-PK row). The scraper already drops empties, but a hand-
# built record could slip one through.
if not member_id:
continue
member_rec: dict[str, Any] = {
"channel": channel,
"room_id": room_id,
"member_id": member_id,
}
await _upsert_one(session, GroupMember, member_rec)