feat: wechat group+member and talker scrapers
Add scrape_wechat_groups (POST /wechatweb/chatroom/getChatroomList) and scrape_wechat_talkers (POST /wechatweb/wxFriends/talkerList), both reusing the page-number paginator. Groups parse the semicolon-joined memberIds into a member_ids_list so upsert_chat_group can mirror GroupMember rows (composite PK channel/room_id/member_id) idempotently alongside the ChatGroup row. Talkers are a sync-planning input (member_count>0 marks a group conversation); no dedicated table. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -48,9 +48,18 @@ import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from app.db.models import Customer, Department, Message, Staff, WechatAccount
|
||||
from app.db.models import (
|
||||
ChatGroup,
|
||||
Customer,
|
||||
Department,
|
||||
GroupMember,
|
||||
Message,
|
||||
Staff,
|
||||
WechatAccount,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"upsert_chat_group",
|
||||
"upsert_customer",
|
||||
"upsert_customers",
|
||||
"upsert_department",
|
||||
@@ -383,3 +392,54 @@ async def upsert_staff_from_account(session: AsyncSession, rec: dict[str, Any])
|
||||
if key in rec:
|
||||
staff_rec[key] = rec[key]
|
||||
await upsert_staff(session, staff_rec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ChatGroup + GroupMember upsert (M2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def upsert_chat_group(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
"""Idempotently upsert one :class:`ChatGroup` AND its :class:`GroupMember` rows.
|
||||
|
||||
PK = ``room_id``. The group row is upserted first; then, for every id in
|
||||
the record's ``member_ids_list`` (the scraper-parsed, empties-dropped
|
||||
member wxid list), a :class:`GroupMember` row is upserted against the
|
||||
composite PK ``(channel, room_id, member_id)``. Re-scraping a group
|
||||
therefore refreshes the group row and re-asserts its members without
|
||||
duplication (idempotent).
|
||||
|
||||
The ``member_ids_list`` key is the parsed list shape produced by
|
||||
:func:`app.scraper.wechat._normalize_chat_group`; it is NOT a
|
||||
``ChatGroup`` column, so it is dropped by :func:`_build_column_values`
|
||||
when writing the group row (the raw semicolon-joined string is preserved
|
||||
on ``ChatGroup.member_ids``).
|
||||
|
||||
Per-member ``nickname`` / ``avatar`` / ``display_name`` are left ``None``
|
||||
here — Phase-0 does not yet resolve them (they need a separate per-member
|
||||
API; see data dictionary §9.6). The repo writes a minimal member row keyed
|
||||
by the composite PK so membership is captured now and enriched later.
|
||||
"""
|
||||
await _upsert_one(session, ChatGroup, rec)
|
||||
|
||||
channel = rec.get("channel") or "wechat"
|
||||
room_id = rec.get("room_id")
|
||||
# No room_id -> nothing to key member rows on (defensive; the scraper
|
||||
# always sets room_id). Skip the member loop rather than raising so a
|
||||
# malformed record still lands its (PK-less) group upsert attempt above.
|
||||
if not room_id:
|
||||
return
|
||||
|
||||
member_ids: list[Any] = list(rec.get("member_ids_list") or [])
|
||||
for member_id in member_ids:
|
||||
# Defensive: never insert an empty/falsy member id (would collide on
|
||||
# an empty-PK row). The scraper already drops empties, but a hand-
|
||||
# built record could slip one through.
|
||||
if not member_id:
|
||||
continue
|
||||
member_rec: dict[str, Any] = {
|
||||
"channel": channel,
|
||||
"room_id": room_id,
|
||||
"member_id": member_id,
|
||||
}
|
||||
await _upsert_one(session, GroupMember, member_rec)
|
||||
|
||||
Reference in New Issue
Block a user