feat: wechat group+member and talker scrapers

Add scrape_wechat_groups (POST /wechatweb/chatroom/getChatroomList) and
scrape_wechat_talkers (POST /wechatweb/wxFriends/talkerList), both reusing
the page-number paginator. Groups parse the semicolon-joined memberIds into
a member_ids_list so upsert_chat_group can mirror GroupMember rows
(composite PK channel/room_id/member_id) idempotently alongside the
ChatGroup row. Talkers are a sync-planning input (member_count>0 marks a
group conversation); no dedicated table.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-01 13:54:16 +08:00
parent 2b79e7a232
commit 2a74d3888d
7 changed files with 1163 additions and 4 deletions
+185 -1
View File
@@ -22,10 +22,19 @@ import sqlalchemy as sa
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.db.base import Base
from app.db.models import Customer, Department, Message, Staff, WechatAccount
from app.db.models import (
ChatGroup,
Customer,
Department,
GroupMember,
Message,
Staff,
WechatAccount,
)
from app.storage.repos import (
_coerce_bool,
_coerce_int,
upsert_chat_group,
upsert_customer,
upsert_customers,
upsert_department,
@@ -701,3 +710,178 @@ async def test_upsert_wechat_account_without_user_id_skips_staff():
assert staff_rows == []
finally:
await engine.dispose()
# ---------------------------------------------------------------------------
# ChatGroup + GroupMember upsert (M2)
# ---------------------------------------------------------------------------
async def test_upsert_chat_group_inserts_group_and_members():
"""Upsert writes the ChatGroup row AND one GroupMember per member_ids_list."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"room_id": "12345678901@chatroom",
"channel": "wechat",
"room_name": "VIP客户交流群",
"owner_account_id": "wxid_staff_alice01",
"member_count": 3,
"member_ids": "wxid_m1;wxid_m2;wxid_m3",
"member_ids_list": ["wxid_m1", "wxid_m2", "wxid_m3"],
"message_count": 4521,
}
await upsert_chat_group(session, rec)
await session.commit()
group = (
await session.execute(
sa.select(ChatGroup).where(
ChatGroup.room_id == "12345678901@chatroom"
)
)
).scalar_one()
assert group.room_name == "VIP客户交流群"
assert group.owner_account_id == "wxid_staff_alice01"
assert group.member_count == 3
assert group.message_count == 4521
# The raw semicolon-joined string is stored on the group row.
assert group.member_ids == "wxid_m1;wxid_m2;wxid_m3"
# Members: one row per member_ids_list entry, composite PK.
members = (
await session.execute(
sa.select(GroupMember)
.where(GroupMember.room_id == "12345678901@chatroom")
.order_by(GroupMember.member_id)
)
).scalars().all()
assert len(members) == 3
assert {m.member_id for m in members} == {"wxid_m1", "wxid_m2", "wxid_m3"}
assert {m.channel for m in members} == {"wechat"}
# Phase-0: per-member nickname/avatar are not yet populated.
assert all(m.nickname is None for m in members)
assert all(m.avatar is None for m in members)
finally:
await engine.dispose()
async def test_upsert_chat_group_reupsert_updates_group_keeps_members_unique():
"""Re-upserting a group refreshes its row; members stay unique (no dups)."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec1 = {
"room_id": "98765432109@chatroom",
"channel": "wechat",
"room_name": "Old Name",
"member_count": 2,
"member_ids": "wxid_m1;wxid_m2",
"member_ids_list": ["wxid_m1", "wxid_m2"],
"message_count": 10,
}
await upsert_chat_group(session, rec1)
await session.commit()
rec2 = dict(
rec1,
room_name="New Name",
message_count=99,
)
await upsert_chat_group(session, rec2)
await session.commit()
# One group row, refreshed (name + count updated, not duplicated).
groups = (
await session.execute(
sa.select(ChatGroup)
.where(ChatGroup.room_id == "98765432109@chatroom")
.execution_options(populate_existing=True)
)
).scalars().all()
assert len(groups) == 1
assert groups[0].room_name == "New Name"
assert groups[0].message_count == 99
# Members still exactly 2 (idempotent re-upsert, no duplication).
members = (
await session.execute(
sa.select(GroupMember).where(
GroupMember.room_id == "98765432109@chatroom"
)
)
).scalars().all()
assert len(members) == 2
assert {m.member_id for m in members} == {"wxid_m1", "wxid_m2"}
finally:
await engine.dispose()
async def test_upsert_chat_group_drops_empty_member_id():
"""An empty member id in member_ids_list is dropped (no empty-PK row).
The scraper already drops empties when building ``member_ids_list``; this
test exercises the repo's own defensive guard so a hand-built record that
slips an empty id through cannot create a degenerate empty-PK member row.
"""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"room_id": "55555555555@chatroom",
"channel": "wechat",
# Empty string sneaks through; repo must skip it.
"member_ids_list": ["", "wxid_only", ""],
}
await upsert_chat_group(session, rec)
await session.commit()
members = (
await session.execute(
sa.select(GroupMember).where(
GroupMember.room_id == "55555555555@chatroom"
)
)
).scalars().all()
assert len(members) == 1
assert members[0].member_id == "wxid_only"
finally:
await engine.dispose()
async def test_upsert_chat_group_with_no_members_writes_group_only():
"""A group with an empty member list still writes the group row."""
engine = await _make_engine()
try:
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async with SessionLocal() as session:
rec = {
"room_id": "empty@chatroom",
"channel": "wechat",
"room_name": "No Members Yet",
"member_ids_list": [],
}
await upsert_chat_group(session, rec)
await session.commit()
# Group row written.
group = (
await session.execute(
sa.select(ChatGroup).where(ChatGroup.room_id == "empty@chatroom")
)
).scalar_one()
assert group.room_name == "No Members Yet"
# No member rows.
members = (
await session.execute(
sa.select(GroupMember).where(GroupMember.room_id == "empty@chatroom")
)
).scalars().all()
assert members == []
finally:
await engine.dispose()