feat: wechat group+member and talker scrapers
Add scrape_wechat_groups (POST /wechatweb/chatroom/getChatroomList) and scrape_wechat_talkers (POST /wechatweb/wxFriends/talkerList), both reusing the page-number paginator. Groups parse the semicolon-joined memberIds into a member_ids_list so upsert_chat_group can mirror GroupMember rows (composite PK channel/room_id/member_id) idempotently alongside the ChatGroup row. Talkers are a sync-planning input (member_count>0 marks a group conversation); no dedicated table. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -22,10 +22,19 @@ import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db.base import Base
|
||||
from app.db.models import Customer, Department, Message, Staff, WechatAccount
|
||||
from app.db.models import (
|
||||
ChatGroup,
|
||||
Customer,
|
||||
Department,
|
||||
GroupMember,
|
||||
Message,
|
||||
Staff,
|
||||
WechatAccount,
|
||||
)
|
||||
from app.storage.repos import (
|
||||
_coerce_bool,
|
||||
_coerce_int,
|
||||
upsert_chat_group,
|
||||
upsert_customer,
|
||||
upsert_customers,
|
||||
upsert_department,
|
||||
@@ -701,3 +710,178 @@ async def test_upsert_wechat_account_without_user_id_skips_staff():
|
||||
assert staff_rows == []
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ChatGroup + GroupMember upsert (M2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_upsert_chat_group_inserts_group_and_members():
|
||||
"""Upsert writes the ChatGroup row AND one GroupMember per member_ids_list."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"room_id": "12345678901@chatroom",
|
||||
"channel": "wechat",
|
||||
"room_name": "VIP客户交流群",
|
||||
"owner_account_id": "wxid_staff_alice01",
|
||||
"member_count": 3,
|
||||
"member_ids": "wxid_m1;wxid_m2;wxid_m3",
|
||||
"member_ids_list": ["wxid_m1", "wxid_m2", "wxid_m3"],
|
||||
"message_count": 4521,
|
||||
}
|
||||
await upsert_chat_group(session, rec)
|
||||
await session.commit()
|
||||
|
||||
group = (
|
||||
await session.execute(
|
||||
sa.select(ChatGroup).where(
|
||||
ChatGroup.room_id == "12345678901@chatroom"
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
assert group.room_name == "VIP客户交流群"
|
||||
assert group.owner_account_id == "wxid_staff_alice01"
|
||||
assert group.member_count == 3
|
||||
assert group.message_count == 4521
|
||||
# The raw semicolon-joined string is stored on the group row.
|
||||
assert group.member_ids == "wxid_m1;wxid_m2;wxid_m3"
|
||||
|
||||
# Members: one row per member_ids_list entry, composite PK.
|
||||
members = (
|
||||
await session.execute(
|
||||
sa.select(GroupMember)
|
||||
.where(GroupMember.room_id == "12345678901@chatroom")
|
||||
.order_by(GroupMember.member_id)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(members) == 3
|
||||
assert {m.member_id for m in members} == {"wxid_m1", "wxid_m2", "wxid_m3"}
|
||||
assert {m.channel for m in members} == {"wechat"}
|
||||
# Phase-0: per-member nickname/avatar are not yet populated.
|
||||
assert all(m.nickname is None for m in members)
|
||||
assert all(m.avatar is None for m in members)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_chat_group_reupsert_updates_group_keeps_members_unique():
|
||||
"""Re-upserting a group refreshes its row; members stay unique (no dups)."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec1 = {
|
||||
"room_id": "98765432109@chatroom",
|
||||
"channel": "wechat",
|
||||
"room_name": "Old Name",
|
||||
"member_count": 2,
|
||||
"member_ids": "wxid_m1;wxid_m2",
|
||||
"member_ids_list": ["wxid_m1", "wxid_m2"],
|
||||
"message_count": 10,
|
||||
}
|
||||
await upsert_chat_group(session, rec1)
|
||||
await session.commit()
|
||||
|
||||
rec2 = dict(
|
||||
rec1,
|
||||
room_name="New Name",
|
||||
message_count=99,
|
||||
)
|
||||
await upsert_chat_group(session, rec2)
|
||||
await session.commit()
|
||||
|
||||
# One group row, refreshed (name + count updated, not duplicated).
|
||||
groups = (
|
||||
await session.execute(
|
||||
sa.select(ChatGroup)
|
||||
.where(ChatGroup.room_id == "98765432109@chatroom")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(groups) == 1
|
||||
assert groups[0].room_name == "New Name"
|
||||
assert groups[0].message_count == 99
|
||||
|
||||
# Members still exactly 2 (idempotent re-upsert, no duplication).
|
||||
members = (
|
||||
await session.execute(
|
||||
sa.select(GroupMember).where(
|
||||
GroupMember.room_id == "98765432109@chatroom"
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(members) == 2
|
||||
assert {m.member_id for m in members} == {"wxid_m1", "wxid_m2"}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_chat_group_drops_empty_member_id():
|
||||
"""An empty member id in member_ids_list is dropped (no empty-PK row).
|
||||
|
||||
The scraper already drops empties when building ``member_ids_list``; this
|
||||
test exercises the repo's own defensive guard so a hand-built record that
|
||||
slips an empty id through cannot create a degenerate empty-PK member row.
|
||||
"""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"room_id": "55555555555@chatroom",
|
||||
"channel": "wechat",
|
||||
# Empty string sneaks through; repo must skip it.
|
||||
"member_ids_list": ["", "wxid_only", ""],
|
||||
}
|
||||
await upsert_chat_group(session, rec)
|
||||
await session.commit()
|
||||
|
||||
members = (
|
||||
await session.execute(
|
||||
sa.select(GroupMember).where(
|
||||
GroupMember.room_id == "55555555555@chatroom"
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(members) == 1
|
||||
assert members[0].member_id == "wxid_only"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_chat_group_with_no_members_writes_group_only():
|
||||
"""A group with an empty member list still writes the group row."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"room_id": "empty@chatroom",
|
||||
"channel": "wechat",
|
||||
"room_name": "No Members Yet",
|
||||
"member_ids_list": [],
|
||||
}
|
||||
await upsert_chat_group(session, rec)
|
||||
await session.commit()
|
||||
|
||||
# Group row written.
|
||||
group = (
|
||||
await session.execute(
|
||||
sa.select(ChatGroup).where(ChatGroup.room_id == "empty@chatroom")
|
||||
)
|
||||
).scalar_one()
|
||||
assert group.room_name == "No Members Yet"
|
||||
|
||||
# No member rows.
|
||||
members = (
|
||||
await session.execute(
|
||||
sa.select(GroupMember).where(GroupMember.room_id == "empty@chatroom")
|
||||
)
|
||||
).scalars().all()
|
||||
assert members == []
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
Reference in New Issue
Block a user