feat(group-members): 群列表对账云客总数+群详情成员墙(头像+昵称)
用户报两点:①微信群列表与云客群统计总数不一致(实测云客 1,864 vs 本地 2,012——friend_list/es 时代残留的 148 行 owner 组合已不在 getChatroomList 返回里);②群详情成员列表全是 wxid 无法识别。 - chatroom-stats 全窗模式加对账:名单取满(>=声明 total)才删本地不在 云客名单里的 chat_group(wechat)行——页脚从此与云客一字不差;短页宁 可不对账防误删 - backfill_group_members 增强:头像列(消息最新一条 sender_head_url)+ customer 兜底 UPDATE(member_id=friend_id 命中本司好友补昵称/头像, 覆盖未发言成员);full_sync 8b 原位生效 - 群详情 drawer 成员墙:头像+昵称网格(群昵称>好友昵称>wxid 回退+wxid 副行),替代原 memberIds 原始串;仅微信 tab(WA 群成员未派生) 722 后端测试绿,前端构建过。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -1,12 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
"""从群消息派生 group_member(发言者=成员)。
|
||||
"""从群消息派生 group_member(发言者=成员)+ customer 兜底补全昵称/头像。
|
||||
|
||||
云客 API 不单独给群成员列表(chatroom payload 的 memberIds 我们抓不到,
|
||||
2026-08-20 确认),但群消息的 talker 就是发言成员的 wxid、sender_name 是其
|
||||
群昵称——368k 群消息足够派生出高质量成员表(覆盖所有发过言的成员)。
|
||||
2026-08-20 确认),但群消息的 talker 就是发言成员的 wxid、sender_name/
|
||||
sender_head_url 是其群昵称/头像——群消息足够派生出高质量成员表(覆盖
|
||||
所有发过言的成员)。昵称/头像取该成员在群里**最新一条**消息的值。
|
||||
|
||||
昵称取该成员在群里最新一条消息的 sender_name(SQLite 的 max()+裸列行为,
|
||||
取 max(timestamp) 同行的值)。幂等可重跑;建议在 group-messages 之后执行。
|
||||
未发言的成员(被拉进群从没说话)从 customer 表兜底:member_id=friend_id
|
||||
命中本司好友时补好友昵称/头像(2026-09-10 群详情成员墙)。
|
||||
|
||||
幂等可重跑;建议在 group-messages 之后执行(full_sync_all.sh 8b 已挂)。
|
||||
|
||||
用法:
|
||||
cd yunke-server && DB_URL="sqlite+aiosqlite:///./smoke.db" \
|
||||
@@ -26,13 +29,14 @@ from app.storage.repos import _upsert_one
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
log = logging.getLogger("backfill_group_members")
|
||||
|
||||
# row_number() 窗口取 timestamp 最新一行的 sender_name:MySQL 8 的
|
||||
# only_full_group_by 拒绝 SQLite 式「max()+裸列」(1055),窗口写法两库通用
|
||||
# 且语义严格(最新一条的昵称),不依赖方言碰巧行为
|
||||
# row_number() 窗口取 timestamp 最新一行的 sender_name/sender_head_url:
|
||||
# MySQL 8 的 only_full_group_by 拒绝 SQLite 式「max()+裸列」(1055),窗口
|
||||
# 写法两库通用且语义严格(最新一条的昵称/头像),不依赖方言碰巧行为
|
||||
SQL = sa_text("""
|
||||
SELECT room_id, talker_id, sender_name, last_ts, msg_count
|
||||
SELECT room_id, talker_id, sender_name, sender_head_url, last_ts, msg_count
|
||||
FROM (
|
||||
SELECT room_id, talker_id, sender_name, timestamp AS last_ts,
|
||||
SELECT room_id, talker_id, sender_name, sender_head_url,
|
||||
timestamp AS last_ts,
|
||||
count(*) OVER (PARTITION BY room_id, talker_id) AS msg_count,
|
||||
row_number() OVER (
|
||||
PARTITION BY room_id, talker_id ORDER BY timestamp DESC
|
||||
@@ -44,6 +48,17 @@ FROM (
|
||||
WHERE rn = 1
|
||||
""")
|
||||
|
||||
# 未发言成员兜底:member_id 命中 customer(本司好友)时补 NULL 的昵称/头像。
|
||||
# customer 全局一人一行(PK=friend_id),join 不放大行数;COALESCE 只补不清。
|
||||
CUSTOMER_FALLBACK = sa_text("""
|
||||
UPDATE group_member gm
|
||||
JOIN customer c ON c.friend_id = gm.member_id AND c.channel = gm.channel
|
||||
SET gm.nickname = COALESCE(gm.nickname, c.friend_nickname),
|
||||
gm.avatar = COALESCE(gm.avatar, c.head_image_url)
|
||||
WHERE gm.channel = 'wechat'
|
||||
AND (gm.nickname IS NULL OR gm.avatar IS NULL)
|
||||
""")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with async_engine.connect() as conn:
|
||||
@@ -51,12 +66,13 @@ async def main() -> None:
|
||||
log.info(f"派生出 {len(rows)} 个 (群, 成员) 对")
|
||||
|
||||
async with async_session_maker() as session:
|
||||
for room_id, talker_id, sender_name, last_ts, msg_count in rows:
|
||||
for room_id, talker_id, sender_name, head_url, last_ts, msg_count in rows:
|
||||
await _upsert_one(session, GroupMember, {
|
||||
"channel": "wechat",
|
||||
"room_id": room_id,
|
||||
"member_id": talker_id,
|
||||
"nickname": sender_name,
|
||||
"avatar": head_url,
|
||||
# raw_json 附加派生统计,抽屉可见
|
||||
"raw_json": {
|
||||
"derived_from": "group_messages",
|
||||
@@ -65,7 +81,13 @@ async def main() -> None:
|
||||
},
|
||||
})
|
||||
await session.commit()
|
||||
log.info(f"DONE: group_member upserted={len(rows)}")
|
||||
|
||||
# 兜底补未发言成员(独立事务,幂等)
|
||||
async with async_engine.begin() as conn:
|
||||
res = await conn.execute(CUSTOMER_FALLBACK)
|
||||
log.info(f"customer 兜底补全 {res.rowcount} 行昵称/头像")
|
||||
|
||||
log.info("DONE: group_member upsert 完成")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1779,9 +1779,12 @@ def _normalize_chatroom_stat_row(row: dict, owner_wxid: str) -> dict:
|
||||
return rec
|
||||
|
||||
|
||||
async def _chatroom_rows(page, dept: str, start_str: str, end_str: str, max_pages: int = 100):
|
||||
"""翻完 getChatroomList 一个窗,返回全部行(list)。"""
|
||||
async def _chatroom_rows(
|
||||
page, dept: str, start_str: str, end_str: str, max_pages: int = 100
|
||||
) -> tuple[list[dict], int]:
|
||||
"""翻完 getChatroomList 一个窗,返回 (全部行, 云客声明 total)。"""
|
||||
rows_all: list[dict] = []
|
||||
declared = 0
|
||||
page_num = 1
|
||||
while page_num <= max_pages:
|
||||
body = await _browser_fetch(
|
||||
@@ -1794,13 +1797,13 @@ async def _chatroom_rows(page, dept: str, start_str: str, end_str: str, max_page
|
||||
data = body.get("data") or {}
|
||||
rows = data.get("overviews") or []
|
||||
rows_all.extend(rows)
|
||||
total = data.get("totalCount") or 0
|
||||
logger.info(f" chatroom-list page {page_num}: {len(rows)} rows (total {total})")
|
||||
if not rows or page_num * 50 >= (total or 0):
|
||||
declared = data.get("totalCount") or declared
|
||||
logger.info(f" chatroom-list page {page_num}: {len(rows)} rows (total {declared})")
|
||||
if not rows or page_num * 50 >= (declared or 0):
|
||||
break
|
||||
page_num += 1
|
||||
await asyncio.sleep(1.2)
|
||||
return rows_all
|
||||
return rows_all, declared or 0
|
||||
|
||||
|
||||
async def _owner_alias_map(conn):
|
||||
@@ -1851,9 +1854,11 @@ async def scrape_chatroom_stats(cookie_str: str, day: str | None = None):
|
||||
|
||||
group_recs: list[dict] = []
|
||||
stats_recs: list[dict] = []
|
||||
declared = 0
|
||||
async with PageScraper(cookie_str) as scraper:
|
||||
page = await _open_fetch_page(scraper)
|
||||
for row in await _chatroom_rows(page, dept, start_str, end_str):
|
||||
rows, declared = await _chatroom_rows(page, dept, start_str, end_str)
|
||||
for row in rows:
|
||||
rid = row.get("chatRoomId")
|
||||
if not rid:
|
||||
continue
|
||||
@@ -1877,9 +1882,37 @@ async def scrape_chatroom_stats(cookie_str: str, day: str | None = None):
|
||||
await _upsert_all(upsert_chat_group, group_recs)
|
||||
if stats_recs:
|
||||
await _upsert_all(upsert_chat_group_member_stats, stats_recs)
|
||||
|
||||
# 对账(2026-09-10,全窗模式):删掉本地有而云客名单里没有的 chat_group 行
|
||||
# ——friend_list/es 时代的残留行(owner 组合已不在 getChatroomList 返回,
|
||||
# 实测多 148 行致页脚与云客不一致)。名单取满(>=声明 total)才删,防短页
|
||||
# 误删;只动 wechat(WA 群另一来源);GroupMember 按 room 共享不动。
|
||||
pruned = 0
|
||||
if day_key is None and group_recs and declared and len(group_recs) >= declared:
|
||||
from sqlalchemy import tuple_ as sa_tuple
|
||||
|
||||
from app.db.base import async_session_maker
|
||||
from app.db.models import ChatGroup
|
||||
|
||||
keep = {(r["owner_account_id"], r["room_id"]) for r in group_recs}
|
||||
keep_list = list(keep)
|
||||
async with async_session_maker() as session:
|
||||
for i in range(0, len(keep_list), 500):
|
||||
chunk = keep_list[i:i + 500]
|
||||
res = await session.execute(
|
||||
sa.delete(ChatGroup).where(
|
||||
ChatGroup.channel == "wechat",
|
||||
sa.tuple_(
|
||||
ChatGroup.owner_account_id, ChatGroup.room_id
|
||||
).not_in(sa_tuple(chunk)),
|
||||
)
|
||||
)
|
||||
pruned += int(res.rowcount or 0)
|
||||
await session.commit()
|
||||
|
||||
logger.info(
|
||||
f"DONE: chatroom-stats day={day or '全窗'} groups={len(group_recs)} "
|
||||
f"stats_rows={len(stats_recs)}"
|
||||
f"stats_rows={len(stats_recs)} pruned={pruned}"
|
||||
)
|
||||
return len(group_recs), len(stats_recs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user