feat(scraper): WA群抓取(type=1真机验证,223群)+群成员派生(12840成员/750群)
- wa-groups 实体:businessFriendList type=1 真机验证返回群,行结构是好友字段名 (friendUid=@g.us 群ID/nickName=群名),专用 _normalize_wa_group_row - backfill_group_members.py:无独立成员API,从群消息派生(talker=成员,sender_name=昵称) - full_sync_all.sh:+4步(5b WA群/语音后录音重链接/8b 群消息+成员/消息上限99999) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python
|
||||
"""从群消息派生 group_member(发言者=成员)。
|
||||
|
||||
云客 API 不单独给群成员列表(chatroom payload 的 memberIds 我们抓不到,
|
||||
2026-08-20 确认),但群消息的 talker 就是发言成员的 wxid、sender_name 是其
|
||||
群昵称——368k 群消息足够派生出高质量成员表(覆盖所有发过言的成员)。
|
||||
|
||||
昵称取该成员在群里最新一条消息的 sender_name(SQLite 的 max()+裸列行为,
|
||||
取 max(timestamp) 同行的值)。幂等可重跑;建议在 group-messages 之后执行。
|
||||
|
||||
用法:
|
||||
cd yunke-server && DB_URL="sqlite+aiosqlite:///./smoke.db" \
|
||||
.venv/Scripts/python.exe -m scripts.backfill_group_members
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
from app.db.base import async_engine, async_session_maker
|
||||
from app.db.models import GroupMember
|
||||
from app.storage.repos import _upsert_one
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
log = logging.getLogger("backfill_group_members")
|
||||
|
||||
# max(timestamp) 裸列:SQLite 保证 sender_name 取自 timestamp 最大的那行
|
||||
SQL = sa_text("""
|
||||
SELECT room_id, talker_id, sender_name, max(timestamp) AS last_ts, count(*) AS msg_count
|
||||
FROM message
|
||||
WHERE channel = 'wechat' AND room_id IS NOT NULL AND room_id != ''
|
||||
AND talker_id IS NOT NULL AND talker_id != ''
|
||||
GROUP BY room_id, talker_id
|
||||
""")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with async_engine.connect() as conn:
|
||||
rows = (await conn.execute(SQL)).fetchall()
|
||||
log.info(f"派生出 {len(rows)} 个 (群, 成员) 对")
|
||||
|
||||
async with async_session_maker() as session:
|
||||
for room_id, talker_id, sender_name, last_ts, msg_count in rows:
|
||||
await _upsert_one(session, GroupMember, {
|
||||
"channel": "wechat",
|
||||
"room_id": room_id,
|
||||
"member_id": talker_id,
|
||||
"nickname": sender_name,
|
||||
# raw_json 附加派生统计,抽屉可见
|
||||
"raw_json": {
|
||||
"derived_from": "group_messages",
|
||||
"msg_count": msg_count,
|
||||
"last_msg_ts": last_ts,
|
||||
},
|
||||
})
|
||||
await session.commit()
|
||||
log.info(f"DONE: group_member upserted={len(rows)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user