feat(scraper): WA群抓取(type=1真机验证,223群)+群成员派生(12840成员/750群)

- wa-groups 实体:businessFriendList type=1 真机验证返回群,行结构是好友字段名
  (friendUid=@g.us 群ID/nickName=群名),专用 _normalize_wa_group_row
- backfill_group_members.py:无独立成员API,从群消息派生(talker=成员,sender_name=昵称)
- full_sync_all.sh:+4步(5b WA群/语音后录音重链接/8b 群消息+成员/消息上限99999)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-20 09:21:20 +08:00
parent f92aac1a3d
commit 02dd77a62f
3 changed files with 153 additions and 2 deletions
+81
View File
@@ -946,6 +946,84 @@ async def scrape_wa_friends(cookie_str: str, max_pages: int = 2000):
return total_fetched, total_upserted
def _normalize_wa_group_row(row: dict) -> dict | None:
"""businessFriendList type=1 行 → ChatGroup 形状(真机确认 2026-08-20)。
type=1 返回的是好友字段名结构:friendUid=群 id(@g.us 后缀)、nickName=
群名、uid=员工 WA 号、salesNickName=员工 WA 昵称——与微信群的
friendWxId/friendWxNickname 完全不同,故不能复用 _normalize_chat_group。
非 @g.us 行(异常混入)返回 None。
"""
friend_uid = str(row.get("friendUid") or "")
if not friend_uid.endswith("@g.us"):
return None
return {
"room_id": friend_uid,
"channel": "whatsapp",
"room_name": row.get("nickName"),
"owner_account_id": row.get("uid"),
"owner_account_nickname": row.get("salesNickName"),
"owner_user_id": row.get("userId"),
"owner_user_phone": row.get("salePhone"),
"message_count": row.get("messageGroupCount"),
"raw_json": row,
}
async def scrape_wa_groups(cookie_str: str, max_pages: int = 200):
"""Scrape WhatsApp GROUPS via browser fetch POST businessFriendList type=1.
Same endpoint as WA friends with ``type:"1"`` — live-verified 2026-08-20:
rows carry friendUid ending ``@g.us`` (group id), nickName = group name.
Envelope: data.overviews; upsert to chat_group (channel='whatsapp').
"""
from app.db.base import async_engine
from sqlalchemy import text as sa_text
from app.storage.repos import upsert_chat_group
async with async_engine.connect() as conn:
row = (await conn.execute(sa_text(
"SELECT dept_id FROM department WHERE parent_id IS NULL LIMIT 1"
))).first()
department_id = row[0] if row else ""
if not department_id:
logger.warning("wa-groups: no root department in DB — abort")
return 0, 0
page_size = 50
total_fetched = 0
total_upserted = 0
async with PageScraper(cookie_str) as scraper:
page = await _open_fetch_page(scraper)
page_num = 1
while page_num <= max_pages:
body = await _browser_fetch(
page,
INTERCEPT_PATHS["wa_friend"],
json_body={
"searchInfo": "",
"friendSearchInfo": "",
"page": page_num,
"pageSize": page_size,
"departmentId": department_id,
"type": "1",
},
)
rows = _rows_overviews(body)
records = [rec for r in rows if (rec := _normalize_wa_group_row(r)) is not None]
total_fetched += len(records)
await _upsert_all(upsert_chat_group, records)
total_upserted += len(records)
logger.info(f" wa-groups page {page_num}: {len(rows)} rows, {len(records)} groups")
if len(rows) < page_size:
break
page_num += 1
await asyncio.sleep(1.5)
logger.info(f"DONE: wa-groups fetched={total_fetched}, upserted={total_upserted}")
return total_fetched, total_upserted
# ---------------------------------------------------------------------------
# Voice call records (navigation + interception, both WeChat + WA pages)
# ---------------------------------------------------------------------------
@@ -1299,6 +1377,7 @@ async def main():
"accounts", # 微信员工账号 (wechatlist page interception)
"wa-accounts", # WA员工账号 (waBusiness/userlist interception)
"wa-friends", # WA客户 (waBusiness/friendlist interception)
"wa-groups", # WA群 (browser fetch businessFriendList type=1)
"voice-calls", # 语音通话记录 (voice-call-list + wavoicecalllist)
"voice-clips", # 长语音 (browser fetch call/list per employee)
"wa-messages", # WA消息 (browser fetch chatRecords per conversation)
@@ -1337,6 +1416,8 @@ async def main():
await scrape_wa_accounts(cookie, args.max_pages)
elif args.entity == "wa-friends":
await scrape_wa_friends(cookie, args.max_pages)
elif args.entity == "wa-groups":
await scrape_wa_groups(cookie, args.max_pages)
elif args.entity == "voice-calls":
await scrape_voice_calls(cookie, args.max_pages)
elif args.entity == "voice-clips":