feat(scraper): WA群抓取(type=1真机验证,223群)+群成员派生(12840成员/750群)
- wa-groups 实体:businessFriendList type=1 真机验证返回群,行结构是好友字段名 (friendUid=@g.us 群ID/nickName=群名),专用 _normalize_wa_group_row - backfill_group_members.py:无独立成员API,从群消息派生(talker=成员,sender_name=昵称) - full_sync_all.sh:+4步(5b WA群/语音后录音重链接/8b 群消息+成员/消息上限99999) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -946,6 +946,84 @@ async def scrape_wa_friends(cookie_str: str, max_pages: int = 2000):
|
||||
return total_fetched, total_upserted
|
||||
|
||||
|
||||
def _normalize_wa_group_row(row: dict) -> dict | None:
|
||||
"""businessFriendList type=1 行 → ChatGroup 形状(真机确认 2026-08-20)。
|
||||
|
||||
type=1 返回的是好友字段名结构:friendUid=群 id(@g.us 后缀)、nickName=
|
||||
群名、uid=员工 WA 号、salesNickName=员工 WA 昵称——与微信群的
|
||||
friendWxId/friendWxNickname 完全不同,故不能复用 _normalize_chat_group。
|
||||
非 @g.us 行(异常混入)返回 None。
|
||||
"""
|
||||
friend_uid = str(row.get("friendUid") or "")
|
||||
if not friend_uid.endswith("@g.us"):
|
||||
return None
|
||||
return {
|
||||
"room_id": friend_uid,
|
||||
"channel": "whatsapp",
|
||||
"room_name": row.get("nickName"),
|
||||
"owner_account_id": row.get("uid"),
|
||||
"owner_account_nickname": row.get("salesNickName"),
|
||||
"owner_user_id": row.get("userId"),
|
||||
"owner_user_phone": row.get("salePhone"),
|
||||
"message_count": row.get("messageGroupCount"),
|
||||
"raw_json": row,
|
||||
}
|
||||
|
||||
|
||||
async def scrape_wa_groups(cookie_str: str, max_pages: int = 200):
|
||||
"""Scrape WhatsApp GROUPS via browser fetch POST businessFriendList type=1.
|
||||
|
||||
Same endpoint as WA friends with ``type:"1"`` — live-verified 2026-08-20:
|
||||
rows carry friendUid ending ``@g.us`` (group id), nickName = group name.
|
||||
Envelope: data.overviews; upsert to chat_group (channel='whatsapp').
|
||||
"""
|
||||
from app.db.base import async_engine
|
||||
from sqlalchemy import text as sa_text
|
||||
from app.storage.repos import upsert_chat_group
|
||||
|
||||
async with async_engine.connect() as conn:
|
||||
row = (await conn.execute(sa_text(
|
||||
"SELECT dept_id FROM department WHERE parent_id IS NULL LIMIT 1"
|
||||
))).first()
|
||||
department_id = row[0] if row else ""
|
||||
if not department_id:
|
||||
logger.warning("wa-groups: no root department in DB — abort")
|
||||
return 0, 0
|
||||
|
||||
page_size = 50
|
||||
total_fetched = 0
|
||||
total_upserted = 0
|
||||
async with PageScraper(cookie_str) as scraper:
|
||||
page = await _open_fetch_page(scraper)
|
||||
page_num = 1
|
||||
while page_num <= max_pages:
|
||||
body = await _browser_fetch(
|
||||
page,
|
||||
INTERCEPT_PATHS["wa_friend"],
|
||||
json_body={
|
||||
"searchInfo": "",
|
||||
"friendSearchInfo": "",
|
||||
"page": page_num,
|
||||
"pageSize": page_size,
|
||||
"departmentId": department_id,
|
||||
"type": "1",
|
||||
},
|
||||
)
|
||||
rows = _rows_overviews(body)
|
||||
records = [rec for r in rows if (rec := _normalize_wa_group_row(r)) is not None]
|
||||
total_fetched += len(records)
|
||||
await _upsert_all(upsert_chat_group, records)
|
||||
total_upserted += len(records)
|
||||
logger.info(f" wa-groups page {page_num}: {len(rows)} rows, {len(records)} groups")
|
||||
if len(rows) < page_size:
|
||||
break
|
||||
page_num += 1
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
logger.info(f"DONE: wa-groups fetched={total_fetched}, upserted={total_upserted}")
|
||||
return total_fetched, total_upserted
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Voice call records (navigation + interception, both WeChat + WA pages)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1299,6 +1377,7 @@ async def main():
|
||||
"accounts", # 微信员工账号 (wechatlist page interception)
|
||||
"wa-accounts", # WA员工账号 (waBusiness/userlist interception)
|
||||
"wa-friends", # WA客户 (waBusiness/friendlist interception)
|
||||
"wa-groups", # WA群 (browser fetch businessFriendList type=1)
|
||||
"voice-calls", # 语音通话记录 (voice-call-list + wavoicecalllist)
|
||||
"voice-clips", # 长语音 (browser fetch call/list per employee)
|
||||
"wa-messages", # WA消息 (browser fetch chatRecords per conversation)
|
||||
@@ -1337,6 +1416,8 @@ async def main():
|
||||
await scrape_wa_accounts(cookie, args.max_pages)
|
||||
elif args.entity == "wa-friends":
|
||||
await scrape_wa_friends(cookie, args.max_pages)
|
||||
elif args.entity == "wa-groups":
|
||||
await scrape_wa_groups(cookie, args.max_pages)
|
||||
elif args.entity == "voice-calls":
|
||||
await scrape_voice_calls(cookie, args.max_pages)
|
||||
elif args.entity == "voice-clips":
|
||||
|
||||
Reference in New Issue
Block a user