浅念
ab5a918a3b
- /api/calls 加 scope=friend|group(@chatroom后缀判群,与isRoom实测等价120=120),CallList去掉「全部」tab改服务端查 - 对话发现:全部customer pair ∪ 语音pair(weChatId,talker),未扫过的排最前——修复纯语音关系客户聊天记录永远空(35%语音客户pair从未扫过) - voice-calls改浏览器直发(isRoom=0/1双趟+WA callSource=4,拦截方式已两次失效) - 断连恢复重试30×2min(云客约3h一次断连,已实战扛过多次) - WA messages upsert改整列表直调(_upsert_all逐条传dict被迭代成key字符串→TypeError) - 405测试绿含scope新增3个 Co-Authored-By: Claude <noreply@anthropic.com>
1189 lines
51 KiB
Python
1189 lines
51 KiB
Python
#!/usr/bin/env python
|
||
"""页面浏览器抓取器 — 当 API 接口被风控时，用 Playwright 浏览器方式抓取数据。
|
||
|
||
原理：SPA 页面加载时会自己调用 friend_list/es 等接口渲染表格。
|
||
我们拦截页面的网络响应，获取原始 JSON —— 效果等同 API 调用，
|
||
但因为是浏览器页面上下文（非脚本直调），不触发风控。
|
||
|
||
用法：
|
||
cd yunke-server
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity friends
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity groups
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity messages
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity accounts
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity wa-accounts
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity wa-friends
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity voice-calls
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity voice-clips
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity wa-messages
|
||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity media
|
||
|
||
Cookie 从 .env 的 YK_COOKIE 读取。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import os
|
||
from urllib.parse import urlparse
|
||
|
||
logger = logging.getLogger("scrape_via_page")
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||
|
||
YUNKE_BASE = "https://crm2.yunkecn.com"
|
||
|
||
# API endpoints we intercept (the SPA calls these to render tables)
|
||
INTERCEPT_PATHS = {
|
||
"friend_list": "/wechatweb/wxFriends/friend_list/es", # friends + groups (friendTypeYunke)
|
||
"wxUser_list": "/wechatweb/wxUser/list", # wechat accounts
|
||
"chatroom": "/wechatweb/chatroom/getChatroomList", # legacy groups
|
||
"voice_list": "/wechatweb/call/list", # voice clips
|
||
"call_list": "/wechatweb/callRecord/queryWeChatVoiceList", # voice calls
|
||
"wa_friend": "/qqweb/WhatsAppUser/businessFriendList", # WA friends + groups
|
||
"wa_user": "/qqweb/WhatsAppUser/businessSalesList", # WA accounts
|
||
"msg_chats": "/wechatweb/msg/chats", # chat messages
|
||
}
|
||
|
||
|
||
def parse_cookies(cookie_str: str, domain: str = "crm2.yunkecn.com") -> list[dict]:
|
||
"""Parse a cookie string like 'user=xxx; userToken=yyy' into Playwright cookie dicts."""
|
||
cookies = []
|
||
for pair in cookie_str.split(";"):
|
||
pair = pair.strip()
|
||
if "=" not in pair:
|
||
continue
|
||
name, value = pair.split("=", 1)
|
||
cookies.append({
|
||
"name": name.strip(),
|
||
"value": value.strip(),
|
||
"domain": domain,
|
||
"path": "/",
|
||
})
|
||
return cookies
|
||
|
||
|
||
class PageScraper:
|
||
"""Navigates 云客 pages in a browser, intercepts API responses, extracts JSON."""
|
||
|
||
def __init__(self, cookie_str: str, headless: bool = True):
|
||
self.cookie_str = cookie_str
|
||
self.headless = headless
|
||
self.browser = None
|
||
self.context = None
|
||
self.page = None
|
||
self._captured: list[dict] = []
|
||
self._capture_path: str | None = None
|
||
|
||
async def __aenter__(self):
|
||
from playwright.async_api import async_playwright
|
||
self._pw = await async_playwright().start()
|
||
self.browser = await self._pw.chromium.launch(headless=self.headless)
|
||
self.context = await self.browser.new_context(
|
||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
|
||
viewport={"width": 1920, "height": 1080},
|
||
)
|
||
# Set cookies
|
||
cookies = parse_cookies(self.cookie_str)
|
||
await self.context.add_cookies(cookies)
|
||
self.page = await self.context.new_page()
|
||
|
||
# Intercept API responses
|
||
self.page.on("response", self._on_response)
|
||
return self
|
||
|
||
async def __aexit__(self, *args):
|
||
if self.browser:
|
||
await self.browser.close()
|
||
if self._pw:
|
||
await self._pw.stop()
|
||
|
||
async def _on_response(self, response):
|
||
"""Capture API JSON responses that match our intercept paths."""
|
||
url = response.url
|
||
path = urlparse(url).path
|
||
if self._capture_path and path == self._capture_path:
|
||
try:
|
||
body = await response.json()
|
||
self._captured.append({"url": url, "status": response.status, "body": body})
|
||
logger.info(f" captured {path}: status={response.status}, keys={list(body.keys()) if isinstance(body, dict) else 'N/A'}")
|
||
except Exception as e:
|
||
logger.warning(f" capture failed for {path}: {e}")
|
||
|
||
def start_capture(self, api_path: str):
|
||
"""Start capturing responses for a specific API path."""
|
||
self._capture_path = api_path
|
||
self._captured.clear()
|
||
|
||
def get_captured(self) -> list[dict]:
|
||
"""Return captured responses and stop capturing."""
|
||
self._capture_path = None
|
||
return self._captured
|
||
|
||
async def navigate_and_wait(self, url: str, wait_selector: str = ".ant-table-row", timeout: int = 15000):
|
||
"""Navigate to a URL and wait for table rows to appear."""
|
||
await self.page.goto(url, wait_until="domcontentloaded", timeout=timeout)
|
||
try:
|
||
await self.page.wait_for_selector(wait_selector, timeout=timeout)
|
||
except Exception:
|
||
pass # might be empty page
|
||
await self.page.wait_for_timeout(2000) # extra settle time
|
||
|
||
async def get_total_pages(self) -> int:
|
||
"""Extract total page count from Ant Design pagination."""
|
||
try:
|
||
total_text = await self.page.evaluate("""() => {
|
||
const el = document.querySelector('.ant-pagination-total-text');
|
||
return el ? el.textContent : '';
|
||
}""")
|
||
if total_text:
|
||
import re
|
||
m = re.search(r'(\d+)', total_text)
|
||
if m:
|
||
total = int(m.group(1))
|
||
page_size = 10 # 云客 default
|
||
return (total + page_size - 1) // page_size
|
||
except Exception:
|
||
pass
|
||
return 1
|
||
|
||
async def click_next_page(self) -> bool:
|
||
"""Click the 'next page' button. Returns True if successful."""
|
||
try:
|
||
next_btn = self.page.locator('.ant-pagination-next:not(.ant-pagination-disabled)')
|
||
if await next_btn.count() > 0:
|
||
await next_btn.click()
|
||
await self.page.wait_for_timeout(2000) # wait for data load
|
||
return True
|
||
except Exception:
|
||
pass
|
||
return False
|
||
|
||
async def scrape_entity_pages(
|
||
self,
|
||
list_url: str,
|
||
api_path: str,
|
||
max_pages: int = 100,
|
||
) -> list[dict]:
|
||
"""Navigate to a list page, paginate through all pages, capturing API responses.
|
||
|
||
Returns a list of captured JSON response bodies (one per page).
|
||
"""
|
||
self.start_capture(api_path)
|
||
await self.navigate_and_wait(list_url)
|
||
await self.page.wait_for_timeout(1000) # let API call complete
|
||
|
||
all_responses = list(self.get_captured())
|
||
|
||
total_pages = await self.get_total_pages()
|
||
logger.info(f" total pages: {min(total_pages, max_pages)}")
|
||
|
||
page_num = 1
|
||
while page_num < min(total_pages, max_pages):
|
||
self.start_capture(api_path)
|
||
clicked = await self.click_next_page()
|
||
if not clicked:
|
||
break
|
||
await self.page.wait_for_timeout(1500)
|
||
all_responses.extend(self.get_captured())
|
||
page_num += 1
|
||
if page_num % 10 == 0:
|
||
logger.info(f" page {page_num}/{min(total_pages, max_pages)}")
|
||
|
||
logger.info(f" captured {len(all_responses)} API responses across {page_num} pages")
|
||
return all_responses
|
||
|
||
async def get_employee_links(self) -> list[dict]:
|
||
"""Extract employee wxIds + friend-count links from the staff list page."""
|
||
await self.navigate_and_wait(f"{YUNKE_BASE}/cms/wechat/wechatlist")
|
||
|
||
employees = await self.page.evaluate("""() => {
|
||
const rows = document.querySelectorAll('.ant-table-row');
|
||
const result = [];
|
||
for (const tr of rows) {
|
||
// Find the friend-count link (href contains wechatdetail/1/)
|
||
const friendLink = tr.querySelector('a[href*="wechatdetail/1/"]');
|
||
const groupLink = tr.querySelector('a[href*="wechatdetail/2/"]');
|
||
if (friendLink) {
|
||
const href = friendLink.getAttribute('href');
|
||
const wxId = href.match(/wechatdetail\\/1\\/([^?]+)/)?.[1];
|
||
const friendCount = parseInt(friendLink.textContent) || 0;
|
||
const groupHref = groupLink?.getAttribute('href');
|
||
const groupCount = parseInt(groupLink?.textContent) || 0;
|
||
result.push({ wxId, friendCount, groupCount, friendHref: href, groupHref });
|
||
}
|
||
}
|
||
return result;
|
||
}""")
|
||
|
||
# Paginate through ALL staff list pages
|
||
all_employees = list(employees)
|
||
page = 1
|
||
while await self.click_next_page():
|
||
page += 1
|
||
await self.page.wait_for_timeout(2000)
|
||
more = await self.page.evaluate("""() => {
|
||
const rows = document.querySelectorAll('.ant-table-row');
|
||
const result = [];
|
||
for (const tr of rows) {
|
||
const friendLink = tr.querySelector('a[href*="wechatdetail/1/"]');
|
||
const groupLink = tr.querySelector('a[href*="wechatdetail/2/"]');
|
||
if (friendLink) {
|
||
const href = friendLink.getAttribute('href');
|
||
const wxId = href.match(/wechatdetail\\/1\\/([^?]+)/)?.[1];
|
||
const friendCount = parseInt(friendLink.textContent) || 0;
|
||
const groupHref = groupLink?.getAttribute('href');
|
||
const groupCount = parseInt(groupLink?.textContent) || 0;
|
||
result.push({ wxId, friendCount, groupCount, friendHref: href, groupHref });
|
||
}
|
||
}
|
||
return result;
|
||
}""")
|
||
all_employees.extend(more)
|
||
logger.info(f" staff page {page}: +{len(more)} employees (total: {len(all_employees)})")
|
||
|
||
# Dedup by wxId
|
||
seen = set()
|
||
deduped = []
|
||
for emp in all_employees:
|
||
if emp["wxId"] not in seen:
|
||
seen.add(emp["wxId"])
|
||
deduped.append(emp)
|
||
|
||
logger.info(f"Found {len(deduped)} unique employees with friend links ({page} pages)")
|
||
return deduped
|
||
|
||
|
||
async def scrape_friends(cookie_str: str, max_employees: int = 100):
|
||
"""Scrape wechat customers (friends) via page navigation."""
|
||
from app.db.base import async_session_maker
|
||
from app.storage.repos import upsert_customer
|
||
from app.scraper.wechat import _normalize # reuse the existing normalizer
|
||
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
|
||
async with PageScraper(cookie_str) as scraper:
|
||
employees = await scraper.get_employee_links()
|
||
|
||
for i, emp in enumerate(employees[:max_employees]):
|
||
if emp["friendCount"] == 0:
|
||
continue
|
||
wx_id = emp["wxId"]
|
||
url = f"{YUNKE_BASE}/cms/wechat/wechatlist/wechatdetail/1/{wx_id}?ymdStart=0000-00-00&ymdEnd=2099-12-31"
|
||
logger.info(f"[{i+1}/{len(employees)}] {wx_id}: {emp['friendCount']} friends")
|
||
|
||
responses = await scraper.scrape_entity_pages(
|
||
list_url=url,
|
||
api_path=INTERCEPT_PATHS["friend_list"],
|
||
max_pages=200,
|
||
)
|
||
|
||
# Extract friend records from captured responses
|
||
records = []
|
||
for resp in responses:
|
||
body = resp.get("body", {})
|
||
data = body.get("data", {})
|
||
rows = data.get("basicResult", {}).get("data", []) or data.get("data", [])
|
||
for row in rows:
|
||
if isinstance(row, dict):
|
||
normalized = _normalize(row)
|
||
normalized["channel"] = "wechat"
|
||
normalized["owner_account_id"] = wx_id
|
||
records.append(normalized)
|
||
|
||
total_fetched += len(records)
|
||
|
||
# Batch upsert
|
||
if records:
|
||
async with async_session_maker() as session:
|
||
for rec in records:
|
||
await upsert_customer(session, rec)
|
||
await session.commit()
|
||
total_upserted += len(records)
|
||
logger.info(f" upserted {len(records)} customers (total: {total_upserted})")
|
||
|
||
# Human-like delay between employees
|
||
await asyncio.sleep(3 + (i % 3)) # 3-5s random-ish
|
||
|
||
logger.info(f"DONE: friends fetched={total_fetched}, upserted={total_upserted}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
async def scrape_groups(cookie_str: str, max_employees: int = 100):
|
||
"""Scrape wechat groups via page navigation."""
|
||
from app.db.base import async_session_maker
|
||
from app.storage.repos import upsert_chat_group
|
||
from app.scraper.wechat import _normalize_chat_group
|
||
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
|
||
async with PageScraper(cookie_str) as scraper:
|
||
employees = await scraper.get_employee_links()
|
||
|
||
for i, emp in enumerate(employees[:max_employees]):
|
||
if emp.get("groupCount", 0) == 0 or not emp.get("groupHref"):
|
||
continue
|
||
wx_id = emp["wxId"]
|
||
url = f"{YUNKE_BASE}{emp['groupHref']}"
|
||
logger.info(f"[{i+1}/{len(employees)}] {wx_id}: {emp['groupCount']} groups")
|
||
|
||
responses = await scraper.scrape_entity_pages(
|
||
list_url=url,
|
||
api_path=INTERCEPT_PATHS["friend_list"],
|
||
max_pages=100,
|
||
)
|
||
|
||
records = []
|
||
for resp in responses:
|
||
body = resp.get("body", {})
|
||
data = body.get("data", {})
|
||
rows = data.get("basicResult", {}).get("data", []) or data.get("data", [])
|
||
for row in rows:
|
||
if isinstance(row, dict):
|
||
normalized = _normalize_chat_group(row)
|
||
normalized["channel"] = "wechat"
|
||
records.append(normalized)
|
||
|
||
total_fetched += len(records)
|
||
if records:
|
||
async with async_session_maker() as session:
|
||
for rec in records:
|
||
await upsert_chat_group(session, rec)
|
||
await session.commit()
|
||
total_upserted += len(records)
|
||
logger.info(f" upserted {len(records)} groups (total: {total_upserted})")
|
||
|
||
await asyncio.sleep(3 + (i % 3))
|
||
|
||
logger.info(f"DONE: groups fetched={total_fetched}, upserted={total_upserted}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
async def paginate_chats(fetch_page, end_ms: int, max_pages: int = 200, sleep=None):
|
||
"""按 hasLast/end 游标翻页 chats 类接口（微信 msg/chats 与 WA chatRecords 同协议）。
|
||
|
||
fetch_page(end_ms) -> 响应 dict；逐页 yield data 部分（dict）。
|
||
终止：data 非 dict / hasLast 假 / end 非数值 / 达到 max_pages 页上限（防死循环）。
|
||
"""
|
||
if sleep is None:
|
||
sleep = asyncio.sleep
|
||
pages = 0
|
||
while True:
|
||
body = await fetch_page(end_ms)
|
||
pages += 1
|
||
data = body.get("data") if isinstance(body, dict) else None
|
||
if not isinstance(data, dict):
|
||
return
|
||
yield data
|
||
if not data.get("hasLast"):
|
||
return
|
||
end = data.get("end")
|
||
if not isinstance(end, (int, float)):
|
||
return
|
||
end_ms = int(end) - 1
|
||
if pages >= max_pages:
|
||
logger.warning(f"paginate_chats: page cap {max_pages} hit, stop paging")
|
||
return
|
||
await sleep(1)
|
||
|
||
|
||
async def _discover_conversations(channel: str, voice_channel: str) -> list[tuple[str, str]]:
|
||
"""Conversation-pair discovery for the message scrapers (wechat + WA).
|
||
|
||
Sources (union, dedup):
|
||
* customer table — ALL pairs. The old ``last_chat_time IS NOT NULL`` filter
|
||
skipped customers whose relationship is voice-call-only (NULL last_chat)
|
||
and thus their chat history was never scraped → empty chat pages
|
||
(diagnosed 2026-08-18).
|
||
* voice-call pairs ``(weChatId, talker)`` from phone_call raw_json — peers
|
||
that had calls but have NO customer row at all (deleted friends).
|
||
|
||
Order: pairs with no messages yet FIRST (new coverage survives an
|
||
interruption), then the rest by customer last_chat_time DESC.
|
||
"""
|
||
import json as _json
|
||
|
||
from app.db.base import async_engine
|
||
from sqlalchemy import text as sa_text
|
||
|
||
async with async_engine.connect() as conn:
|
||
cust = (
|
||
await conn.execute(sa_text(
|
||
"SELECT owner_account_id, friend_id FROM customer "
|
||
"WHERE channel=:ch AND owner_account_id IS NOT NULL AND friend_id IS NOT NULL "
|
||
"ORDER BY last_chat_time DESC"
|
||
), {"ch": channel})
|
||
).fetchall()
|
||
voice = (
|
||
await conn.execute(sa_text(
|
||
"SELECT raw_json FROM phone_call WHERE channel=:vc"
|
||
), {"vc": voice_channel})
|
||
).fetchall()
|
||
with_msgs = (
|
||
await conn.execute(sa_text(
|
||
"SELECT DISTINCT owner_account_id, talker_id FROM message "
|
||
"WHERE channel=:ch AND (room_id IS NULL OR room_id='')"
|
||
), {"ch": channel})
|
||
).fetchall()
|
||
|
||
have = {(r[0], r[1]) for r in with_msgs}
|
||
seen: set[tuple[str, str]] = set()
|
||
fresh: list[tuple[str, str]] = []
|
||
known: list[tuple[str, str]] = []
|
||
|
||
def _add(key: tuple[str, str]) -> None:
|
||
if key in seen:
|
||
return
|
||
seen.add(key)
|
||
(fresh if key not in have else known).append(key)
|
||
|
||
for owner, friend in cust:
|
||
_add((owner, friend))
|
||
voice_only = 0
|
||
for (rj,) in voice:
|
||
try:
|
||
d = _json.loads(rj)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
owner, talker = d.get("weChatId"), d.get("talker")
|
||
if not owner or not talker or str(talker).endswith("@chatroom"):
|
||
continue
|
||
before = len(seen)
|
||
_add((str(owner), str(talker)))
|
||
if len(seen) > before and (str(owner), str(talker)) not in cust:
|
||
voice_only += 1
|
||
|
||
logger.info(
|
||
f"discovery[{channel}]: {len(cust)} customer pairs + {voice_only} voice-only pairs, "
|
||
f"{len(fresh)} unscanned (first) / {len(known)} already have messages"
|
||
)
|
||
return fresh + known
|
||
|
||
|
||
async def scrape_messages(cookie_str: str, max_conversations: int = 100):
|
||
"""Scrape wechat messages via browser fetch POST /wechatweb/msg/chats.
|
||
|
||
Conversation pairs come from _discover_conversations (customer table ∪
|
||
voice-call pairs). Pages backward through full history via data.end /
|
||
data.hasLast (paginate_chats, same cursor strategy as the WA chatRecords
|
||
scraper).
|
||
"""
|
||
from app.db.base import async_session_maker
|
||
from app.storage.repos import upsert_messages
|
||
|
||
conversations = await _discover_conversations("wechat", "voice_call")
|
||
|
||
logger.info(f"Discovered {len(conversations)} conversations, will scrape {min(max_conversations, len(conversations))}")
|
||
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
total_pages = 0
|
||
|
||
async with PageScraper(cookie_str) as scraper:
|
||
# Just load the base URL to establish cookie context — no need for specific page
|
||
# The fetch() calls work from any page (even login redirect) as long as cookies are set
|
||
fetch_page = await scraper.context.new_page()
|
||
await fetch_page.goto(f"{YUNKE_BASE}/", wait_until="domcontentloaded", timeout=15000)
|
||
await fetch_page.wait_for_timeout(2000)
|
||
|
||
page = fetch_page # use the stable page for all evaluate() calls
|
||
|
||
for i, (owner_id, friend_id) in enumerate(conversations[:max_conversations]):
|
||
records = []
|
||
pages = 0
|
||
try:
|
||
# Use browser fetch() to call the message API from the page context
|
||
# (the browser sends it with proper cookies/headers → no 风控)
|
||
# NOTE: msg/chats uses FORM-ENCODED + cursor params (start/end, not pageNum)
|
||
import time as _time
|
||
now_ms = int(_time.time() * 1000)
|
||
|
||
async def _fetch(end_ms: int, _owner=owner_id, _friend=friend_id):
|
||
return await page.evaluate("""async ([ownerId, friendId, endMs]) => {
|
||
const params = new URLSearchParams();
|
||
params.append('wechatId', ownerId);
|
||
params.append('friendId', friendId);
|
||
params.append('types', '');
|
||
params.append('start', '0');
|
||
params.append('end', String(endMs));
|
||
const resp = await fetch('/wechatweb/msg/chats', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: params.toString(),
|
||
});
|
||
return await resp.json();
|
||
}""", [_owner, _friend, end_ms])
|
||
|
||
# 翻页取完整历史：hasLast 真 → 用 data.end-1 继续向前翻（页间 sleep 1s）
|
||
async for data in paginate_chats(_fetch, now_ms):
|
||
pages += 1
|
||
# Debug: log first response structure
|
||
if i == 0 and pages == 1:
|
||
logger.info(f" DEBUG first response keys: {list(data.keys())}")
|
||
msgs0 = data.get("messages")
|
||
logger.info(f" DEBUG messages type: {type(msgs0)}, len: {len(msgs0) if isinstance(msgs0, list) else 'N/A'}")
|
||
rows = data.get("messages") or []
|
||
for row in rows:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
# 跳过 type=-2 日期分隔线（非真实消息，仅显示日期）
|
||
if str(row.get("type")) == "-2":
|
||
continue
|
||
# Normalize to Message shape
|
||
ts_raw = row.get("createTime") or row.get("timestamp")
|
||
try:
|
||
ts = int(str(ts_raw)) if ts_raw else None
|
||
except (ValueError, TypeError):
|
||
ts = None
|
||
records.append({
|
||
"msg_id": str(row.get("msgId") or row.get("msgSvrId") or row.get("id") or f"{owner_id}:{friend_id}:{i}:{len(records)}"),
|
||
"channel": "wechat",
|
||
"owner_account_id": owner_id,
|
||
# 用发送的 friend_id（与客户表一致）优先，API 返回的 talker 可能脱敏
|
||
"talker_id": friend_id or row.get("talker") or row.get("oriTalker"),
|
||
"room_id": None,
|
||
"is_send": bool(row.get("isSend", 0)) if row.get("isSend") is not None else None,
|
||
"type": _normalize_msg_type(row.get("type")),
|
||
"type_raw": row.get("type"),
|
||
"text": row.get("content") or row.get("text") or "",
|
||
"file_url": row.get("file") or row.get("fileUrl"),
|
||
"timestamp": ts,
|
||
"sender_name": row.get("name") or row.get("senderName"),
|
||
"raw_json": row,
|
||
})
|
||
|
||
except Exception as e:
|
||
logger.warning(f" [{i+1}] {owner_id[:12]}:{friend_id[:12]} failed: {e}")
|
||
# Recreate the fetch page if context was destroyed.
|
||
# A network blip / yunke outage also breaks this recovery goto —
|
||
# retry patiently (30 × 2min = 1h) so a longer outage doesn't
|
||
# kill the whole run (crashed twice on 2026-08-18 this way,
|
||
# outages ~3h apart, ~1.5h into each run).
|
||
for attempt in range(30):
|
||
try:
|
||
try:
|
||
fetch_page.close()
|
||
except Exception:
|
||
pass
|
||
fetch_page = await scraper.context.new_page()
|
||
await fetch_page.goto(f"{YUNKE_BASE}/cms/home/statistics", wait_until="networkidle", timeout=20000)
|
||
await fetch_page.wait_for_timeout(1000)
|
||
page = fetch_page
|
||
break
|
||
except Exception as rec:
|
||
logger.warning(f" recovery {attempt + 1}/30 failed: {rec}")
|
||
if attempt == 29:
|
||
raise
|
||
await asyncio.sleep(120)
|
||
continue
|
||
|
||
total_fetched += len(records)
|
||
total_pages += pages
|
||
|
||
if records:
|
||
async with async_session_maker() as session:
|
||
await upsert_messages(session, records)
|
||
await session.commit()
|
||
total_upserted += len(records)
|
||
|
||
if (i + 1) % 10 == 0:
|
||
logger.info(f" [{i+1}/{min(max_conversations, len(conversations))}] msgs total: {total_fetched}, pages: {total_pages}")
|
||
|
||
# Human-like delay
|
||
await asyncio.sleep(2 + (i % 3))
|
||
|
||
logger.info(f"DONE: messages fetched={total_fetched}, upserted={total_upserted}, pages={total_pages}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
def _normalize_msg_type(raw_type) -> str | None:
|
||
"""Normalize a raw message type to our canonical type string."""
|
||
if raw_type is None:
|
||
return None
|
||
t = str(raw_type)
|
||
mapping = {"1": "text", "3": "image_legacy", "34": "voice", "43": "video", "47": "emoji", "49": "file"}
|
||
return mapping.get(t, "unknown")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Shared helpers for the fetch()-based entities
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def _open_fetch_page(scraper: PageScraper):
|
||
"""Open a stable page in the scraper's browser context for fetch() calls.
|
||
|
||
The fetch() calls work from any same-origin page (even a login redirect)
|
||
as long as the context carries the session cookies — same trick the
|
||
message scraper uses.
|
||
"""
|
||
page = await scraper.context.new_page()
|
||
await page.goto(f"{YUNKE_BASE}/", wait_until="domcontentloaded", timeout=15000)
|
||
await page.wait_for_timeout(2000)
|
||
return page
|
||
|
||
|
||
async def _browser_fetch(page, path: str, json_body: dict | None = None, form_body: dict | None = None):
|
||
"""POST from inside the browser context (proper cookies/referrer => no 风控).
|
||
|
||
``json_body`` sends a JSON body; ``form_body`` sends form-urlencoded.
|
||
Returns the parsed JSON response.
|
||
"""
|
||
if json_body is not None:
|
||
return await page.evaluate(
|
||
"""async ([path, body]) => {
|
||
const resp = await fetch(path, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify(body),
|
||
});
|
||
return await resp.json();
|
||
}""",
|
||
[path, json_body],
|
||
)
|
||
return await page.evaluate(
|
||
"""async ([path, params]) => {
|
||
const ps = new URLSearchParams();
|
||
for (const [k, v] of Object.entries(params)) ps.append(k, String(v));
|
||
const resp = await fetch(path, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: ps.toString(),
|
||
});
|
||
return await resp.json();
|
||
}""",
|
||
[path, form_body or {}],
|
||
)
|
||
|
||
|
||
def _rows_flat(body: dict) -> list[dict]:
|
||
"""Extract rows from the flat envelope: ``data.data[]`` (wxUser/list,
|
||
queryWeChatVoiceList, call/list)."""
|
||
data = body.get("data") if isinstance(body, dict) else None
|
||
if not isinstance(data, dict):
|
||
return []
|
||
rows = data.get("data")
|
||
return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []
|
||
|
||
|
||
def _rows_overviews(body: dict) -> list[dict]:
|
||
"""Extract rows from the WA envelope: ``data.overviews[]``
|
||
(businessSalesList / businessFriendList)."""
|
||
data = body.get("data") if isinstance(body, dict) else None
|
||
if not isinstance(data, dict):
|
||
return []
|
||
rows = data.get("overviews")
|
||
return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []
|
||
|
||
|
||
async def _upsert_all(upsert_fn, records: list[dict]) -> None:
|
||
"""Batch-upsert records (each fn per record) in one committed session."""
|
||
from app.db.base import async_session_maker
|
||
|
||
if not records:
|
||
return
|
||
async with async_session_maker() as session:
|
||
for rec in records:
|
||
await upsert_fn(session, rec)
|
||
await session.commit()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# WeChat staff accounts (navigation + interception)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_accounts(cookie_str: str, max_pages: int = 100):
|
||
"""Scrape WeChat staff accounts: navigate /cms/wechat/wechatlist,
|
||
intercept /wechatweb/wxUser/list, upsert to wechat_account (+staff)."""
|
||
from app.scraper.wechat import _normalize_wxuser_account
|
||
from app.storage.repos import upsert_wechat_account
|
||
|
||
total_fetched = 0
|
||
async with PageScraper(cookie_str) as scraper:
|
||
responses = await scraper.scrape_entity_pages(
|
||
list_url=f"{YUNKE_BASE}/cms/wechat/wechatlist",
|
||
api_path=INTERCEPT_PATHS["wxUser_list"],
|
||
max_pages=max_pages,
|
||
)
|
||
records = []
|
||
for resp in responses:
|
||
for row in _rows_flat(resp.get("body", {})):
|
||
records.append(_normalize_wxuser_account(row))
|
||
total_fetched = len(records)
|
||
await _upsert_all(upsert_wechat_account, records)
|
||
|
||
logger.info(f"DONE: accounts fetched={total_fetched}, upserted={total_fetched}")
|
||
return total_fetched, total_fetched
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# WhatsApp staff accounts (navigation + interception)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_wa_accounts(cookie_str: str, max_pages: int = 100):
|
||
"""Scrape WA staff accounts: navigate /cms/wechat/waBusiness/userlist,
|
||
intercept /qqweb/WhatsAppUser/businessSalesList, upsert to whatsapp_account."""
|
||
from app.scraper.whatsapp import _normalize_whatsapp_account
|
||
from app.storage.repos import upsert_whatsapp_account
|
||
|
||
async with PageScraper(cookie_str) as scraper:
|
||
responses = await scraper.scrape_entity_pages(
|
||
list_url=f"{YUNKE_BASE}/cms/wechat/waBusiness/userlist",
|
||
api_path=INTERCEPT_PATHS["wa_user"],
|
||
max_pages=max_pages,
|
||
)
|
||
records = []
|
||
for resp in responses:
|
||
for row in _rows_overviews(resp.get("body", {})):
|
||
records.append(_normalize_whatsapp_account(row))
|
||
total = len(records)
|
||
await _upsert_all(upsert_whatsapp_account, records)
|
||
|
||
logger.info(f"DONE: wa-accounts fetched={total}, upserted={total}")
|
||
return total, total
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# WhatsApp customers (navigation + interception)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_wa_friends(cookie_str: str, max_pages: int = 2000):
|
||
"""Scrape WA customers: navigate /cms/wechat/waBusiness/friendlist,
|
||
intercept /qqweb/WhatsAppUser/businessFriendList, upsert to customer
|
||
(channel='whatsapp'). Live total ~11k rows => high default max_pages."""
|
||
from app.scraper.whatsapp import _normalize_whatsapp_friend
|
||
from app.storage.repos import upsert_customer
|
||
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
async with PageScraper(cookie_str) as scraper:
|
||
responses = await scraper.scrape_entity_pages(
|
||
list_url=f"{YUNKE_BASE}/cms/wechat/waBusiness/friendlist",
|
||
api_path=INTERCEPT_PATHS["wa_friend"],
|
||
max_pages=max_pages,
|
||
)
|
||
# Upsert in chunks so one giant page-walk doesn't build a 11k-record list
|
||
# before the first commit.
|
||
chunk: list[dict] = []
|
||
for resp in responses:
|
||
records = [_normalize_whatsapp_friend(row) for row in _rows_overviews(resp.get("body", {}))]
|
||
total_fetched += len(records)
|
||
chunk.extend(records)
|
||
if len(chunk) >= 500:
|
||
await _upsert_all(upsert_customer, chunk)
|
||
total_upserted += len(chunk)
|
||
chunk = []
|
||
await _upsert_all(upsert_customer, chunk)
|
||
total_upserted += len(chunk)
|
||
|
||
logger.info(f"DONE: wa-friends fetched={total_fetched}, upserted={total_upserted}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Voice call records (navigation + interception, both WeChat + WA pages)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_voice_calls(cookie_str: str, max_pages: int = 200):
|
||
"""Scrape voice call records from BOTH channels, upsert to phone_call.
|
||
|
||
* WeChat: browser-fetch queryWeChatVoiceList (callSource=0) with TWO
|
||
passes isRoom=0(好友)/1(群). The voice-call-list page only sends
|
||
isRoom=0 on load — group voice calls live behind the 微信群 tab, so
|
||
pure navigation+interception never captured them (root cause of the
|
||
empty 群 scope tab). Body must include callSource+departmentId+isRoom+
|
||
startTime+endTime or the API returns 200 with empty data.
|
||
* WA: browser-fetch the same unified endpoint with callSource=4
|
||
(no group concept; interception stopped capturing on 2026-08-18).
|
||
"""
|
||
from app.db.base import async_engine
|
||
from sqlalchemy import text as sa_text
|
||
from app.scraper.wechat import _normalize_voice_call
|
||
from app.scraper.whatsapp import _normalize_wa_voice_call
|
||
from app.storage.repos import upsert_voice_call
|
||
|
||
# queryWeChatVoiceList requires departmentId — take the company-root dept
|
||
async with async_engine.connect() as conn:
|
||
row = (await conn.execute(sa_text(
|
||
"SELECT dept_id FROM department WHERE parent_id IS NULL LIMIT 1"
|
||
))).first()
|
||
department_id = row[0] if row else ""
|
||
if not department_id:
|
||
logger.warning("voice-calls: no root department in DB — skipping wechat fetch passes")
|
||
|
||
page_size = 100
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
async with PageScraper(cookie_str) as scraper:
|
||
# --- WeChat: browser fetch, isRoom 0(好友)/1(群) two passes ---
|
||
if department_id:
|
||
page = await _open_fetch_page(scraper)
|
||
for is_room, label in (("0", "好友"), ("1", "群")):
|
||
records = []
|
||
page_num = 1
|
||
while page_num <= max_pages:
|
||
body = await _browser_fetch(
|
||
page,
|
||
INTERCEPT_PATHS["call_list"],
|
||
json_body={
|
||
"pageNum": page_num,
|
||
"pageSize": page_size,
|
||
"callSource": "0",
|
||
"isRoom": is_room,
|
||
"startTime": "2000-01-01 00:00:00",
|
||
"endTime": "2099-12-31 23:59:59",
|
||
"sort": "",
|
||
"ids": [],
|
||
"userId": "",
|
||
"departmentId": department_id,
|
||
},
|
||
)
|
||
rows = _rows_flat(body)
|
||
for row in rows:
|
||
rec = _normalize_voice_call(row)
|
||
if rec.get("call_id") is not None:
|
||
records.append(rec)
|
||
if len(rows) < page_size:
|
||
break
|
||
page_num += 1
|
||
await asyncio.sleep(1)
|
||
total_fetched += len(records)
|
||
await _upsert_all(upsert_voice_call, records)
|
||
total_upserted += len(records)
|
||
logger.info(f" wechat isRoom={is_room}({label}): {len(records)} calls, {page_num} page(s)")
|
||
await asyncio.sleep(2)
|
||
|
||
# --- WA: browser fetch (callSource=4, no group concept; same unified endpoint) ---
|
||
# Interception on the wavoicecalllist page used to work but captured 0
|
||
# responses on 2026-08-18 (page no longer fires the API on load) — direct
|
||
# browser fetch with the full body is the reliable path, same as WeChat.
|
||
if department_id:
|
||
records = []
|
||
page_num = 1
|
||
while page_num <= max_pages:
|
||
body = await _browser_fetch(
|
||
page,
|
||
INTERCEPT_PATHS["call_list"],
|
||
json_body={
|
||
"pageNum": page_num,
|
||
"pageSize": page_size,
|
||
"callSource": "4",
|
||
"isRoom": "0",
|
||
"startTime": "2000-01-01 00:00:00",
|
||
"endTime": "2099-12-31 23:59:59",
|
||
"sort": "",
|
||
"ids": [],
|
||
"userId": "",
|
||
"departmentId": department_id,
|
||
},
|
||
)
|
||
rows = _rows_flat(body)
|
||
for row in rows:
|
||
rec = _normalize_wa_voice_call(row)
|
||
if rec.get("call_id") is not None:
|
||
records.append(rec)
|
||
if len(rows) < page_size:
|
||
break
|
||
page_num += 1
|
||
await asyncio.sleep(1)
|
||
total_fetched += len(records)
|
||
await _upsert_all(upsert_voice_call, records)
|
||
total_upserted += len(records)
|
||
logger.info(f" wa(callSource=4): {len(records)} calls, {page_num} page(s)")
|
||
|
||
logger.info(f"DONE: voice-calls fetched={total_fetched}, upserted={total_upserted}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Long voice clips (browser fetch, per employee)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_voice_clips(cookie_str: str, max_employees: int = 200, page_size: int = 50):
|
||
"""Scrape long voice clips via browser fetch POST /wechatweb/call/list.
|
||
|
||
The endpoint is per-employee: employee user_ids come from the LOCAL
|
||
wechat_account / whatsapp_account tables (channel resolved the same way).
|
||
JSON body {pageNum, pageSize, userId} (真机确认 2026-08-07 spec §2).
|
||
"""
|
||
from app.db.base import async_engine
|
||
from app.scraper.voice_clip import _normalize_voice_clip
|
||
from app.storage.repos import upsert_voice_clip
|
||
from sqlalchemy import text as sa_text
|
||
|
||
# Resolve employee user_id -> channel from local staff-account tables.
|
||
# WeChat wins when an id appears in both tables.
|
||
user_channel: dict[str, str] = {}
|
||
async with async_engine.connect() as conn:
|
||
result = await conn.execute(sa_text(
|
||
"SELECT DISTINCT user_id FROM whatsapp_account "
|
||
"WHERE user_id IS NOT NULL AND user_id != ''"
|
||
))
|
||
for (uid,) in result.fetchall():
|
||
user_channel[uid] = "whatsapp"
|
||
result = await conn.execute(sa_text(
|
||
"SELECT DISTINCT user_id FROM wechat_account "
|
||
"WHERE user_id IS NOT NULL AND user_id != ''"
|
||
))
|
||
for (uid,) in result.fetchall():
|
||
user_channel[uid] = "wechat"
|
||
|
||
employees = sorted(user_channel.items())
|
||
logger.info(f"Found {len(employees)} employees; scraping {min(max_employees, len(employees))}")
|
||
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
|
||
async with PageScraper(cookie_str) as scraper:
|
||
page = await _open_fetch_page(scraper)
|
||
|
||
for i, (user_id, channel) in enumerate(employees[:max_employees]):
|
||
records = []
|
||
try:
|
||
page_num = 1
|
||
while True:
|
||
body = await _browser_fetch(
|
||
page,
|
||
INTERCEPT_PATHS["voice_list"],
|
||
json_body={"pageNum": page_num, "pageSize": page_size, "userId": user_id},
|
||
)
|
||
rows = _rows_flat(body if isinstance(body, dict) else {})
|
||
if not rows:
|
||
break
|
||
records.extend(_normalize_voice_clip(row, channel=channel) for row in rows)
|
||
data = body.get("data") or {}
|
||
total = data.get("total")
|
||
if isinstance(total, (int, float)) and page_num * page_size >= int(total):
|
||
break
|
||
page_num += 1
|
||
await asyncio.sleep(1)
|
||
except Exception as e:
|
||
logger.warning(f" [{i+1}] user {user_id} failed: {e}")
|
||
try:
|
||
await page.close()
|
||
except Exception:
|
||
pass
|
||
page = await _open_fetch_page(scraper)
|
||
continue
|
||
|
||
total_fetched += len(records)
|
||
if records:
|
||
await _upsert_all(upsert_voice_clip, records)
|
||
total_upserted += len(records)
|
||
|
||
if (i + 1) % 10 == 0 or not records:
|
||
logger.info(f" [{i+1}/{min(max_employees, len(employees))}] {user_id}({channel}): {len(records)} clips (total: {total_fetched})")
|
||
await asyncio.sleep(2 + (i % 3))
|
||
|
||
logger.info(f"DONE: voice-clips fetched={total_fetched}, upserted={total_upserted}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# WhatsApp messages (browser fetch, per conversation, backward cursor)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_wa_messages(cookie_str: str, max_conversations: int = 100):
|
||
"""Scrape WhatsApp chat records via browser fetch POST
|
||
/qqweb/WhatsAppUser/chatRecords (JSON body {uid, friendUid, start, end}).
|
||
|
||
Conversation pairs come from _discover_conversations (customer table ∪
|
||
voice-call pairs). Pages backward via data.end / data.hasLast
|
||
(same cursor strategy as the WeChat msg/chats scraper).
|
||
"""
|
||
import time as _time
|
||
from app.db.base import async_session_maker
|
||
from app.scraper.whatsapp import _is_wa_divider, _normalize_whatsapp_message
|
||
from app.storage.repos import upsert_messages
|
||
|
||
conversations = await _discover_conversations("whatsapp", "wa_voice_call")
|
||
|
||
logger.info(f"Discovered {len(conversations)} WA conversations, will scrape {min(max_conversations, len(conversations))}")
|
||
|
||
total_fetched = 0
|
||
total_upserted = 0
|
||
|
||
async with PageScraper(cookie_str) as scraper:
|
||
page = await _open_fetch_page(scraper)
|
||
|
||
for i, (uid, friend_uid) in enumerate(conversations[:max_conversations]):
|
||
records = []
|
||
try:
|
||
end_ms = int(_time.time() * 1000)
|
||
while True:
|
||
body = await _browser_fetch(
|
||
page,
|
||
"/qqweb/WhatsAppUser/chatRecords",
|
||
json_body={"uid": uid, "friendUid": friend_uid, "start": 0, "end": end_ms},
|
||
)
|
||
data = body.get("data") if isinstance(body, dict) else None
|
||
if not isinstance(data, dict):
|
||
break
|
||
msgs = data.get("messages") or []
|
||
for row in msgs:
|
||
if isinstance(row, dict) and not _is_wa_divider(row):
|
||
records.append(_normalize_whatsapp_message(row))
|
||
if not data.get("hasLast"):
|
||
break
|
||
end = data.get("end")
|
||
if not isinstance(end, (int, float)):
|
||
break
|
||
end_ms = int(end) - 1
|
||
await asyncio.sleep(1)
|
||
except Exception as e:
|
||
logger.warning(f" [{i+1}] {str(uid)[:20]}:{str(friend_uid)[:20]} failed: {e}")
|
||
# Recreate the fetch page; retry patiently on outages (same
|
||
# resilience as the WeChat message scraper — 30 × 2min).
|
||
for attempt in range(30):
|
||
try:
|
||
try:
|
||
await page.close()
|
||
except Exception:
|
||
pass
|
||
page = await _open_fetch_page(scraper)
|
||
break
|
||
except Exception as rec:
|
||
logger.warning(f" recovery {attempt + 1}/30 failed: {rec}")
|
||
if attempt == 29:
|
||
raise
|
||
await asyncio.sleep(120)
|
||
continue
|
||
|
||
total_fetched += len(records)
|
||
if records:
|
||
# upsert_messages 收整列表(逐条版 _upsert_all 会把 dict 迭代成
|
||
# key 字符串 → TypeError, 2026-08-19 WA 阶段首条即崩的根因)
|
||
async with async_session_maker() as session:
|
||
await upsert_messages(session, records)
|
||
await session.commit()
|
||
total_upserted += len(records)
|
||
|
||
if (i + 1) % 10 == 0:
|
||
logger.info(f" [{i+1}/{min(max_conversations, len(conversations))}] msgs total: {total_fetched}")
|
||
await asyncio.sleep(2 + (i % 3))
|
||
|
||
logger.info(f"DONE: wa-messages fetched={total_fetched}, upserted={total_upserted}")
|
||
return total_fetched, total_upserted
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Media downloads (OSS CDN — NOT API calls, safe from 风控)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def scrape_media(cookie_str: str, batch_size: int = 50):
|
||
"""Download media artifacts into the local object store:
|
||
|
||
* call recordings (phone_call.record_file OSS URLs)
|
||
* voice clip audio (voice_clip.oss_file_name OSS URLs)
|
||
* message images/videos (message.file_url / file_th_url / file_hd_url)
|
||
|
||
These are plain OSS CDN downloads (not CRM API calls), so httpx direct
|
||
fetching is safe. The YunkeClient is only used as the httpx transport.
|
||
"""
|
||
from app.config import get_settings
|
||
from app.db.base import async_session_maker
|
||
from app.scraper.media import (
|
||
download_call_recordings,
|
||
download_message_images,
|
||
download_voice_clip_audio,
|
||
)
|
||
from app.session.yunke_client import YunkeClient
|
||
from app.storage.object_store import ObjectStore
|
||
|
||
settings = get_settings()
|
||
cookie_dict = {c["name"]: c["value"] for c in parse_cookies(cookie_str)}
|
||
# Session-mode client purely as an httpx transport for OSS downloads —
|
||
# userid/company only matter for login(), which we never call here.
|
||
client = YunkeClient(
|
||
base_url=YUNKE_BASE,
|
||
userid=settings.yk_userid or "oss",
|
||
company=settings.yk_company or "oss",
|
||
cookies=cookie_dict,
|
||
)
|
||
store = ObjectStore()
|
||
totals: dict[str, tuple[int, int]] = {}
|
||
try:
|
||
async with async_session_maker() as session:
|
||
totals["call_recordings"] = await download_call_recordings(client, store, session, batch_size=batch_size)
|
||
totals["voice_clip_audio"] = await download_voice_clip_audio(client, store, session, batch_size=batch_size)
|
||
totals["message_images"] = await download_message_images(client, store, session, batch_size=batch_size)
|
||
finally:
|
||
await client.aclose()
|
||
|
||
for name, (attempted, localized) in totals.items():
|
||
logger.info(f" {name}: attempted={attempted}, localized={localized}")
|
||
logger.info(f"DONE: media {totals}")
|
||
return totals
|
||
|
||
|
||
async def main():
|
||
parser = argparse.ArgumentParser(description="页面浏览器抓取器")
|
||
parser.add_argument(
|
||
"--entity",
|
||
required=True,
|
||
choices=[
|
||
"friends", # 微信客户 (per-employee friend pages)
|
||
"groups", # 微信群 (per-employee group pages)
|
||
"messages", # 微信消息 (browser fetch msg/chats)
|
||
"accounts", # 微信员工账号 (wechatlist page interception)
|
||
"wa-accounts", # WA员工账号 (waBusiness/userlist interception)
|
||
"wa-friends", # WA客户 (waBusiness/friendlist interception)
|
||
"voice-calls", # 语音通话记录 (voice-call-list + wavoicecalllist)
|
||
"voice-clips", # 长语音 (browser fetch call/list per employee)
|
||
"wa-messages", # WA消息 (browser fetch chatRecords per conversation)
|
||
"media", # OSS媒体下载 (录音/长语音音频/消息图片)
|
||
],
|
||
help="抓取实体",
|
||
)
|
||
parser.add_argument("--max-employees", type=int, default=100, help="最多遍历多少员工 (friends/groups/voice-clips)")
|
||
parser.add_argument("--max-conversations", type=int, default=100, help="消息模式：最多遍历多少对话 (messages/wa-messages)")
|
||
parser.add_argument("--max-pages", type=int, default=200, help="列表翻页上限 (accounts/wa-accounts/wa-friends/voice-calls)")
|
||
parser.add_argument("--batch-size", type=int, default=50, help="媒体下载批次大小 (media)")
|
||
args = parser.parse_args()
|
||
|
||
cookie = os.environ.get("YK_COOKIE", "")
|
||
if not cookie:
|
||
# Try reading from .env
|
||
from app.config import get_settings
|
||
cookie = get_settings().yk_cookie
|
||
if not cookie:
|
||
logger.error("YK_COOKIE not set in .env or environment")
|
||
return
|
||
|
||
logger.info(f"Starting page-based scrape: entity={args.entity}")
|
||
|
||
if args.entity == "friends":
|
||
await scrape_friends(cookie, args.max_employees)
|
||
elif args.entity == "groups":
|
||
await scrape_groups(cookie, args.max_employees)
|
||
elif args.entity == "messages":
|
||
await scrape_messages(cookie, args.max_conversations)
|
||
elif args.entity == "accounts":
|
||
await scrape_accounts(cookie, args.max_pages)
|
||
elif args.entity == "wa-accounts":
|
||
await scrape_wa_accounts(cookie, args.max_pages)
|
||
elif args.entity == "wa-friends":
|
||
await scrape_wa_friends(cookie, args.max_pages)
|
||
elif args.entity == "voice-calls":
|
||
await scrape_voice_calls(cookie, args.max_pages)
|
||
elif args.entity == "voice-clips":
|
||
await scrape_voice_clips(cookie, args.max_employees)
|
||
elif args.entity == "wa-messages":
|
||
await scrape_wa_messages(cookie, args.max_conversations)
|
||
elif args.entity == "media":
|
||
await scrape_media(cookie, args.batch_size)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|