diff --git a/yunke-server/scripts/scrape_via_page.py b/yunke-server/scripts/scrape_via_page.py index 218faae..864c06a 100644 --- a/yunke-server/scripts/scrape_via_page.py +++ b/yunke-server/scripts/scrape_via_page.py @@ -380,28 +380,45 @@ async def scrape_messages(cookie_str: str, max_conversations: int = 100): total_upserted = 0 async with PageScraper(cookie_str) as scraper: + page = scraper.page + # First navigate to the CRM to establish session context + await page.goto(f"{YUNKE_BASE}/cms/wechat/wechatlist", wait_until="domcontentloaded") + await page.wait_for_timeout(3000) + for i, (owner_id, friend_id) in enumerate(conversations[:max_conversations]): - # Chat window URL: opens this specific conversation - chat_url = f"{YUNKE_BASE}/cms/wechat/salewechat/message/wechat-window/1/{owner_id}/{friend_id}" - - # Intercept the message API response - scraper.start_capture(INTERCEPT_PATHS["msg_chats"]) - try: - await scraper.navigate_and_wait(chat_url, wait_selector=".yk-chat", timeout=10000) - await scraper.page.wait_for_timeout(3000) # let the message API complete - except Exception as e: - logger.warning(f" [{i+1}] {owner_id[:12]}:{friend_id[:12]} nav failed: {e}") - continue + # Use browser fetch() to call the message API from the page context + # (the browser sends it with proper cookies/headers → no 风控) + body = await page.evaluate("""async ([ownerId, friendId]) => { + const resp = await fetch('/wechatweb/msg/chats', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + pageNum: 1, + pageSize: 50, + wechatId: ownerId, + friendId: friendId, + type: '', + }), + }); + return await resp.json(); + }""", [owner_id, friend_id]) - responses = scraper.get_captured() + responses = [{"body": body, "status": 200}] + + except Exception as e: + logger.warning(f" [{i+1}] {owner_id[:12]}:{friend_id[:12]} failed: {e}") + # Re-navigate to reset the page if session expired + await page.goto(f"{YUNKE_BASE}/cms/wechat/wechatlist", wait_until="domcontentloaded") + await page.wait_for_timeout(2000) + continue # Extract message records records = [] for resp in responses: body = resp.get("body", {}) - data = body.get("data", {}) - rows = data.get("data", []) or [] + data = body.get("data") or {} + rows = data.get("data") or [] for row in rows: if not isinstance(row, dict): continue