fix(scraper): 消息页面抓取器改用浏览器fetch(可靠)+空数据安全

yunke-server/scripts/scrape_via_page.py

@@ -380,28 +380,45 @@ async def scrape_messages(cookie_str: str, max_conversations: int = 100):
     total_upserted = 0
 
     async with PageScraper(cookie_str) as scraper:
+        page = scraper.page
+        # First navigate to the CRM to establish session context
+        await page.goto(f"{YUNKE_BASE}/cms/wechat/wechatlist", wait_until="domcontentloaded")
+        await page.wait_for_timeout(3000)
+
         for i, (owner_id, friend_id) in enumerate(conversations[:max_conversations]):
-            # Chat window URL: opens this specific conversation
-            chat_url = f"{YUNKE_BASE}/cms/wechat/salewechat/message/wechat-window/1/{owner_id}/{friend_id}"
-
-            # Intercept the message API response
-            scraper.start_capture(INTERCEPT_PATHS["msg_chats"])
-
             try:
-                await scraper.navigate_and_wait(chat_url, wait_selector=".yk-chat", timeout=10000)
+                # Use browser fetch() to call the message API from the page context
-                await scraper.page.wait_for_timeout(3000)  # let the message API complete
+                # (the browser sends it with proper cookies/headers → no 风控)
-            except Exception as e:
+                body = await page.evaluate("""async ([ownerId, friendId]) => {
-                logger.warning(f"  [{i+1}] {owner_id[:12]}:{friend_id[:12]} nav failed: {e}")
+                    const resp = await fetch('/wechatweb/msg/chats', {
-                continue
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify({
+                            pageNum: 1,
+                            pageSize: 50,
+                            wechatId: ownerId,
+                            friendId: friendId,
+                            type: '',
+                        }),
+                    });
+                    return await resp.json();
+                }""", [owner_id, friend_id])
 
-            responses = scraper.get_captured()
+                responses = [{"body": body, "status": 200}]
+
+            except Exception as e:
+                logger.warning(f"  [{i+1}] {owner_id[:12]}:{friend_id[:12]} failed: {e}")
+                # Re-navigate to reset the page if session expired
+                await page.goto(f"{YUNKE_BASE}/cms/wechat/wechatlist", wait_until="domcontentloaded")
+                await page.wait_for_timeout(2000)
+                continue
 
             # Extract message records
             records = []
             for resp in responses:
                 body = resp.get("body", {})
-                data = body.get("data", {})
+                data = body.get("data") or {}
-                rows = data.get("data", []) or []
+                rows = data.get("data") or []
                 for row in rows:
                     if not isinstance(row, dict):
                         continue