From cf858a00c23573fa767777336eda07ec2f766f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= <email@sample.com>
Date: Fri, 14 Aug 2026 15:30:02 +0800
Subject: [PATCH] =?UTF-8?q?fix(scraper):=20=E6=B6=88=E6=81=AF=E9=A1=B5?=
 =?UTF-8?q?=E9=9D=A2=E6=8A=93=E5=8F=96=E5=99=A8=E6=94=B9=E7=94=A8=E6=B5=8F?=
 =?UTF-8?q?=E8=A7=88=E5=99=A8fetch(=E5=8F=AF=E9=9D=A0)+=E7=A9=BA=E6=95=B0?=
 =?UTF-8?q?=E6=8D=AE=E5=AE=89=E5=85=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 yunke-server/scripts/scrape_via_page.py | 45 +++++++++++++++++--------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/yunke-server/scripts/scrape_via_page.py b/yunke-server/scripts/scrape_via_page.py
index 218faae..864c06a 100644
--- a/yunke-server/scripts/scrape_via_page.py
+++ b/yunke-server/scripts/scrape_via_page.py
@@ -380,28 +380,45 @@ async def scrape_messages(cookie_str: str, max_conversations: int = 100):
     total_upserted = 0
 
     async with PageScraper(cookie_str) as scraper:
+        page = scraper.page
+        # First navigate to the CRM to establish session context
+        await page.goto(f"{YUNKE_BASE}/cms/wechat/wechatlist", wait_until="domcontentloaded")
+        await page.wait_for_timeout(3000)
+
         for i, (owner_id, friend_id) in enumerate(conversations[:max_conversations]):
-            # Chat window URL: opens this specific conversation
-            chat_url = f"{YUNKE_BASE}/cms/wechat/salewechat/message/wechat-window/1/{owner_id}/{friend_id}"
-
-            # Intercept the message API response
-            scraper.start_capture(INTERCEPT_PATHS["msg_chats"])
-
             try:
-                await scraper.navigate_and_wait(chat_url, wait_selector=".yk-chat", timeout=10000)
-                await scraper.page.wait_for_timeout(3000)  # let the message API complete
-            except Exception as e:
-                logger.warning(f"  [{i+1}] {owner_id[:12]}:{friend_id[:12]} nav failed: {e}")
-                continue
+                # Use browser fetch() to call the message API from the page context
+                # (the browser sends it with proper cookies/headers → no 风控)
+                body = await page.evaluate("""async ([ownerId, friendId]) => {
+                    const resp = await fetch('/wechatweb/msg/chats', {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify({
+                            pageNum: 1,
+                            pageSize: 50,
+                            wechatId: ownerId,
+                            friendId: friendId,
+                            type: '',
+                        }),
+                    });
+                    return await resp.json();
+                }""", [owner_id, friend_id])
 
-            responses = scraper.get_captured()
+                responses = [{"body": body, "status": 200}]
+
+            except Exception as e:
+                logger.warning(f"  [{i+1}] {owner_id[:12]}:{friend_id[:12]} failed: {e}")
+                # Re-navigate to reset the page if session expired
+                await page.goto(f"{YUNKE_BASE}/cms/wechat/wechatlist", wait_until="domcontentloaded")
+                await page.wait_for_timeout(2000)
+                continue
 
             # Extract message records
             records = []
             for resp in responses:
                 body = resp.get("body", {})
-                data = body.get("data", {})
-                rows = data.get("data", []) or []
+                data = body.get("data") or {}
+                rows = data.get("data") or []
                 for row in rows:
                     if not isinstance(row, dict):
                         continue