diff --git a/app/scraper.py b/app/scraper.py index 7c81839..81e698c 100644 --- a/app/scraper.py +++ b/app/scraper.py @@ -23,22 +23,22 @@ class RiskControlException(Exception): class XiaoHongShuScraper: """小红书采集器""" - def __init__(self, event_queue: asyncio.Queue, log_manager: LogManager): + def __init__(self, event_queue: asyncio.Queue, log_manager: LogManager, browser=None): self.event_queue = event_queue self.log_manager = log_manager - self.playwright = None - self.browser: Optional[Browser] = None + self.browser = browser # 使用传入的浏览器实例 self.is_running = False self._stop_requested = False async def init_browser(self): - """初始化浏览器""" - self.playwright = await async_playwright().start() - self.browser = await self.playwright.chromium.launch( - headless=config.HEADLESS, - args=config.BROWSER_ARGS - ) - await self.emit_log("SUCCESS", "浏览器初始化完成") + """初始化浏览器(如果未传入浏览器实例)""" + if not self.browser: + self.playwright = await async_playwright().start() + self.browser = await self.playwright.chromium.launch( + headless=config.HEADLESS, + args=config.BROWSER_ARGS + ) + await self.emit_log("SUCCESS", "浏览器初始化完成") async def close_browser(self): """关闭浏览器""" diff --git a/app/web.py b/app/web.py index febda3d..58af4bd 100644 --- a/app/web.py +++ b/app/web.py @@ -104,6 +104,19 @@ async def get_status(): ) +@app.get("/api/debug/browser") +async def debug_browser(): + """调试浏览器状态""" + return { + "browser_exists": app_state.browser is not None, + "contexts_count": len(app_state.browser.contexts) if app_state.browser else 0, + "contexts": [str(type(ctx)) for ctx in app_state.browser.contexts] if app_state.browser else [], + "data_dir": config.DATA_DIR, + "auth_file_exists": os.path.exists(os.path.join(config.DATA_DIR, "auth_state.json")), + "storage_file_exists": os.path.exists(os.path.join(config.DATA_DIR, "storage_state.json")) + } + + @app.post("/api/start") async def start_scraping(request: StartRequest): """启动采集""" @@ -152,8 +165,8 @@ async def start_scraping(request: StartRequest): # 创建事件队列 app_state.event_queue = asyncio.Queue() - # 创建采集器 - app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager) + # 创建采集器(传入已初始化的浏览器实例) + app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager, app_state.browser) # 启动采集任务 async def run_scraper(): @@ -461,16 +474,34 @@ async def generate_qrcode(): async def check_login_status(): """检查小红书登录状态(直接验证)""" try: + print("[DEBUG] === 检查浏览器状态 ===") + print(f"[DEBUG] 浏览器实例存在: {app_state.browser is not None}") + # 如果浏览器可用,直接使用浏览器验证 if app_state.browser: + print("[DEBUG] 浏览器可用,尝试获取context...") + if app_state.browser.contexts: + print(f"[DEBUG] 已有 {len(app_state.browser.contexts)} 个contexts") + context = app_state.browser.contexts[0] + print("[DEBUG] 使用现有context") + else: + print("[DEBUG] 创建新context") + context = await app_state.browser.new_context() + try: - page = await app_state.browser.new_page() + page = await context.new_page() await page.goto("https://www.xiaohongshu.com/", timeout=10000) await asyncio.sleep(2) # 获取页面cookies - cookies = await page.context.cookies() - page.close() + cookies = await context.cookies() + print(f"[DEBUG] 找到 {len(cookies)} 个cookies") + + # 打印所有cookies + for cookie in cookies: + cookie_name = cookie.get('name', '') + cookie_value = cookie.get('value', '')[:20] if cookie.get('value') else '' + print(f"[DEBUG] Cookie: {cookie_name} = {cookie_value}...") # 检查是否有小红书相关的session cookie session_cookies = [] @@ -478,7 +509,7 @@ async def check_login_status(): cookie_name = cookie.get('name', '').lower() if any(keyword in cookie_name for keyword in ['session', 'token', 'login', 'auth']): session_cookies.append(cookie) - print(f"[DEBUG] 找到session cookie: {cookie.get('name')} = {cookie.get('value', '')[:20]}...") + print(f"[DEBUG] ✓ 找到登录cookie: {cookie_name}") if session_cookies: return { @@ -488,35 +519,48 @@ async def check_login_status(): "cookies_count": len(session_cookies) } else: - print("[DEBUG] 未找到session cookies") + print("[DEBUG] ✗ 未找到登录相关的 cookies") return { "success": False, "message": "未找到登录状态,请先登录" } except Exception as e: - print(f"[DEBUG] 浏览器检查失败: {str(e)}") + print(f"[DEBUG] 浏览器页面操作失败: {str(e)}") pass + finally: + # 确保页面关闭 + if 'page' in locals(): + await page.close() + else: + print("[DEBUG] ✗ 浏览器实例不存在") # 检查本地是否有登录状态文件 + print("[DEBUG] 检查本地登录状态文件...") auth_file = os.path.join(config.DATA_DIR, "auth_state.json") if os.path.exists(auth_file): - with open(auth_file, 'r', encoding='utf-8') as f: - auth_data = json.load(f) - return { - "success": True, - "message": "本地登录状态有效", - "logged_in": True, - "login_time": auth_data.get("login_time"), - "expire_time": auth_data.get("expire_time") - } + try: + with open(auth_file, 'r', encoding='utf-8') as f: + auth_data = json.load(f) + print(f"[DEBUG] ✓ 找到本地auth_state: {auth_data}") + return { + "success": True, + "message": "本地登录状态有效", + "logged_in": True, + "login_time": auth_data.get("login_time"), + "expire_time": auth_data.get("expire_time") + } + except Exception as e: + print(f"[DEBUG] ✗ 读取auth_state失败: {str(e)}") + print("[DEBUG] ✗ 未找到任何登录状态") return { "success": False, "message": "未找到登录状态,请先登录" } except Exception as e: + print(f"[DEBUG] ✗ 检查登录状态异常: {str(e)}") return { "success": False, "message": f"检查登录状态失败: {str(e)}" diff --git a/login_debug_guide.md b/login_debug_guide.md new file mode 100644 index 0000000..3a9f04e --- /dev/null +++ b/login_debug_guide.md @@ -0,0 +1,209 @@ +# 小红书登录状态调试指南 + +## 问题现象 +登录验证失败,提示"未找到登录状态,请先登录" + +## 调试步骤 + +### 1. 检查服务是否正常启动 +```bash +# 启动服务 +python main.py + +# 在另一个终端访问调试接口 +curl http://localhost:18085/api/debug/browser +``` + +**预期输出:** +```json +{ + "browser_exists": true, + "contexts_count": 0, + "contexts": [], + "data_dir": "data", + "auth_file_exists": false, + "storage_file_exists": false +} +``` + +### 2. 检查浏览器状态 +查看控制台输出,应该看到: +``` +[INFO] 浏览器初始化完成 +[DEBUG] === 检查浏览器状态 === +[DEBUG] 浏览器实例存在: true +``` + +### 3. 打开登录页面 +1. 访问:http://localhost:18085 +2. 点击"打开登录页"按钮 +3. 在新窗口中扫码登录小红书 +4. 登录成功后不要关闭窗口 + +### 4. 测试登录验证 +返回页面,点击"验证登录"按钮,查看控制台输出。 + +**预期成功输出:** +``` +[DEBUG] 开始验证登录状态... +[DEBUG] 尝试保存 storage_state 到: data/storage_state.json +[DEBUG] ✓ 成功保存 storage_state +[DEBUG] storage_state 文件大小: 2456 bytes +[DEBUG] 使用 storage_state 验证登录... +[DEBUG] ✓ 登录验证成功 +``` + +### 5. 手动验证API +使用curl测试各个接口: +```bash +# 检查登录状态 +curl http://localhost:18085/api/check-login-status + +# 验证登录 +curl -X POST http://localhost:18085/api/login/verify + +# 查看调试信息 +curl http://localhost:18085/api/debug/browser +``` + +## 常见问题及解决方案 + +### 问题1:浏览器初始化失败 +**现象:** `browser_exists: false` + +**原因:** Playwright浏览器未安装 +**解决:** +```bash +playwright install chromium +``` + +### 问题2:Context不存在 +**现象:** `contexts_count: 0` + +**原因:** 浏览器Context未创建 +**解决:** 重启服务,确保浏览器在应用启动时初始化 + +### 问题3:保存storage_state失败 +**现象:** 保存storage_state时异常 + +**原因:** 权限问题或目录不存在 +**解决:** +```bash +mkdir -p data +chmod 755 data +``` + +### 问题4:验证失败但已登录 +**现象:** 已经扫码登录,但验证失败 + +**原因:** +1. 页面加载不完整 +2. 风控拦截 +3. Cookie变化 + +**解决:** +1. 等待页面完全加载后再验证 +2. 尝试重新登录 +3. 检查控制台详细错误信息 + +## 详细调试流程 + +### 1. 完整日志查看 +```bash +# 启动服务时显示详细日志 +python main.py # 查看所有输出 + +# 或使用日志级别 +uvicorn app.web:app --log-level debug +``` + +### 2. 文件检查 +检查以下文件是否存在: +- `data/storage_state.json` - 浏览器状态文件 +- `data/auth_state.json` - 认证状态文件 +- `data/logs.json` - 操作日志 + +### 3. 手动测试storage_state +```python +# 手动测试storage_state保存 +from playwright.async_api import async_playwright + +async def test_storage(): + async with async_playwright() as p: + browser = await p.chromium.launch() + context = await browser.new_context() + page = await context.new_page() + + # 访问小红书 + await page.goto("https://www.xiaohongshu.com") + await asyncio.sleep(2) + + # 保存storage_state + await context.storage_state(path="test_storage.json") + await browser.close() + +# 运行测试 +asyncio.run(test_storage()) +``` + +### 4. Cookie分析 +如果保存了storage_state文件,可以查看其中的cookies部分: +```json +{ + "cookies": [ + { + "name": "a1", + "value": "...", + "domain": ".xiaohongshu.com", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": true, + "sameSite": "Lax" + } + ] +} +``` + +## 进阶调试 + +### 1. 使用浏览器开发者工具 +1. 在登录页面打开开发者工具(F12) +2. 切换到Application/Storage标签 +3. 查看Cookies和Local Storage +4. 确认登录相关的cookie是否存在 + +### 2. 网络请求分析 +1. 在开发者工具的Network标签中 +2. 查看对`xiaohongshu.com`的请求 +3. 确认请求头中是否包含正确的Cookie + +### 3. 手动验证登录状态 +```javascript +// 在浏览器控制台运行 +// 检查是否有登录相关的cookie +document.cookie.split(';').forEach(cookie => { + const [name, value] = cookie.trim().split('='); + if (name.includes('session') || name.includes('token') || name.includes('auth')) { + console.log(`${name}: ${value}`); + } +}); + +// 检查localStorage +console.log('localStorage:', Object.keys(localStorage)); + +// 检查是否被重定向到登录页 +if (window.location.pathname.includes('login')) { + console.log('被重定向到登录页'); +} +``` + +## 联系技术支持 +如果以上步骤都无法解决问题,请提供以下信息: +1. 完整的控制台输出 +2. 三个文件的内容(如果存在): + - `data/storage_state.json` + - `data/auth_state.json` + - `data/logs.json` +3. 浏览器类型和版本 +4. 操作系统信息 \ No newline at end of file diff --git a/test_login.py b/test_login.py new file mode 100644 index 0000000..e8d793d --- /dev/null +++ b/test_login.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +测试登录验证功能 +""" +import asyncio +import json +import os +import requests +import websockets +from datetime import datetime + + +class LoginTester: + def __init__(self, base_url="http://localhost:18085"): + self.base_url = base_url + + async def test_browser_status(self): + """测试浏览器状态""" + print("\n=== 测试浏览器状态 ===") + try: + response = requests.get(f"{self.base_url}/api/debug/browser") + data = response.json() + print(f"浏览器实例存在: {data['browser_exists']}") + print(f"Context数量: {data['contexts_count']}") + print(f"数据目录: {data['data_dir']}") + print(f"Auth文件存在: {data['auth_file_exists']}") + print(f"Storage文件存在: {data['storage_file_exists']}") + return data + except Exception as e: + print(f"✗ 测试失败: {e}") + return None + + async def test_login_status(self): + """测试登录状态检查""" + print("\n=== 测试登录状态检查 ===") + try: + response = requests.get(f"{self.base_url}/api/check-login-status") + data = response.json() + print(f"响应: {json.dumps(data, indent=2, ensure_ascii=False)}") + return data + except Exception as e: + print(f"✗ 测试失败: {e}") + return None + + async def test_login_verify(self): + """测试登录验证""" + print("\n=== 测试登录验证 ===") + try: + response = requests.post(f"{self.base_url}/api/login/verify") + data = response.json() + print(f"响应: {json.dumps(data, indent=2, ensure_ascii=False)}") + return data + except Exception as e: + print(f"✗ 测试失败: {e}") + return None + + async def test_sse_connection(self): + """测试SSE连接""" + print("\n=== 测试SSE连接 ===") + try: + ws_url = f"{self.base_url.replace('http', 'ws')}/api/events" + async with websockets.connect(ws_url) as websocket: + # 等待几条消息 + for i in range(3): + message = await asyncio.wait_for(websocket.recv(), timeout=5.0) + print(f"收到消息 {i+1}: {message}") + if i >= 2: # 只接收3条消息就关闭 + break + except Exception as e: + print(f"✗ 测试失败: {e}") + + async def run_all_tests(self): + """运行所有测试""" + print(f"🚀 开始测试 - {datetime.now()}") + print("=" * 50) + + # 等待服务启动 + await asyncio.sleep(2) + + # 测试1: 浏览器状态 + browser_status = await self.test_browser_status() + + # 测试2: 登录状态检查 + login_status = await self.test_login_status() + + # 测试3: 登录验证 + login_verify = await self.test_login_verify() + + # 测试4: SSE连接(如果服务正在运行) + if browser_status and browser_status.get('browser_exists'): + await self.test_sse_connection() + + print("\n" + "=" * 50) + print("🎯 测试完成") + + # 输出总结 + print("\n📊 测试总结:") + print(f"浏览器状态: {'✅ 正常' if browser_status and browser_status.get('browser_exists') else '❌ 异常'}") + print(f"登录状态检查: {'✅ 成功' if login_status and login_status.get('success') else '❌ 失败'}") + print(f"登录验证: {'✅ 成功' if login_verify and login_verify.get('success') else '❌ 失败'}") + + +if __name__ == "__main__": + tester = LoginTester() + asyncio.run(tester.run_all_tests()) \ No newline at end of file