xinyouhe/xhs_crawler_claude
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ts

Browse Source
This commit is contained in:
mzl
2026-01-12 18:19:24 +08:00
parent 950c61226d
commit fb98a5d221
4 changed files with 385 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/scraper.py
+4 -4
View File
@@ -23,16 +23,16 @@ class RiskControlException(Exception):
class XiaoHongShuScraper:
"""小红书采集器"""
def __init__(self, event_queue: asyncio.Queue, log_manager: LogManager):
def __init__(self, event_queue: asyncio.Queue, log_manager: LogManager, browser=None):
self.event_queue = event_queue
self.log_manager = log_manager
self.playwright = None
self.browser: Optional[Browser] = None
self.browser = browser # 使用传入的浏览器实例
self.is_running = False
self._stop_requested = False
async def init_browser(self):
"""初始化浏览器"""
"""初始化浏览器（如果未传入浏览器实例）"""
if not self.browser:
self.playwright = await async_playwright().start()
self.browser = await self.playwright.chromium.launch(
headless=config.HEADLESS,
app/web.py
+52 -8
View File
@@ -104,6 +104,19 @@ async def get_status():
)
@app.get("/api/debug/browser")
async def debug_browser():
"""调试浏览器状态"""
return {
"browser_exists": app_state.browser is not None,
"contexts_count": len(app_state.browser.contexts) if app_state.browser else 0,
"contexts": [str(type(ctx)) for ctx in app_state.browser.contexts] if app_state.browser else [],
"data_dir": config.DATA_DIR,
"auth_file_exists": os.path.exists(os.path.join(config.DATA_DIR, "auth_state.json")),
"storage_file_exists": os.path.exists(os.path.join(config.DATA_DIR, "storage_state.json"))
}
@app.post("/api/start")
async def start_scraping(request: StartRequest):
"""启动采集"""
@@ -152,8 +165,8 @@ async def start_scraping(request: StartRequest):
# 创建事件队列
app_state.event_queue = asyncio.Queue()
# 创建采集器
app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager)
# 创建采集器（传入已初始化的浏览器实例）
app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager, app_state.browser)
# 启动采集任务
async def run_scraper():
@@ -461,16 +474,34 @@ async def generate_qrcode():
async def check_login_status():
"""检查小红书登录状态（直接验证）"""
try:
print("[DEBUG] === 检查浏览器状态 ===")
print(f"[DEBUG] 浏览器实例存在: {app_state.browser is not None}")
# 如果浏览器可用，直接使用浏览器验证
if app_state.browser:
print("[DEBUG] 浏览器可用，尝试获取context...")
if app_state.browser.contexts:
print(f"[DEBUG] 已有 {len(app_state.browser.contexts)} 个contexts")
context = app_state.browser.contexts[0]
print("[DEBUG] 使用现有context")
else:
print("[DEBUG] 创建新context")
context = await app_state.browser.new_context()
try:
page = await app_state.browser.new_page()
page = await context.new_page()
await page.goto("https://www.xiaohongshu.com/", timeout=10000)
await asyncio.sleep(2)
# 获取页面cookies
cookies = await page.context.cookies()
page.close()
cookies = await context.cookies()
print(f"[DEBUG] 找到 {len(cookies)} 个cookies")
# 打印所有cookies
for cookie in cookies:
cookie_name = cookie.get('name', '')
cookie_value = cookie.get('value', '')[:20] if cookie.get('value') else ''
print(f"[DEBUG] Cookie: {cookie_name} = {cookie_value}...")
# 检查是否有小红书相关的session cookie
session_cookies = []
@@ -478,7 +509,7 @@ async def check_login_status():
cookie_name = cookie.get('name', '').lower()
if any(keyword in cookie_name for keyword in ['session', 'token', 'login', 'auth']):
session_cookies.append(cookie)
print(f"[DEBUG] 找到session cookie: {cookie.get('name')} = {cookie.get('value', '')[:20]}...")
print(f"[DEBUG] ✓ 找到登录cookie: {cookie_name}")
if session_cookies:
return {
@@ -488,21 +519,30 @@ async def check_login_status():
"cookies_count": len(session_cookies)
}
else:
print("[DEBUG] 未找到session cookies")
print("[DEBUG] ✗ 未找到登录相关的 cookies")
return {
"success": False,
"message": "未找到登录状态，请先登录"
}
except Exception as e:
print(f"[DEBUG] 浏览器检查失败: {str(e)}")
print(f"[DEBUG] 浏览器页面操作失败: {str(e)}")
pass
finally:
# 确保页面关闭
if 'page' in locals():
await page.close()
else:
print("[DEBUG] ✗ 浏览器实例不存在")
# 检查本地是否有登录状态文件
print("[DEBUG] 检查本地登录状态文件...")
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
if os.path.exists(auth_file):
try:
with open(auth_file, 'r', encoding='utf-8') as f:
auth_data = json.load(f)
print(f"[DEBUG] ✓ 找到本地auth_state: {auth_data}")
return {
"success": True,
"message": "本地登录状态有效",
@@ -510,13 +550,17 @@ async def check_login_status():
"login_time": auth_data.get("login_time"),
"expire_time": auth_data.get("expire_time")
}
except Exception as e:
print(f"[DEBUG] ✗ 读取auth_state失败: {str(e)}")
print("[DEBUG] ✗ 未找到任何登录状态")
return {
"success": False,
"message": "未找到登录状态，请先登录"
}
except Exception as e:
print(f"[DEBUG] ✗ 检查登录状态异常: {str(e)}")
return {
"success": False,
"message": f"检查登录状态失败: {str(e)}"
login_debug_guide.md
+209
View File
@@ -0,0 +1,209 @@
# 小红书登录状态调试指南
## 问题现象
登录验证失败，提示"未找到登录状态，请先登录"
## 调试步骤
### 1. 检查服务是否正常启动
```bash
# 启动服务
python main.py
# 在另一个终端访问调试接口
curl http://localhost:18085/api/debug/browser
```
**预期输出：**
```json
{
"browser_exists": true,
"contexts_count": 0,
"contexts": [],
"data_dir": "data",
"auth_file_exists": false,
"storage_file_exists": false
}
```
### 2. 检查浏览器状态
查看控制台输出，应该看到：
```
[INFO] 浏览器初始化完成
[DEBUG] === 检查浏览器状态 ===
[DEBUG] 浏览器实例存在: true
```
### 3. 打开登录页面
1. 访问：http://localhost:18085
2. 点击"打开登录页"按钮
3. 在新窗口中扫码登录小红书
4. 登录成功后不要关闭窗口
### 4. 测试登录验证
返回页面，点击"验证登录"按钮，查看控制台输出。
**预期成功输出：**
```
[DEBUG] 开始验证登录状态...
[DEBUG] 尝试保存 storage_state 到: data/storage_state.json
[DEBUG] ✓ 成功保存 storage_state
[DEBUG] storage_state 文件大小: 2456 bytes
[DEBUG] 使用 storage_state 验证登录...
[DEBUG] ✓ 登录验证成功
```
### 5. 手动验证API
使用curl测试各个接口：
```bash
# 检查登录状态
curl http://localhost:18085/api/check-login-status
# 验证登录
curl -X POST http://localhost:18085/api/login/verify
# 查看调试信息
curl http://localhost:18085/api/debug/browser
```
## 常见问题及解决方案
### 问题1：浏览器初始化失败
**现象：** `browser_exists: false`
**原因：** Playwright浏览器未安装
**解决：**
```bash
playwright install chromium
```
### 问题2：Context不存在
**现象：** `contexts_count: 0`
**原因：** 浏览器Context未创建
**解决：** 重启服务，确保浏览器在应用启动时初始化
### 问题3：保存storage_state失败
**现象：** 保存storage_state时异常
**原因：** 权限问题或目录不存在
**解决：**
```bash
mkdir -p data
chmod 755 data
```
### 问题4：验证失败但已登录
**现象：** 已经扫码登录，但验证失败
**原因：**
1. 页面加载不完整
2. 风控拦截
3. Cookie变化
**解决：**
1. 等待页面完全加载后再验证
2. 尝试重新登录
3. 检查控制台详细错误信息
## 详细调试流程
### 1. 完整日志查看
```bash
# 启动服务时显示详细日志
python main.py # 查看所有输出
# 或使用日志级别
uvicorn app.web:app --log-level debug
```
### 2. 文件检查
检查以下文件是否存在：
- `data/storage_state.json` - 浏览器状态文件
- `data/auth_state.json` - 认证状态文件
- `data/logs.json` - 操作日志
### 3. 手动测试storage_state
```python
# 手动测试storage_state保存
from playwright.async_api import async_playwright
async def test_storage():
async with async_playwright() as p:
browser = await p.chromium.launch()
context = await browser.new_context()
page = await context.new_page()
# 访问小红书
await page.goto("https://www.xiaohongshu.com")
await asyncio.sleep(2)
# 保存storage_state
await context.storage_state(path="test_storage.json")
await browser.close()
# 运行测试
asyncio.run(test_storage())
```
### 4. Cookie分析
如果保存了storage_state文件，可以查看其中的cookies部分：
```json
{
"cookies": [
{
"name": "a1",
"value": "...",
"domain": ".xiaohongshu.com",
"path": "/",
"expires": -1,
"httpOnly": false,
"secure": true,
"sameSite": "Lax"
}
]
}
```
## 进阶调试
### 1. 使用浏览器开发者工具
1. 在登录页面打开开发者工具（F12）
2. 切换到Application/Storage标签
3. 查看Cookies和Local Storage
4. 确认登录相关的cookie是否存在
### 2. 网络请求分析
1. 在开发者工具的Network标签中
2. 查看对`xiaohongshu.com`的请求
3. 确认请求头中是否包含正确的Cookie
### 3. 手动验证登录状态
```javascript
// 在浏览器控制台运行
// 检查是否有登录相关的cookie
document.cookie.split(';').forEach(cookie => {
const [name, value] = cookie.trim().split('=');
if (name.includes('session') || name.includes('token') || name.includes('auth')) {
console.log(`${name}: ${value}`);
}
});
// 检查localStorage
console.log('localStorage:', Object.keys(localStorage));
// 检查是否被重定向到登录页
if (window.location.pathname.includes('login')) {
console.log('被重定向到登录页');
}
```
## 联系技术支持
如果以上步骤都无法解决问题，请提供以下信息：
1. 完整的控制台输出
2. 三个文件的内容（如果存在）：
- `data/storage_state.json`
- `data/auth_state.json`
- `data/logs.json`
3. 浏览器类型和版本
4. 操作系统信息
test_login.py
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""
测试登录验证功能
"""
import asyncio
import json
import os
import requests
import websockets
from datetime import datetime
class LoginTester:
def __init__(self, base_url="http://localhost:18085"):
self.base_url = base_url
async def test_browser_status(self):
"""测试浏览器状态"""
print("\n=== 测试浏览器状态 ===")
try:
response = requests.get(f"{self.base_url}/api/debug/browser")
data = response.json()
print(f"浏览器实例存在: {data['browser_exists']}")
print(f"Context数量: {data['contexts_count']}")
print(f"数据目录: {data['data_dir']}")
print(f"Auth文件存在: {data['auth_file_exists']}")
print(f"Storage文件存在: {data['storage_file_exists']}")
return data
except Exception as e:
print(f"✗ 测试失败: {e}")
return None
async def test_login_status(self):
"""测试登录状态检查"""
print("\n=== 测试登录状态检查 ===")
try:
response = requests.get(f"{self.base_url}/api/check-login-status")
data = response.json()
print(f"响应: {json.dumps(data, indent=2, ensure_ascii=False)}")
return data
except Exception as e:
print(f"✗ 测试失败: {e}")
return None
async def test_login_verify(self):
"""测试登录验证"""
print("\n=== 测试登录验证 ===")
try:
response = requests.post(f"{self.base_url}/api/login/verify")
data = response.json()
print(f"响应: {json.dumps(data, indent=2, ensure_ascii=False)}")
return data
except Exception as e:
print(f"✗ 测试失败: {e}")
return None
async def test_sse_connection(self):
"""测试SSE连接"""
print("\n=== 测试SSE连接 ===")
try:
ws_url = f"{self.base_url.replace('http', 'ws')}/api/events"
async with websockets.connect(ws_url) as websocket:
# 等待几条消息
for i in range(3):
message = await asyncio.wait_for(websocket.recv(), timeout=5.0)
print(f"收到消息 {i+1}: {message}")
if i >= 2: # 只接收3条消息就关闭
break
except Exception as e:
print(f"✗ 测试失败: {e}")
async def run_all_tests(self):
"""运行所有测试"""
print(f"🚀 开始测试 - {datetime.now()}")
print("=" * 50)
# 等待服务启动
await asyncio.sleep(2)
# 测试1: 浏览器状态
browser_status = await self.test_browser_status()
# 测试2: 登录状态检查
login_status = await self.test_login_status()
# 测试3: 登录验证
login_verify = await self.test_login_verify()
# 测试4: SSE连接（如果服务正在运行）
if browser_status and browser_status.get('browser_exists'):
await self.test_sse_connection()
print("\n" + "=" * 50)
print("🎯 测试完成")
# 输出总结
print("\n📊 测试总结:")
print(f"浏览器状态: {'✅ 正常' if browser_status and browser_status.get('browser_exists') else '❌ 异常'}")
print(f"登录状态检查: {'✅ 成功' if login_status and login_status.get('success') else '❌ 失败'}")
print(f"登录验证: {'✅ 成功' if login_verify and login_verify.get('success') else '❌ 失败'}")
if __name__ == "__main__":
tester = LoginTester()
asyncio.run(tester.run_all_tests())