1097 lines
36 KiB
Python
1097 lines
36 KiB
Python
"""
|
|
Web Backend - FastAPI 应用
|
|
提供 API 和 SSE 事件推送
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, List, Optional
|
|
from fastapi import FastAPI, HTTPException, BackgroundTasks
|
|
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.templating import Jinja2Templates
|
|
from fastapi.requests import Request
|
|
from pydantic import BaseModel
|
|
from typing import List as ListType
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
from app import config
|
|
from app.log_manager import get_log_manager
|
|
from app.scraper import XiaoHongShuScraper
|
|
|
|
|
|
# ========== Pydantic 模型 ==========
|
|
class StartRequest(BaseModel):
|
|
mode: str = "single" # single / multi
|
|
max_threads: int = 3
|
|
|
|
|
|
class StatusResponse(BaseModel):
|
|
is_running: bool
|
|
mode: str
|
|
max_threads: int
|
|
total_users: int
|
|
completed_users: int
|
|
failed_users: int
|
|
active_threads: int
|
|
users_status: Dict[str, Dict]
|
|
|
|
|
|
# ========== 全局状态 ==========
|
|
class AppState:
|
|
def __init__(self):
|
|
self.is_running = False
|
|
self.mode = "single"
|
|
self.max_threads = 1
|
|
self.total_users = 0
|
|
self.completed_users = 0
|
|
self.failed_users = 0
|
|
self.active_threads = 0
|
|
self.users_status: Dict[str, Dict] = {}
|
|
self.thread_status: Dict[str, Dict] = {}
|
|
self.event_queue: asyncio.Queue = None
|
|
self.scraper: Optional[XiaoHongShuScraper] = None
|
|
self.scraper_task: Optional[asyncio.Task] = None
|
|
self.browser = None
|
|
self.playwright = None
|
|
|
|
|
|
app_state = AppState()
|
|
log_manager = get_log_manager()
|
|
|
|
# ========== FastAPI 应用 ==========
|
|
app = FastAPI(title="小红书数据采集器", version="1.0.0")
|
|
|
|
# 添加 CORS 中间件
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"], # 生产环境应该指定具体域名
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
expose_headers=["*"]
|
|
)
|
|
|
|
# 挂载静态文件和模板
|
|
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
templates = Jinja2Templates(directory="templates")
|
|
|
|
|
|
# ========== 路由 ==========
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def index(request: Request):
|
|
"""首页"""
|
|
return templates.TemplateResponse("index.html", {"request": request})
|
|
|
|
|
|
|
|
|
|
@app.get("/api/status", response_model=StatusResponse)
|
|
async def get_status():
|
|
"""获取当前状态"""
|
|
return StatusResponse(
|
|
is_running=app_state.is_running,
|
|
mode=app_state.mode,
|
|
max_threads=app_state.max_threads,
|
|
total_users=app_state.total_users,
|
|
completed_users=app_state.completed_users,
|
|
failed_users=app_state.failed_users,
|
|
active_threads=app_state.active_threads,
|
|
users_status=app_state.users_status
|
|
)
|
|
|
|
|
|
@app.get("/api/debug/browser")
|
|
async def debug_browser():
|
|
"""调试浏览器状态"""
|
|
return {
|
|
"browser_exists": app_state.browser is not None,
|
|
"contexts_count": len(app_state.browser.contexts) if app_state.browser else 0,
|
|
"contexts": [str(type(ctx)) for ctx in app_state.browser.contexts] if app_state.browser else [],
|
|
"data_dir": config.DATA_DIR,
|
|
"auth_file_exists": os.path.exists(os.path.join(config.DATA_DIR, "auth_state.json")),
|
|
"storage_file_exists": os.path.exists(os.path.join(config.DATA_DIR, "storage_state.json"))
|
|
}
|
|
|
|
|
|
@app.post("/api/start")
|
|
async def start_scraping(request: StartRequest):
|
|
"""启动采集"""
|
|
if app_state.is_running:
|
|
raise HTTPException(status_code=400, detail="采集任务正在运行中")
|
|
|
|
# 读取用户 ID 列表
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
if not os.path.exists(users_file):
|
|
raise HTTPException(status_code=400, detail="用户文件不存在，请先创建 data/users.txt")
|
|
|
|
with open(users_file, 'r', encoding='utf-8') as f:
|
|
user_ids = [line.strip() for line in f if line.strip()]
|
|
|
|
if not user_ids:
|
|
raise HTTPException(status_code=400, detail="用户 ID 列表为空")
|
|
|
|
# 验证模式
|
|
if request.mode not in ["single", "multi"]:
|
|
raise HTTPException(status_code=400, detail="无效的模式，必须是 single 或 multi")
|
|
|
|
# 验证线程数
|
|
if request.mode == "multi":
|
|
if request.max_threads < 1 or request.max_threads > config.MAX_CONCURRENT_THREADS:
|
|
raise HTTPException(status_code=400, detail=f"线程数必须在 1-{config.MAX_CONCURRENT_THREADS} 之间")
|
|
|
|
# 重置状态
|
|
app_state.is_running = True
|
|
app_state.mode = request.mode
|
|
app_state.max_threads = request.max_threads if request.mode == "multi" else 1
|
|
app_state.total_users = len(user_ids)
|
|
app_state.completed_users = 0
|
|
app_state.failed_users = 0
|
|
app_state.active_threads = 0
|
|
app_state.users_status = {}
|
|
app_state.thread_status = {}
|
|
|
|
# 初始化用户状态
|
|
for user_id in user_ids:
|
|
app_state.users_status[user_id] = {
|
|
"status": "pending",
|
|
"note_count": 0,
|
|
"thread_id": None
|
|
}
|
|
|
|
# 创建事件队列
|
|
app_state.event_queue = asyncio.Queue()
|
|
|
|
# 创建采集器（传入已初始化的浏览器实例）
|
|
app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager, app_state.browser)
|
|
|
|
# 启动采集任务
|
|
async def run_scraper():
|
|
try:
|
|
await app_state.scraper.run(
|
|
user_ids=user_ids,
|
|
mode=request.mode,
|
|
max_threads=request.max_threads
|
|
)
|
|
finally:
|
|
app_state.is_running = False
|
|
await app_state.event_queue.put({
|
|
"type": "finished"
|
|
})
|
|
|
|
app_state.scraper_task = asyncio.create_task(run_scraper())
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"采集任务已启动，模式: {request.mode}, 用户数: {len(user_ids)}",
|
|
"total_users": len(user_ids),
|
|
"mode": request.mode,
|
|
"max_threads": app_state.max_threads
|
|
}
|
|
|
|
|
|
@app.post("/api/stop")
|
|
async def stop_scraping():
|
|
"""停止采集"""
|
|
if not app_state.is_running:
|
|
raise HTTPException(status_code=400, detail="没有正在运行的采集任务")
|
|
|
|
if app_state.scraper:
|
|
app_state.scraper.stop()
|
|
|
|
# 取消任务
|
|
if app_state.scraper_task:
|
|
app_state.scraper_task.cancel()
|
|
try:
|
|
await app_state.scraper_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
|
|
app_state.is_running = False
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "采集任务已停止"
|
|
}
|
|
|
|
|
|
@app.post("/api/logs/clear")
|
|
async def clear_logs():
|
|
"""清空日志"""
|
|
log_manager.clear()
|
|
return {
|
|
"success": True,
|
|
"message": "日志已清空"
|
|
}
|
|
|
|
|
|
@app.get("/api/logs")
|
|
async def get_logs(level: Optional[str] = None, limit: Optional[int] = None):
|
|
"""获取日志"""
|
|
logs = log_manager.get_logs(level=level, limit=limit)
|
|
return {
|
|
"logs": logs,
|
|
"stats": log_manager.get_stats()
|
|
}
|
|
|
|
|
|
@app.get("/api/logs/search")
|
|
async def search_logs(keyword: str, limit: Optional[int] = None):
|
|
"""搜索日志"""
|
|
logs = log_manager.search_logs(keyword, limit)
|
|
return {
|
|
"logs": logs,
|
|
"count": len(logs)
|
|
}
|
|
|
|
|
|
@app.get("/api/logs/download")
|
|
async def download_logs():
|
|
"""下载日志文件"""
|
|
text = log_manager.export_to_text()
|
|
|
|
return StreamingResponse(
|
|
iter([text]),
|
|
media_type="text/plain",
|
|
headers={
|
|
"Content-Disposition": f"attachment; filename=xiaohongshu_logs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
|
|
}
|
|
)
|
|
|
|
|
|
@app.get("/api/users")
|
|
async def get_users():
|
|
"""获取用户列表"""
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
|
|
if not os.path.exists(users_file):
|
|
return {"users": []}
|
|
|
|
with open(users_file, 'r', encoding='utf-8') as f:
|
|
users = [{"id": line.strip()} for line in f if line.strip()]
|
|
|
|
# 添加采集状态
|
|
for user in users:
|
|
user_id = user["id"]
|
|
if user_id in app_state.users_status:
|
|
user.update(app_state.users_status[user_id])
|
|
else:
|
|
user["status"] = "pending"
|
|
user["note_count"] = 0
|
|
|
|
return {"users": users}
|
|
|
|
|
|
@app.post("/api/users")
|
|
async def add_user(request: dict):
|
|
"""添加用户"""
|
|
user_id = request.get("user_id", "").strip()
|
|
|
|
if not user_id:
|
|
raise HTTPException(status_code=400, detail="用户ID不能为空")
|
|
|
|
# 验证用户ID格式
|
|
if not user_id or len(user_id) < 10:
|
|
raise HTTPException(status_code=400, detail="用户ID格式不正确")
|
|
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
|
|
# 检查是否已存在
|
|
if os.path.exists(users_file):
|
|
with open(users_file, 'r', encoding='utf-8') as f:
|
|
existing_users = [line.strip() for line in f if line.strip()]
|
|
if user_id in existing_users:
|
|
raise HTTPException(status_code=400, detail="用户ID已存在")
|
|
|
|
# 添加到文件
|
|
with open(users_file, 'a', encoding='utf-8') as f:
|
|
f.write(f"{user_id}\n")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"用户 {user_id} 已添加",
|
|
"user_id": user_id
|
|
}
|
|
|
|
|
|
@app.delete("/api/users/{user_id}")
|
|
async def remove_user(user_id: str):
|
|
"""删除用户"""
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
|
|
if not os.path.exists(users_file):
|
|
raise HTTPException(status_code=404, detail="用户文件不存在")
|
|
|
|
# 读取所有用户
|
|
with open(users_file, 'r', encoding='utf-8') as f:
|
|
users = [line.strip() for line in f if line.strip()]
|
|
|
|
# 检查用户是否存在
|
|
if user_id not in users:
|
|
raise HTTPException(status_code=404, detail="用户不存在")
|
|
|
|
# 删除用户
|
|
users.remove(user_id)
|
|
|
|
# 写回文件
|
|
with open(users_file, 'w', encoding='utf-8') as f:
|
|
for user in users:
|
|
f.write(f"{user}\n")
|
|
|
|
# 删除采集数据文件
|
|
data_file = os.path.join(config.DATA_DIR, f"{user_id}.json")
|
|
if os.path.exists(data_file):
|
|
os.remove(data_file)
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"用户 {user_id} 已删除"
|
|
}
|
|
|
|
|
|
@app.delete("/api/users")
|
|
async def clear_all_users():
|
|
"""清空所有用户"""
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
|
|
if not os.path.exists(users_file):
|
|
raise HTTPException(status_code=404, detail="用户文件不存在")
|
|
|
|
# 清空文件
|
|
with open(users_file, 'w', encoding='utf-8') as f:
|
|
f.write("")
|
|
|
|
# 删除所有采集数据
|
|
data_dir = config.DATA_DIR
|
|
for filename in os.listdir(data_dir):
|
|
if filename.endswith('.json') and filename != 'auth_state.json':
|
|
file_path = os.path.join(data_dir, filename)
|
|
os.remove(file_path)
|
|
|
|
# 重置状态
|
|
app_state.users_status = {}
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "所有用户已清空"
|
|
}
|
|
|
|
|
|
@app.post("/api/users/import/txt")
|
|
async def import_users_from_txt():
|
|
"""从users.txt导入用户（模拟）"""
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
|
|
if not os.path.exists(users_file):
|
|
raise HTTPException(status_code=404, detail="users.txt文件不存在")
|
|
|
|
with open(users_file, 'r', encoding='utf-8') as f:
|
|
user_ids = [line.strip() for line in f if line.strip()]
|
|
|
|
return {
|
|
"success": True,
|
|
"count": len(user_ids),
|
|
"users": user_ids
|
|
}
|
|
|
|
|
|
@app.get("/api/users/export/txt")
|
|
async def export_users_to_txt():
|
|
"""导出用户列表到txt文件"""
|
|
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
|
|
|
if not os.path.exists(users_file):
|
|
raise HTTPException(status_code=404, detail="用户文件不存在")
|
|
|
|
with open(users_file, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
return StreamingResponse(
|
|
iter([content]),
|
|
media_type="text/plain",
|
|
headers={
|
|
"Content-Disposition": f"attachment; filename=users_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
|
|
}
|
|
)
|
|
|
|
|
|
# ========== 登录相关 API ==========
|
|
|
|
class LoginStatusResponse(BaseModel):
|
|
logged_in: bool
|
|
login_time: Optional[str] = None
|
|
expire_time: Optional[str] = None
|
|
|
|
|
|
@app.get("/api/login/status", response_model=LoginStatusResponse)
|
|
async def get_login_status():
|
|
"""获取登录状态"""
|
|
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
|
|
|
|
if not os.path.exists(auth_file):
|
|
return LoginStatusResponse(logged_in=False)
|
|
|
|
try:
|
|
with open(auth_file, 'r', encoding='utf-8') as f:
|
|
auth_data = json.load(f)
|
|
|
|
# 检查是否过期
|
|
login_time = auth_data.get("login_time")
|
|
expire_time = auth_data.get("expire_time")
|
|
current_time = datetime.now().isoformat()
|
|
|
|
if expire_time and current_time > expire_time:
|
|
# 登录已过期，清除状态
|
|
os.remove(auth_file)
|
|
return LoginStatusResponse(logged_in=False)
|
|
|
|
return LoginStatusResponse(
|
|
logged_in=auth_data.get("logged_in", False),
|
|
login_time=login_time,
|
|
expire_time=expire_time
|
|
)
|
|
except Exception:
|
|
return LoginStatusResponse(logged_in=False)
|
|
|
|
|
|
@app.post("/api/login/qrcode")
|
|
async def generate_qrcode():
|
|
"""重定向到小红书官方登录页面"""
|
|
# 返回小红书登录页面 URL
|
|
login_url = "https://www.xiaohongshu.com/login"
|
|
|
|
return {
|
|
"login_url": login_url,
|
|
"message": "请在打开的小红书页面完成扫码登录",
|
|
"instruction": "登录成功后，返回此页面点击'验证登录'按钮"
|
|
}
|
|
|
|
|
|
@app.get("/api/check-login-status")
|
|
async def check_login_status():
|
|
"""检查小红书登录状态（直接验证）"""
|
|
try:
|
|
print("[DEBUG] === 检查浏览器状态 ===")
|
|
print(f"[DEBUG] 浏览器实例存在: {app_state.browser is not None}")
|
|
|
|
# 如果浏览器可用，直接使用浏览器验证
|
|
if app_state.browser:
|
|
print("[DEBUG] 浏览器可用，尝试获取context...")
|
|
if app_state.browser.contexts:
|
|
print(f"[DEBUG] 已有 {len(app_state.browser.contexts)} 个contexts")
|
|
context = app_state.browser.contexts[0]
|
|
print("[DEBUG] 使用现有context")
|
|
else:
|
|
print("[DEBUG] 创建新context")
|
|
context = await app_state.browser.new_context()
|
|
|
|
try:
|
|
page = await context.new_page()
|
|
await page.goto("https://www.xiaohongshu.com/", timeout=10000)
|
|
await asyncio.sleep(2)
|
|
|
|
# 获取页面cookies
|
|
cookies = await context.cookies()
|
|
print(f"[DEBUG] 找到 {len(cookies)} 个cookies")
|
|
|
|
# 打印所有cookies
|
|
for cookie in cookies:
|
|
cookie_name = cookie.get('name', '')
|
|
cookie_value = cookie.get('value', '')[:20] if cookie.get('value') else ''
|
|
print(f"[DEBUG] Cookie: {cookie_name} = {cookie_value}...")
|
|
|
|
# 检查是否有小红书相关的session cookie
|
|
session_cookies = []
|
|
for cookie in cookies:
|
|
cookie_name = cookie.get('name', '').lower()
|
|
if any(keyword in cookie_name for keyword in ['session', 'token', 'login', 'auth']):
|
|
session_cookies.append(cookie)
|
|
print(f"[DEBUG] ✓ 找到登录cookie: {cookie_name}")
|
|
|
|
if session_cookies:
|
|
return {
|
|
"success": True,
|
|
"message": "已登录小红书",
|
|
"logged_in": True,
|
|
"cookies_count": len(session_cookies)
|
|
}
|
|
else:
|
|
print("[DEBUG] ✗ 未找到登录相关的 cookies")
|
|
return {
|
|
"success": False,
|
|
"message": "未找到登录状态，请先登录"
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"[DEBUG] 浏览器页面操作失败: {str(e)}")
|
|
pass
|
|
finally:
|
|
# 确保页面关闭
|
|
if 'page' in locals():
|
|
await page.close()
|
|
else:
|
|
print("[DEBUG] ✗ 浏览器实例不存在")
|
|
|
|
# 检查本地是否有登录状态文件
|
|
print("[DEBUG] 检查本地登录状态文件...")
|
|
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
|
|
if os.path.exists(auth_file):
|
|
try:
|
|
with open(auth_file, 'r', encoding='utf-8') as f:
|
|
auth_data = json.load(f)
|
|
print(f"[DEBUG] ✓ 找到本地auth_state: {auth_data}")
|
|
return {
|
|
"success": True,
|
|
"message": "本地登录状态有效",
|
|
"logged_in": True,
|
|
"login_time": auth_data.get("login_time"),
|
|
"expire_time": auth_data.get("expire_time")
|
|
}
|
|
except Exception as e:
|
|
print(f"[DEBUG] ✗ 读取auth_state失败: {str(e)}")
|
|
|
|
print("[DEBUG] ✗ 未找到任何登录状态")
|
|
return {
|
|
"success": False,
|
|
"message": "未找到登录状态，请先登录"
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"[DEBUG] ✗ 检查登录状态异常: {str(e)}")
|
|
return {
|
|
"success": False,
|
|
"message": f"检查登录状态失败: {str(e)}"
|
|
}
|
|
|
|
|
|
@app.post("/api/login/verify")
|
|
async def verify_login():
|
|
"""验证登录状态"""
|
|
try:
|
|
print("[DEBUG] 开始验证登录状态...")
|
|
|
|
# 首先尝试保存 storage_state
|
|
if app_state.browser and app_state.browser.context:
|
|
try:
|
|
# 创建 storage_state 文件
|
|
storage_file = os.path.join(config.DATA_DIR, "storage_state.json")
|
|
print(f"[DEBUG] 尝试保存 storage_state 到: {storage_file}")
|
|
await app_state.browser.context.storage_state(path=storage_file)
|
|
print(f"[DEBUG] ✓ 成功保存 storage_state: {storage_file}")
|
|
|
|
# 检查文件是否存在和大小
|
|
if os.path.exists(storage_file):
|
|
size = os.path.getsize(storage_file)
|
|
print(f"[DEBUG] storage_state 文件大小: {size} bytes")
|
|
if size > 0:
|
|
# 使用 storage_state 验证登录
|
|
print("[DEBUG] 使用 storage_state 验证登录...")
|
|
return await verify_login_with_storage(storage_file)
|
|
else:
|
|
print("[DEBUG] storage_state 文件为空")
|
|
else:
|
|
print("[DEBUG] storage_state 文件未创建")
|
|
|
|
except Exception as e:
|
|
print(f"[DEBUG] ✗ 保存 storage_state 失败: {str(e)}")
|
|
pass
|
|
|
|
# 如果保存失败，尝试直接检查 cookies
|
|
print("[DEBUG] 尝试直接检查 cookies...")
|
|
if app_state.browser:
|
|
try:
|
|
cookies = await app_state.browser.context.cookies()
|
|
print(f"[DEBUG] 找到 {len(cookies)} 个cookies")
|
|
|
|
# 打印所有 cookies 用于调试
|
|
for cookie in cookies:
|
|
cookie_name = cookie.get('name', '')
|
|
cookie_value = cookie.get('value', '')[:20] if cookie.get('value') else ''
|
|
print(f"[DEBUG] Cookie: {cookie_name} = {cookie_value}...")
|
|
|
|
# 检查是否有登录相关的 cookie
|
|
session_cookies = []
|
|
for cookie in cookies:
|
|
cookie_name = cookie.get('name', '').lower()
|
|
if any(keyword in cookie_name for keyword in ['session', 'token', 'auth', 'passport', 'sid']):
|
|
session_cookies.append(cookie)
|
|
print(f"[DEBUG] ✓ 找到登录cookie: {cookie_name}")
|
|
|
|
# 创建 auth_state 文件
|
|
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
|
|
auth_data = {
|
|
"login_time": datetime.now().isoformat(),
|
|
"expire_time": (datetime.now() + timedelta(hours=24)).isoformat()
|
|
}
|
|
with open(auth_file, 'w', encoding='utf-8') as f:
|
|
json.dump(auth_data, f, ensure_ascii=False, indent=2)
|
|
|
|
print("[DEBUG] ✓ 创建 auth_state 文件成功")
|
|
return {
|
|
"success": True,
|
|
"message": "登录成功",
|
|
"logged_in": True,
|
|
"login_time": auth_data["login_time"],
|
|
"expire_time": auth_data["expire_time"],
|
|
"expire_in_hours": 24
|
|
}
|
|
|
|
if not session_cookies:
|
|
print("[DEBUG] ✗ 未找到登录相关的 cookies")
|
|
|
|
except Exception as e:
|
|
print(f"[DEBUG] ✗ 检查cookies失败: {str(e)}")
|
|
pass
|
|
|
|
# 检查本地登录状态文件
|
|
print("[DEBUG] 检查本地登录状态文件...")
|
|
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
|
|
if os.path.exists(auth_file):
|
|
try:
|
|
with open(auth_file, 'r', encoding='utf-8') as f:
|
|
auth_data = json.load(f)
|
|
print(f"[DEBUG] ✓ 找到本地 auth_state 文件: {auth_data}")
|
|
|
|
# 检查登录是否过期
|
|
login_time = auth_data.get("login_time")
|
|
expire_time = auth_data.get("expire_time")
|
|
current_time = datetime.now().isoformat()
|
|
|
|
if expire_time and current_time > expire_time:
|
|
# 登录已过期，清除状态
|
|
print("[DEBUG] ✗ 登录已过期")
|
|
os.remove(auth_file)
|
|
return {
|
|
"success": False,
|
|
"message": "登录已过期，请重新登录"
|
|
}
|
|
|
|
print("[DEBUG] ✓ 本地登录状态有效")
|
|
return {
|
|
"success": True,
|
|
"message": "登录成功",
|
|
"logged_in": True,
|
|
"login_time": login_time,
|
|
"expire_time": expire_time,
|
|
"expire_in_hours": 24
|
|
}
|
|
except Exception as e:
|
|
print(f"[DEBUG] ✗ 读取 auth_state 文件失败: {str(e)}")
|
|
|
|
print("[DEBUG] ✗ 未找到任何登录状态")
|
|
return {
|
|
"success": False,
|
|
"message": "未找到登录状态，请先登录"
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"[DEBUG] ✗ 验证登录异常: {str(e)}")
|
|
return {
|
|
"success": False,
|
|
"message": f"验证失败: {str(e)}"
|
|
}
|
|
|
|
|
|
async def verify_login_with_storage(storage_file: str):
|
|
"""使用 storage_state 文件验证登录状态"""
|
|
try:
|
|
from pathlib import Path
|
|
from playwright.async_api import async_playwright
|
|
|
|
pth = Path(storage_file)
|
|
if not pth.exists():
|
|
return {
|
|
"success": False,
|
|
"message": "未找到登录状态文件"
|
|
}
|
|
|
|
# 使用 async_playwright 创建临时浏览器实例验证
|
|
async with async_playwright() as p:
|
|
# 使用与主浏览器相同的配置
|
|
browser = await p.chromium.launch(
|
|
headless=True, # 无头模式验证
|
|
args=["--no-sandbox", "--disable-setuid-sandbox"]
|
|
)
|
|
|
|
# 使用 storage_state 创建新 context
|
|
context = await browser.new_context(
|
|
storage_state=str(pth),
|
|
locale="zh-CN",
|
|
viewport={"width": 1200, "height": 800}
|
|
)
|
|
|
|
page = await context.new_page()
|
|
|
|
# 访问小红书主页
|
|
await page.goto("https://www.xiaohongshu.com", wait_until="domcontentloaded", timeout=60000)
|
|
await asyncio.sleep(1.2) # 等待页面加载
|
|
|
|
# 强拦截关键词 - 使用更可靠的元素检查方式
|
|
block_texts = ["扫码登录", "安全验证", "请先登录", "登录后查看"]
|
|
for text in block_texts:
|
|
try:
|
|
if await page.get_by_text(text).first.is_visible():
|
|
await browser.close()
|
|
return {
|
|
"success": False,
|
|
"message": f"登录状态无效或被拦截：页面出现「{text}」"
|
|
}
|
|
except Exception:
|
|
pass
|
|
|
|
# 弱判断：若明显出现"登录"且可见，可能未登录
|
|
try:
|
|
if await page.get_by_text("登录").first.is_visible():
|
|
await browser.close()
|
|
return {
|
|
"success": False,
|
|
"message": "疑似未登录（页面出现明显「登录」入口），建议重新登录保存状态"
|
|
}
|
|
except Exception:
|
|
pass
|
|
|
|
# 检查 URL 是否被重定向
|
|
current_url = page.url
|
|
if "login" in current_url.lower() or "404" in current_url:
|
|
await browser.close()
|
|
return {
|
|
"success": False,
|
|
"message": "被重定向到登录页面，登录状态无效"
|
|
}
|
|
|
|
# 检查页面标题
|
|
title = await page.title()
|
|
if "登录" in title or "404" in title:
|
|
await browser.close()
|
|
return {
|
|
"success": False,
|
|
"message": "页面标题显示登录或错误，登录状态无效"
|
|
}
|
|
|
|
# 所有检查都通过，认为登录有效
|
|
await browser.close()
|
|
|
|
# 创建 auth_state 文件作为备份
|
|
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
|
|
auth_data = {
|
|
"login_time": datetime.now().isoformat(),
|
|
"expire_time": (datetime.now() + timedelta(hours=24)).isoformat()
|
|
}
|
|
with open(auth_file, 'w', encoding='utf-8') as f:
|
|
json.dump(auth_data, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"[DEBUG] 登录验证成功")
|
|
return {
|
|
"success": True,
|
|
"message": "登录成功",
|
|
"logged_in": True,
|
|
"login_time": auth_data["login_time"],
|
|
"expire_time": auth_data["expire_time"],
|
|
"expire_in_hours": 24
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"[DEBUG] storage_state 验证失败: {str(e)}")
|
|
return {
|
|
"success": False,
|
|
"message": f"登录验证失败: {str(e)}"
|
|
}
|
|
|
|
|
|
@app.post("/api/logout")
|
|
async def logout():
|
|
"""退出登录"""
|
|
auth_file = os.path.join(config.DATA_DIR, "auth_state.json")
|
|
|
|
try:
|
|
if os.path.exists(auth_file):
|
|
os.remove(auth_file)
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "已退出登录"
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"退出登录失败: {str(e)}")
|
|
|
|
|
|
@app.get("/api/data/{user_id}")
|
|
async def get_user_data(user_id: str):
|
|
"""获取用户采集数据"""
|
|
file_path = os.path.join(config.DATA_DIR, f"{user_id}.json")
|
|
|
|
if not os.path.exists(file_path):
|
|
raise HTTPException(status_code=404, detail="用户数据不存在")
|
|
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
|
|
return data
|
|
|
|
|
|
@app.get("/api/events")
|
|
async def event_stream():
|
|
"""SSE 事件流"""
|
|
async def event_generator():
|
|
while True:
|
|
try:
|
|
# 等待事件，超时 1 秒发送心跳
|
|
event = await asyncio.wait_for(
|
|
app_state.event_queue.get(),
|
|
timeout=1.0
|
|
)
|
|
|
|
# 更新状态
|
|
if event["type"] == "log":
|
|
# 日志不需要更新状态
|
|
pass
|
|
elif event["type"] == "progress":
|
|
app_state.completed_users = event.get("completed", app_state.completed_users)
|
|
app_state.failed_users = event.get("failed", app_state.failed_users)
|
|
app_state.active_threads = event.get("active_threads", app_state.active_threads)
|
|
elif event["type"] == "user_status":
|
|
if event["user_id"] in app_state.users_status:
|
|
app_state.users_status[event["user_id"]]["status"] = event["status"]
|
|
app_state.users_status[event["user_id"]]["note_count"] = event.get("note_count", 0)
|
|
app_state.users_status[event["user_id"]]["thread_id"] = event.get("thread_id")
|
|
elif event["type"] == "thread_status":
|
|
app_state.thread_status[event["thread_id"]] = {
|
|
"status": event["status"],
|
|
"current_user": event.get("current_user")
|
|
}
|
|
# 更新活跃线程数
|
|
app_state.active_threads = sum(
|
|
1 for s in app_state.thread_status.values() if s.get("status") == "busy"
|
|
)
|
|
|
|
# 发送事件
|
|
yield f"data: {json.dumps(event)}\n\n"
|
|
|
|
except asyncio.TimeoutError:
|
|
# 发送心跳
|
|
yield ": heartbeat\n\n"
|
|
|
|
except Exception as e:
|
|
# 错误事件
|
|
yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
|
|
|
|
return StreamingResponse(
|
|
event_generator(),
|
|
media_type="text/event-stream",
|
|
headers={
|
|
"Cache-Control": "no-cache",
|
|
"Connection": "keep-alive",
|
|
"Access-Control-Allow-Origin": "*"
|
|
}
|
|
)
|
|
|
|
|
|
# ========== Git 相关 API ==========
|
|
|
|
@app.get("/api/git/status")
|
|
async def git_status():
|
|
"""获取 Git 状态"""
|
|
if not os.path.exists("/app/.git"):
|
|
return {
|
|
"enabled": False,
|
|
"message": "不是 Git 仓库"
|
|
}
|
|
|
|
try:
|
|
# 获取当前分支
|
|
result = subprocess.run(
|
|
["git", "branch", "--show-current"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
current_branch = result.stdout.strip()
|
|
|
|
# 获取最新提交
|
|
result = subprocess.run(
|
|
["git", "log", "-1", "--format=%h %s %cr"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
latest_commit = result.stdout.strip()
|
|
|
|
# 检查是否有未提交的更改
|
|
result = subprocess.run(
|
|
["git", "status", "--porcelain"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
has_changes = len(result.stdout.strip()) > 0
|
|
|
|
return {
|
|
"enabled": True,
|
|
"branch": current_branch,
|
|
"latest_commit": latest_commit,
|
|
"has_changes": has_changes,
|
|
"auto_pull_enabled": config.ENABLE_GIT_PULL,
|
|
"auto_pull_interval": config.AUTO_PULL_INTERVAL
|
|
}
|
|
|
|
except Exception as e:
|
|
return {
|
|
"enabled": False,
|
|
"error": str(e)
|
|
}
|
|
|
|
|
|
@app.post("/api/git/pull")
|
|
async def git_pull():
|
|
"""手动拉取最新代码"""
|
|
if not os.path.exists("/app/.git"):
|
|
raise HTTPException(status_code=400, detail="不是 Git 仓库")
|
|
|
|
try:
|
|
# 获取当前分支
|
|
result = subprocess.run(
|
|
["git", "branch", "--show-current"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
current_branch = result.stdout.strip()
|
|
|
|
# 拉取最新代码
|
|
result = subprocess.run(
|
|
["git", "fetch", "origin", current_branch],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
raise HTTPException(status_code=500, detail=f"拉取失败: {result.stderr}")
|
|
|
|
# 检查是否有更新
|
|
result = subprocess.run(
|
|
["git", "rev-list", "HEAD..origin/" + current_branch, "--count"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
commits_behind = int(result.stdout.strip())
|
|
|
|
if commits_behind == 0:
|
|
return {
|
|
"success": True,
|
|
"message": "已是最新代码",
|
|
"needs_restart": False
|
|
}
|
|
|
|
# 重置到最新代码
|
|
result = subprocess.run(
|
|
["git", "reset", "--hard", "origin/" + current_branch],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
raise HTTPException(status_code=500, detail=f"重置失败: {result.stderr}")
|
|
|
|
# 检查 Python 文件是否有变化
|
|
result = subprocess.run(
|
|
["git", "diff", "--name-only", "HEAD@{1}", "HEAD"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/app"
|
|
)
|
|
changed_files = result.stdout.strip().split('\n')
|
|
python_changed = any(f.endswith('.py') for f in changed_files if f)
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"已拉取 {commits_behind} 个新提交",
|
|
"changed_files": changed_files,
|
|
"needs_restart": python_changed,
|
|
"restart_hint": "Python 代码有变化，建议重启容器以应用更新"
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"拉取失败: {str(e)}")
|
|
|
|
|
|
@app.post("/api/git/restart")
|
|
async def restart_container():
|
|
"""
|
|
请求重启容器
|
|
注意：这需要在容器外配置重启策略
|
|
"""
|
|
return {
|
|
"success": True,
|
|
"message": "重启请求已接收",
|
|
"methods": [
|
|
"docker restart xiaohongshu-scraper",
|
|
"docker-compose restart"
|
|
]
|
|
}
|
|
|
|
|
|
# ========== 启动事件 ==========
|
|
@app.on_event("startup")
|
|
async def startup_event():
|
|
"""启动时初始化"""
|
|
# 确保数据目录存在
|
|
os.makedirs(config.DATA_DIR, exist_ok=True)
|
|
os.makedirs(config.SCREENSHOT_DIR, exist_ok=True)
|
|
|
|
# 初始化事件队列
|
|
app_state.event_queue = asyncio.Queue()
|
|
|
|
# 初始化浏览器
|
|
try:
|
|
from playwright.async_api import async_playwright
|
|
app_state.playwright = await async_playwright().start()
|
|
app_state.browser = await app_state.playwright.chromium.launch(
|
|
headless=config.HEADLESS,
|
|
args=config.BROWSER_ARGS
|
|
)
|
|
print("[INFO] 浏览器初始化完成")
|
|
except Exception as e:
|
|
print(f"[ERROR] 浏览器初始化失败: {str(e)}")
|
|
|
|
print("=" * 60)
|
|
print("小红书数据采集器 Web 服务启动")
|
|
print(f"访问地址: http://{config.WEB_HOST}:{config.WEB_PORT}")
|
|
print("=" * 60)
|
|
|
|
|
|
@app.on_event("shutdown")
|
|
async def shutdown_event():
|
|
"""关闭时清理"""
|
|
if app_state.scraper:
|
|
app_state.scraper.stop()
|
|
|
|
if app_state.scraper_task:
|
|
app_state.scraper_task.cancel()
|
|
|
|
# 关闭浏览器
|
|
if app_state.browser:
|
|
await app_state.browser.close()
|
|
if app_state.playwright:
|
|
await app_state.playwright.stop()
|
|
print("[INFO] 浏览器已关闭")
|
|
|
|
print("Web 服务已关闭")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(
|
|
"app.web:app",
|
|
host=config.WEB_HOST,
|
|
port=config.WEB_PORT,
|
|
reload=True
|
|
)
|