first commit
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# 虚拟环境
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# 数据文件（运行时生成）
|
||||
data/*.json
|
||||
data/logs.json
|
||||
data/auth_state.json
|
||||
|
||||
# Git
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Docker
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
.dockerignore
|
||||
|
||||
# 文档
|
||||
README.md
|
||||
*.md
|
||||
|
||||
# 环境变量
|
||||
.env
|
||||
|
||||
# 日志
|
||||
*.log
|
||||
@@ -0,0 +1,21 @@
|
||||
# 环境变量配置示例
|
||||
# 复制此文件为 .env 并根据需要修改
|
||||
|
||||
# Web 服务配置
|
||||
HOST=0.0.0.0
|
||||
PORT=18080
|
||||
|
||||
# 浏览器配置
|
||||
HEADLESS=false
|
||||
SCREENSHOT_ON_ERROR=true
|
||||
|
||||
# 采集配置
|
||||
MAX_THREADS=5
|
||||
|
||||
# Git 自动更新配置（可选）
|
||||
# 启用后，容器会自动从 Git 仓库拉取最新代码
|
||||
# 注意：Python 代码变化后需要重启容器才能生效
|
||||
ENABLE_GIT_PULL=false
|
||||
GIT_REPO_URL=http://219.157.255.213:25311/xinyouhe/xhs_crawler_claude.git
|
||||
GIT_BRANCH=master
|
||||
AUTO_PULL_INTERVAL=3600
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
FROM python:3.10-slim
|
||||
|
||||
# 安装系统依赖（包含 Git）
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
gnupg \
|
||||
ca-certificates \
|
||||
fonts-liberation \
|
||||
libasound2 \
|
||||
libatk-bridge2.0-0 \
|
||||
libatk1.0-0 \
|
||||
libatspi2.0-0 \
|
||||
libcups2 \
|
||||
libdbus-1-3 \
|
||||
libdrm2 \
|
||||
libgbm1 \
|
||||
libgtk-3-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libwayland-client0 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxfixes3 \
|
||||
libxkbcommon0 \
|
||||
libxrandr2 \
|
||||
xdg-utils \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 复制依赖文件
|
||||
COPY requirements.txt .
|
||||
|
||||
# 安装 Python 依赖
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# 安装 Playwright 浏览器
|
||||
RUN playwright install chromium
|
||||
RUN playwright install-deps chromium
|
||||
|
||||
# 复制应用代码
|
||||
COPY . .
|
||||
|
||||
# 创建数据目录
|
||||
RUN mkdir -p /app/data /app/data/screenshots
|
||||
|
||||
# 暴露端口
|
||||
EXPOSE 18080
|
||||
|
||||
# 设置环境变量
|
||||
ENV HOST=0.0.0.0
|
||||
ENV PORT=18080
|
||||
ENV HEADLESS=false
|
||||
ENV SCREENSHOT_ON_ERROR=true
|
||||
|
||||
# 启动脚本（支持 Git 自动更新）
|
||||
COPY entrypoint.sh /app/entrypoint.sh
|
||||
RUN chmod +x /app/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
@@ -0,0 +1,296 @@
|
||||
# 小红书数据采集器
|
||||
|
||||
一个带 Web 可视化监控界面的小红书批量数据采集器，支持多账号增量采集、实时进度展示、持久化日志追踪、单/多线程切换。
|
||||
|
||||
## 功能特性
|
||||
|
||||
- **Web 可视化界面**: 实时监控采集进度和日志
|
||||
- **单/多线程模式**: 支持切换单线程或多线程采集，可配置线程数
|
||||
- **增量采集**: 自动识别已采集内容，仅采集新增笔记
|
||||
- **持久化日志**: 日志保存到文件，重启后可查看历史记录
|
||||
- **风控策略**: 拟人化延迟、平滑滚动、风控检测
|
||||
- **云部署友好**: 支持 Docker 部署
|
||||
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
demo/
|
||||
├── app/
|
||||
│ ├── __init__.py
|
||||
│ ├── config.py # 配置文件
|
||||
│ ├── log_manager.py # 日志管理器
|
||||
│ ├── scraper.py # 核心采集逻辑
|
||||
│ └── web.py # Web API
|
||||
├── data/
|
||||
│ ├── users.txt # 用户 ID 输入文件
|
||||
│ ├── logs.json # 持久化日志
|
||||
│ └── {user_id}.json # 采集数据
|
||||
├── templates/
|
||||
│ └── index.html # 前端页面
|
||||
├── static/
|
||||
│ ├── style.css # 样式
|
||||
│ └── app.js # 前端逻辑
|
||||
├── main.py # 程序入口
|
||||
├── requirements.txt # 依赖列表
|
||||
└── README.md
|
||||
```
|
||||
|
||||
## 安装
|
||||
|
||||
### 1. 克隆项目
|
||||
|
||||
```bash
|
||||
cd demo
|
||||
```
|
||||
|
||||
### 2. 安装依赖
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### 3. 安装 Playwright 浏览器
|
||||
|
||||
```bash
|
||||
playwright install chromium
|
||||
```
|
||||
|
||||
## 使用
|
||||
|
||||
### 1. 启动服务
|
||||
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
服务将在 `http://localhost:18080` 启动。
|
||||
|
||||
### 2. 添加用户 ID
|
||||
|
||||
在 `data/users.txt` 中添加要采集的用户 ID，每行一个：
|
||||
|
||||
```
|
||||
5ef74296000000000100003a
|
||||
5b6c1f9e0000000001000388
|
||||
```
|
||||
|
||||
### 3. 配置采集模式
|
||||
|
||||
在 Web 界面中选择：
|
||||
- **单线程模式**: 一次采集一个用户，更安全
|
||||
- **多线程模式**: 同时采集多个用户，更快
|
||||
|
||||
多线程模式下可设置线程数（1-10）。
|
||||
|
||||
### 4. 启动采集
|
||||
|
||||
点击「启动采集」按钮，在弹出的浏览器窗口中扫码登录。
|
||||
|
||||
登录成功后，采集器会自动开始工作，你可以在 Web 界面实时查看：
|
||||
- 整体进度
|
||||
- 用户状态
|
||||
- 线程监控（多线程模式）
|
||||
- 实时日志
|
||||
|
||||
## 配置
|
||||
|
||||
### 环境变量
|
||||
|
||||
创建 `.env` 文件配置以下参数：
|
||||
|
||||
```env
|
||||
HOST=0.0.0.0
|
||||
PORT=18080
|
||||
HEADLESS=false
|
||||
SCREENSHOT_ON_ERROR=true
|
||||
MAX_THREADS=5
|
||||
```
|
||||
|
||||
### 采集参数
|
||||
|
||||
修改 `app/config.py` 中的参数：
|
||||
|
||||
```python
|
||||
# 延迟配置（秒）
|
||||
PAGE_LOAD_DELAY = (2, 5) # 页面加载后等待
|
||||
NOTE_READ_DELAY = (3, 8) # 笔记详情页停留
|
||||
NOTE_INTERVAL_DELAY = (3, 6) # 采集笔记间隔
|
||||
USER_SWITCH_DELAY = (30, 60) # 切换用户间隔
|
||||
|
||||
# 多线程配置
|
||||
THREAD_MIN_DELAY = 2 # 线程间最小间隔
|
||||
THREAD_OFFSET = 5 # 线程启动偏移
|
||||
```
|
||||
|
||||
## 数据格式
|
||||
|
||||
### 输入文件 (data/users.txt)
|
||||
|
||||
```
|
||||
5ef74296000000000100003a
|
||||
5b6c1f9e0000000001000388
|
||||
```
|
||||
|
||||
### 输出文件 (data/{user_id}.json)
|
||||
|
||||
```json
|
||||
{
|
||||
"user_id": "5ef74296000000000100003a",
|
||||
"last_updated": "2025-01-12T10:30:00",
|
||||
"total_notes": 12,
|
||||
"notes": [
|
||||
{
|
||||
"title": "笔记标题",
|
||||
"publish_time": "2025-01-10 14:30",
|
||||
"location": "上海",
|
||||
"comment_count": 128,
|
||||
"like_count": 5234,
|
||||
"collect_count": 892,
|
||||
"note_url": "https://www.xiaohongshu.com/explore/..."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Docker 部署
|
||||
|
||||
### 方式一：使用 Docker Compose（推荐）
|
||||
|
||||
```bash
|
||||
# 构建并启动
|
||||
docker-compose up -d
|
||||
|
||||
# 查看日志
|
||||
docker-compose logs -f
|
||||
|
||||
# 停止服务
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
### 方式二：使用 Docker 命令
|
||||
|
||||
#### 构建镜像
|
||||
|
||||
```bash
|
||||
docker build -t xiaohongshu-scraper .
|
||||
```
|
||||
|
||||
#### 运行容器
|
||||
|
||||
```bash
|
||||
# 本地开发模式（可以看到浏览器窗口）
|
||||
docker run -d \
|
||||
-p 18080:18080 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
-e HEADLESS=false \
|
||||
--name xiaohongshu-scraper \
|
||||
xiaohongshu-scraper
|
||||
|
||||
# 云服务器模式（无头模式）
|
||||
docker run -d \
|
||||
-p 18080:18080 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
-e HEADLESS=true \
|
||||
--name xiaohongshu-scraper \
|
||||
xiaohongshu-scraper
|
||||
```
|
||||
|
||||
#### 管理容器
|
||||
|
||||
```bash
|
||||
# 查看日志
|
||||
docker logs -f xiaohongshu-scraper
|
||||
|
||||
# 停止容器
|
||||
docker stop xiaohongshu-scraper
|
||||
|
||||
# 启动容器
|
||||
docker start xiaohongshu-scraper
|
||||
|
||||
# 删除容器
|
||||
docker rm xiaohongshu-scraper
|
||||
|
||||
# 进入容器
|
||||
docker exec -it xiaohongshu-scraper bash
|
||||
```
|
||||
|
||||
### Docker 环境变量
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| HOST | 0.0.0.0 | 监听地址 |
|
||||
| PORT | 18080 | 监听端口（范围 10000-50000）|
|
||||
| HEADLESS | false | 是否无头模式（云服务器设为 true）|
|
||||
| SCREENSHOT_ON_ERROR | true | 错误时截图 |
|
||||
| MAX_THREADS | 5 | 最大线程数 |
|
||||
| ENABLE_GIT_PULL | false | 启用 Git 自动更新 |
|
||||
| GIT_REPO_URL | - | Git 仓库地址 |
|
||||
| GIT_BRANCH | main | Git 分支 |
|
||||
| AUTO_PULL_INTERVAL | 3600 | 自动拉取间隔（秒）|
|
||||
|
||||
### 数据持久化
|
||||
|
||||
通过挂载 `./data` 目录到容器的 `/app/data`，确保以下数据持久化：
|
||||
|
||||
- `users.txt` - 用户 ID 列表
|
||||
- `{user_id}.json` - 采集数据
|
||||
- `logs.json` - 运行日志
|
||||
- `auth_state.json` - 登录状态（Cookie）
|
||||
- `screenshots/` - 错误截图
|
||||
|
||||
### Git 自动更新
|
||||
|
||||
支持通过 Git 自动更新代码（适用于云部署）：
|
||||
|
||||
#### 1. 配置 Git 自动更新
|
||||
|
||||
在 `docker-compose.yml` 中添加环境变量：
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- ENABLE_GIT_PULL=true
|
||||
- GIT_REPO_URL=https://github.com/yourusername/xiaohongshu-scraper.git
|
||||
- GIT_BRANCH=main
|
||||
- AUTO_PULL_INTERVAL=3600
|
||||
```
|
||||
|
||||
#### 2. 工作原理
|
||||
|
||||
- 容器启动后，会定期从 Git 仓库拉取最新代码
|
||||
- 如果检测到 Python 文件变化，会提示需要重启容器
|
||||
- 静态文件（HTML/CSS/JS）变化会立即生效，无需重启
|
||||
|
||||
#### 3. 手动更新
|
||||
|
||||
通过 API 手动触发更新：
|
||||
|
||||
```bash
|
||||
# 检查 Git 状态
|
||||
curl http://your-server:18080/api/git/status
|
||||
|
||||
# 拉取最新代码
|
||||
curl -X POST http://your-server:18080/api/git/pull
|
||||
```
|
||||
|
||||
#### 4. 重启容器
|
||||
|
||||
如果 Python 代码有变化，需要重启容器：
|
||||
|
||||
```bash
|
||||
# 使用 Docker Compose
|
||||
docker-compose restart
|
||||
|
||||
# 使用 Docker 命令
|
||||
docker restart xiaohongshu-scraper
|
||||
```
|
||||
|
||||
## 注意事项
|
||||
|
||||
1. **风控风险**: 本工具仅供学习交流使用，请遵守小红书的使用条款
|
||||
2. **采集频率**: 建议使用单线程模式，多线程模式可能增加风控风险
|
||||
3. **登录状态**: 首次登录后会保存 Cookie，后续无需重新登录
|
||||
4. **页面结构**: 小红书页面结构可能变化，如遇采集失败请检查 CSS 选择器
|
||||
|
||||
## 许可证
|
||||
|
||||
MIT License
|
||||
@@ -0,0 +1,2 @@
|
||||
# XiaoHongShu Scraper Package
|
||||
__version__ = "1.0.0"
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
XiaoHongShu Scraper Configuration
|
||||
采集器配置文件
|
||||
"""
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# ========== 基础配置 ==========
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
SCREENSHOT_DIR = os.path.join(DATA_DIR, "screenshots")
|
||||
|
||||
# 确保目录存在
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
os.makedirs(SCREENSHOT_DIR, exist_ok=True)
|
||||
|
||||
|
||||
# ========== 采集字段配置 ==========
|
||||
FIELDS = ['title', 'publish_time', 'location', 'comment_count', 'like_count', 'collect_count', 'note_url']
|
||||
|
||||
|
||||
# ========== 延迟配置（秒）==========
|
||||
PAGE_LOAD_DELAY = (2, 5) # 页面加载后等待
|
||||
NOTE_READ_DELAY = (3, 8) # 笔记详情页停留（模拟阅读）
|
||||
NOTE_INTERVAL_DELAY = (3, 6) # 采集完一个笔记后的间隔
|
||||
USER_SWITCH_DELAY = (30, 60) # 切换用户时的间隔
|
||||
|
||||
|
||||
# ========== 多线程配置 ==========
|
||||
THREAD_MIN_DELAY = 2 # 线程间最小间隔（秒）
|
||||
THREAD_OFFSET = 5 # 线程启动偏移（秒）
|
||||
MAX_CONCURRENT_THREADS = 10 # 最大线程数
|
||||
|
||||
|
||||
# ========== 滚动配置 ==========
|
||||
SCROLL_STEP_MIN = 100 # 每次滚动最小像素
|
||||
SCROLL_STEP_MAX = 300 # 每次滚动最大像素
|
||||
SCROLL_PAUSE_MIN = 0.5 # 滚动后暂停最小时间（秒）
|
||||
SCROLL_PAUSE_MAX = 2.0 # 滚动后暂停最大时间（秒）
|
||||
MAX_SCROLL_COUNT = 50 # 最大滚动次数（防止无限滚动）
|
||||
|
||||
|
||||
# ========== 风控检测关键词 ==========
|
||||
RISK_KEYWORDS = [
|
||||
'验证码',
|
||||
'访问过于频繁',
|
||||
'页面不存在',
|
||||
'验证',
|
||||
'流量异常',
|
||||
'安全验证',
|
||||
'人机验证',
|
||||
'滑块验证'
|
||||
]
|
||||
|
||||
|
||||
# ========== 小红书 URL 配置 ==========
|
||||
XIAOHONGSHU_BASE_URL = "https://www.xiaohongshu.com"
|
||||
USER_PROFILE_URL_TEMPLATE = f"{XIAOHONGSHU_BASE_URL}/user/profile/{{user_id}}"
|
||||
|
||||
|
||||
# ========== 浏览器配置 ==========
|
||||
HEADLESS = os.getenv("HEADLESS", "false").lower() == "true"
|
||||
BROWSER_ARGS = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled'
|
||||
]
|
||||
|
||||
# 视口配置
|
||||
VIEWPORT_WIDTH = 1920
|
||||
VIEWPORT_HEIGHT = 1080
|
||||
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
|
||||
|
||||
# ========== 日志配置 ==========
|
||||
LOG_FILE = os.path.join(DATA_DIR, "logs.json")
|
||||
MAX_LOGS = 10000 # 最大日志条数
|
||||
|
||||
|
||||
# ========== Web 配置 ==========
|
||||
WEB_HOST = os.getenv("HOST", "0.0.0.0")
|
||||
WEB_PORT = int(os.getenv("PORT", "18080")) # 默认端口 18080，范围 10000-50000
|
||||
SCREENSHOT_ON_ERROR = os.getenv("SCREENSHOT_ON_ERROR", "true").lower() == "true"
|
||||
|
||||
# ========== Git 更新配置 ==========
|
||||
ENABLE_GIT_PULL = os.getenv("ENABLE_GIT_PULL", "false").lower() == "true"
|
||||
GIT_REPO_URL = os.getenv("GIT_REPO_URL", "")
|
||||
GIT_BRANCH = os.getenv("GIT_BRANCH", "main")
|
||||
AUTO_PULL_INTERVAL = int(os.getenv("AUTO_PULL_INTERVAL", "3600")) # 自动拉取间隔（秒），默认1小时
|
||||
|
||||
|
||||
# ========== CSS 选择器（可能需要根据小红书页面结构调整）==========
|
||||
# 用户主页
|
||||
SELECTOR_USER_CONTAINER = "div.user-notes"
|
||||
SELECTOR_NOTE_CARD = "div[class*='note-item'], section[class*='note']"
|
||||
SELECTOR_NOTE_LINK = "a[href*='/explore/']"
|
||||
SELECTOR_NOTE_TITLE = "span[class*='title'], div[class*='title']"
|
||||
SELECTOR_NOTE_CONTENT = "div[class*='content'], span[class*='desc']"
|
||||
|
||||
# 笔记详情页
|
||||
SELECTOR_DETAIL_TITLE = "div[class*='title'], span[class*='title']"
|
||||
SELECTOR_DETAIL_PUBLISH_TIME = "span[class*='time'], div[class*='time']"
|
||||
SELECTOR_DETAIL_LOCATION = "span[class*='location'], div[class*='location']"
|
||||
SELECTOR_DETAIL_COMMENT_COUNT = "span[class*='comment'], button[class*='comment']"
|
||||
SELECTOR_DETAIL_LIKE_COUNT = "span[class*='like'], button[class*='like']"
|
||||
SELECTOR_DETAIL_COLLECT_COUNT = "span[class*='collect'], button[class*='star']"
|
||||
|
||||
|
||||
# ========== 登录配置 ==========
|
||||
LOGIN_URL = f"{XIAOHONGSHU_BASE_URL}/login"
|
||||
AUTH_STATE_FILE = os.path.join(DATA_DIR, "auth_state.json")
|
||||
LOGIN_TIMEOUT = 300 # 登录等待超时（秒）
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Log Manager - 持久化日志管理器
|
||||
支持日志加载、保存、追加、清空
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Optional
|
||||
from app import config
|
||||
|
||||
|
||||
class LogManager:
|
||||
"""日志管理器 - 线程安全"""
|
||||
|
||||
def __init__(self, log_file: str = None, max_logs: int = None):
|
||||
self.log_file = log_file or config.LOG_FILE
|
||||
self.max_logs = max_logs or config.MAX_LOGS
|
||||
self._lock = threading.Lock()
|
||||
self.logs: List[Dict] = []
|
||||
self._load_logs()
|
||||
|
||||
def _load_logs(self):
|
||||
"""启动时加载历史日志"""
|
||||
with self._lock:
|
||||
if os.path.exists(self.log_file):
|
||||
try:
|
||||
with open(self.log_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
self.logs = data.get('logs', [])
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
print(f"[LogManager] 加载日志文件失败: {e}")
|
||||
self.logs = []
|
||||
else:
|
||||
self.logs = []
|
||||
|
||||
def append(self, level: str, message: str, thread_id: Optional[str] = None) -> Dict:
|
||||
"""
|
||||
追加新日志
|
||||
返回日志条目
|
||||
"""
|
||||
with self._lock:
|
||||
log_entry = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"level": level,
|
||||
"message": message,
|
||||
"thread_id": thread_id
|
||||
}
|
||||
self.logs.append(log_entry)
|
||||
|
||||
# 限制日志数量（保留最新的 max_logs 条）
|
||||
if len(self.logs) > self.max_logs:
|
||||
self.logs = self.logs[-self.max_logs:]
|
||||
|
||||
# 异步保存到文件
|
||||
self._save()
|
||||
|
||||
return log_entry
|
||||
|
||||
def _save(self):
|
||||
"""保存到文件（需在锁内调用）"""
|
||||
try:
|
||||
with open(self.log_file, 'w', encoding='utf-8') as f:
|
||||
json.dump({
|
||||
"logs": self.logs,
|
||||
"last_updated": datetime.now().isoformat()
|
||||
}, f, ensure_ascii=False, indent=2)
|
||||
except IOError as e:
|
||||
print(f"[LogManager] 保存日志文件失败: {e}")
|
||||
|
||||
def get_logs(self, level: Optional[str] = None, limit: Optional[int] = None) -> List[Dict]:
|
||||
"""
|
||||
获取日志
|
||||
:param level: 过滤级别 (None/INFO/SUCCESS/WARNING/ERROR/DEBUG)
|
||||
:param limit: 限制返回数量
|
||||
"""
|
||||
with self._lock:
|
||||
if level:
|
||||
filtered = [log for log in self.logs if log['level'] == level]
|
||||
else:
|
||||
filtered = self.logs
|
||||
|
||||
if limit:
|
||||
return filtered[-limit:]
|
||||
return filtered
|
||||
|
||||
def search_logs(self, keyword: str, limit: Optional[int] = None) -> List[Dict]:
|
||||
"""
|
||||
搜索日志
|
||||
:param keyword: 搜索关键词
|
||||
:param limit: 限制返回数量
|
||||
"""
|
||||
with self._lock:
|
||||
filtered = [
|
||||
log for log in self.logs
|
||||
if keyword.lower() in log['message'].lower()
|
||||
]
|
||||
|
||||
if limit:
|
||||
return filtered[-limit:]
|
||||
return filtered
|
||||
|
||||
def clear(self):
|
||||
"""清空日志"""
|
||||
with self._lock:
|
||||
self.logs = []
|
||||
self._save()
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
"""获取日志统计信息"""
|
||||
with self._lock:
|
||||
stats = {
|
||||
"total": len(self.logs),
|
||||
"by_level": {}
|
||||
}
|
||||
|
||||
for log in self.logs:
|
||||
level = log['level']
|
||||
stats["by_level"][level] = stats["by_level"].get(level, 0) + 1
|
||||
|
||||
return stats
|
||||
|
||||
def export_to_text(self) -> str:
|
||||
"""导出为文本格式"""
|
||||
with self._lock:
|
||||
lines = []
|
||||
lines.append("=" * 80)
|
||||
lines.append("小红书采集器日志")
|
||||
lines.append(f"导出时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
lines.append(f"总日志数: {len(self.logs)}")
|
||||
lines.append("=" * 80)
|
||||
lines.append("")
|
||||
|
||||
for log in self.logs:
|
||||
timestamp = log['timestamp'][:19] # 去掉微秒
|
||||
level = log['level'].ljust(8)
|
||||
thread = f"[{log['thread_id']}]" if log.get('thread_id') else ""
|
||||
message = log['message']
|
||||
lines.append(f"{timestamp} | {level} | {thread} {message}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# 全局日志管理器实例
|
||||
_log_manager_instance = None
|
||||
|
||||
|
||||
def get_log_manager() -> LogManager:
|
||||
"""获取全局日志管理器实例"""
|
||||
global _log_manager_instance
|
||||
if _log_manager_instance is None:
|
||||
_log_manager_instance = LogManager()
|
||||
return _log_manager_instance
|
||||
+641
@@ -0,0 +1,641 @@
|
||||
"""
|
||||
XiaoHongShu Scraper - 核心采集逻辑
|
||||
支持单线程和多线程模式
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Optional, AsyncIterator
|
||||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page, TimeoutError as PlaywrightTimeoutError
|
||||
from app import config
|
||||
from app.log_manager import LogManager
|
||||
|
||||
|
||||
class RiskControlException(Exception):
|
||||
"""风控异常"""
|
||||
def __init__(self, reason: str):
|
||||
self.reason = reason
|
||||
super().__init__(f"触发风控: {reason}")
|
||||
|
||||
|
||||
class XiaoHongShuScraper:
|
||||
"""小红书采集器"""
|
||||
|
||||
def __init__(self, event_queue: asyncio.Queue, log_manager: LogManager):
|
||||
self.event_queue = event_queue
|
||||
self.log_manager = log_manager
|
||||
self.playwright = None
|
||||
self.browser: Optional[Browser] = None
|
||||
self.is_running = False
|
||||
self._stop_requested = False
|
||||
|
||||
async def init_browser(self):
|
||||
"""初始化浏览器"""
|
||||
self.playwright = await async_playwright().start()
|
||||
self.browser = await self.playwright.chromium.launch(
|
||||
headless=config.HEADLESS,
|
||||
args=config.BROWSER_ARGS
|
||||
)
|
||||
await self.emit_log("SUCCESS", "浏览器初始化完成")
|
||||
|
||||
async def close_browser(self):
|
||||
"""关闭浏览器"""
|
||||
if self.browser:
|
||||
await self.browser.close()
|
||||
if self.playwright:
|
||||
await self.playwright.stop()
|
||||
await self.emit_log("INFO", "浏览器已关闭")
|
||||
|
||||
async def emit_log(self, level: str, message: str, thread_id: Optional[str] = None):
|
||||
"""发送日志事件"""
|
||||
# 持久化到日志管理器
|
||||
log_entry = self.log_manager.append(level, message, thread_id)
|
||||
|
||||
# 推送到 Web 前端
|
||||
await self.event_queue.put({
|
||||
"type": "log",
|
||||
**log_entry
|
||||
})
|
||||
|
||||
async def emit_progress(self, **kwargs):
|
||||
"""发送进度更新"""
|
||||
await self.event_queue.put({
|
||||
"type": "progress",
|
||||
**kwargs
|
||||
})
|
||||
|
||||
async def emit_user_status(self, user_id: str, status: str, note_count: int = 0, thread_id: Optional[str] = None):
|
||||
"""发送用户状态"""
|
||||
await self.event_queue.put({
|
||||
"type": "user_status",
|
||||
"user_id": user_id,
|
||||
"status": status,
|
||||
"note_count": note_count,
|
||||
"thread_id": thread_id
|
||||
})
|
||||
|
||||
async def emit_thread_status(self, thread_id: str, status: str, current_user: Optional[str] = None):
|
||||
"""发送线程状态"""
|
||||
await self.event_queue.put({
|
||||
"type": "thread_status",
|
||||
"thread_id": thread_id,
|
||||
"status": status,
|
||||
"current_user": current_user
|
||||
})
|
||||
|
||||
async def handle_login(self, context: BrowserContext) -> bool:
|
||||
"""
|
||||
处理登录
|
||||
返回是否已登录
|
||||
"""
|
||||
page = await context.new_page()
|
||||
await page.goto(config.LOGIN_URL)
|
||||
|
||||
await self.emit_log("WARNING", "请在浏览器中完成扫码登录...")
|
||||
await self.emit_log("INFO", "等待登录确认...")
|
||||
|
||||
# 等待登录成功（检查 URL 变化或特定元素）
|
||||
try:
|
||||
# 等待跳转到首页或出现用户头像
|
||||
await page.wait_for_url(
|
||||
f"{config.XIAOHONGSHU_BASE_URL}/**",
|
||||
timeout=config.LOGIN_TIMEOUT * 1000
|
||||
)
|
||||
await self.emit_log("SUCCESS", "登录成功！")
|
||||
|
||||
# 保存登录状态
|
||||
await context.storage_state(path=config.AUTH_STATE_FILE)
|
||||
await self.emit_log("INFO", "登录状态已保存")
|
||||
|
||||
await page.close()
|
||||
return True
|
||||
|
||||
except PlaywrightTimeoutError:
|
||||
await self.emit_log("ERROR", "登录超时，请重试")
|
||||
await page.close()
|
||||
return False
|
||||
|
||||
async def copy_cookies(self, target_context: BrowserContext, source_context: BrowserContext):
|
||||
"""复制 Cookies 到目标 Context"""
|
||||
cookies = await source_context.cookies()
|
||||
await target_context.add_cookies(cookies)
|
||||
|
||||
async def check_risk_control(self, page: Page) -> tuple[bool, str]:
|
||||
"""
|
||||
检测页面是否触发风控
|
||||
返回 (是否触发, 触发原因)
|
||||
"""
|
||||
try:
|
||||
content = await page.content()
|
||||
for keyword in config.RISK_KEYWORDS:
|
||||
if keyword in content:
|
||||
return True, f"检测到关键词: {keyword}"
|
||||
return False, ""
|
||||
except Exception as e:
|
||||
return False, f"检测失败: {str(e)}"
|
||||
|
||||
async def human_like_scroll(self, page: Page, thread_id: Optional[str] = None):
|
||||
"""
|
||||
模拟人类间歇性平滑滚动
|
||||
"""
|
||||
await self.emit_log("DEBUG", f"[{thread_id}] 开始平滑滚动加载笔记...", thread_id=thread_id)
|
||||
|
||||
scroll_count = 0
|
||||
last_height = 0
|
||||
|
||||
while scroll_count < config.MAX_SCROLL_COUNT:
|
||||
# 获取当前页面高度
|
||||
current_height = await page.evaluate("document.body.scrollHeight")
|
||||
|
||||
# 如果高度没有变化，可能到底了
|
||||
if current_height == last_height:
|
||||
await self.emit_log("DEBUG", f"[{thread_id}] 已滚动到底部", thread_id=thread_id)
|
||||
break
|
||||
|
||||
last_height = current_height
|
||||
|
||||
# 随机滚动距离
|
||||
scroll_step = random.randint(config.SCROLL_STEP_MIN, config.SCROLL_STEP_MAX)
|
||||
scroll_position = await page.evaluate("window.scrollY")
|
||||
|
||||
# 平滑滚动
|
||||
await page.evaluate(f'''
|
||||
window.scrollTo({{
|
||||
top: {scroll_position + scroll_step},
|
||||
behavior: 'smooth'
|
||||
}});
|
||||
''')
|
||||
|
||||
# 随机暂停
|
||||
pause_time = random.uniform(config.SCROLL_PAUSE_MIN, config.SCROLL_PAUSE_MAX)
|
||||
await asyncio.sleep(pause_time)
|
||||
|
||||
scroll_count += 1
|
||||
|
||||
# 每滚动几次检查一下风控
|
||||
if scroll_count % 5 == 0:
|
||||
is_risk, reason = await self.check_risk_control(page)
|
||||
if is_risk:
|
||||
raise RiskControlException(reason)
|
||||
|
||||
await self.emit_log("DEBUG", f"[{thread_id}] 滚动完成，共滚动 {scroll_count} 次", thread_id=thread_id)
|
||||
|
||||
async def extract_note_cards(self, page: Page, thread_id: Optional[str] = None) -> List[Dict]:
|
||||
"""
|
||||
从用户主页提取笔记卡片
|
||||
返回 [{'url': '...', 'title': '...'}]
|
||||
"""
|
||||
await self.emit_log("DEBUG", f"[{thread_id}] 提取笔记卡片...", thread_id=thread_id)
|
||||
|
||||
note_urls = []
|
||||
|
||||
try:
|
||||
# 尝试多种选择器
|
||||
selectors = [
|
||||
"a[href*='/explore/']",
|
||||
"section[class*='note'] a[href*='/explore/']",
|
||||
"div[class*='note-item'] a[href*='/explore/']",
|
||||
]
|
||||
|
||||
links = []
|
||||
for selector in selectors:
|
||||
try:
|
||||
elements = await page.query_selector_all(selector)
|
||||
if elements:
|
||||
links = elements
|
||||
await self.emit_log("DEBUG", f"[{thread_id}] 使用选择器: {selector}", thread_id=thread_id)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# 提取 URL
|
||||
for link in links:
|
||||
url = await link.get_attribute('href')
|
||||
if url and '/explore/' in url:
|
||||
# 补全 URL
|
||||
if url.startswith('/'):
|
||||
url = f"{config.XIAOHONGSHU_BASE_URL}{url}"
|
||||
|
||||
# 尝试获取标题
|
||||
try:
|
||||
title_elem = await link.query_selector("span, div")
|
||||
title = await title_elem.inner_text() if title_elem else "无标题"
|
||||
except Exception:
|
||||
title = "无标题"
|
||||
|
||||
note_urls.append({
|
||||
'url': url,
|
||||
'title': title[:50] # 限制标题长度
|
||||
})
|
||||
|
||||
await self.emit_log("INFO", f"[{thread_id}] 提取到 {len(note_urls)} 条笔记", thread_id=thread_id)
|
||||
|
||||
return note_urls
|
||||
|
||||
except Exception as e:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 提取笔记卡片失败: {str(e)}", thread_id=thread_id)
|
||||
return []
|
||||
|
||||
async def extract_note_detail(self, page: Page, thread_id: Optional[str] = None) -> Optional[Dict]:
|
||||
"""
|
||||
从笔记详情页提取数据
|
||||
"""
|
||||
try:
|
||||
# 等待页面加载
|
||||
await asyncio.sleep(random.uniform(*config.PAGE_LOAD_DELAY))
|
||||
|
||||
# 检查风控
|
||||
is_risk, reason = await self.check_risk_control(page)
|
||||
if is_risk:
|
||||
raise RiskControlException(reason)
|
||||
|
||||
data = {}
|
||||
|
||||
# 提取标题
|
||||
try:
|
||||
title_elem = await page.query_selector(config.SELECTOR_DETAIL_TITLE)
|
||||
data['title'] = await title_elem.inner_text() if title_elem else "无标题"
|
||||
except Exception:
|
||||
data['title'] = "无标题"
|
||||
|
||||
# 提取发布时间
|
||||
try:
|
||||
time_elem = await page.query_selector(config.SELECTOR_DETAIL_PUBLISH_TIME)
|
||||
data['publish_time'] = await time_elem.inner_text() if time_elem else ""
|
||||
except Exception:
|
||||
data['publish_time'] = ""
|
||||
|
||||
# 提取发布地
|
||||
try:
|
||||
loc_elem = await page.query_selector(config.SELECTOR_DETAIL_LOCATION)
|
||||
data['location'] = await loc_elem.inner_text() if loc_elem else ""
|
||||
except Exception:
|
||||
data['location'] = ""
|
||||
|
||||
# 提取互动数据（尝试从页面内容解析）
|
||||
content = await page.content()
|
||||
|
||||
# 评论数
|
||||
data['comment_count'] = self._extract_count(content, ['评论', 'comment'])
|
||||
|
||||
# 点赞数
|
||||
data['like_count'] = self._extract_count(content, ['点赞', '赞', 'like'])
|
||||
|
||||
# 收藏数
|
||||
data['collect_count'] = self._extract_count(content, ['收藏', 'star', 'collect'])
|
||||
|
||||
# 笔记链接
|
||||
data['note_url'] = page.url
|
||||
|
||||
return data
|
||||
|
||||
except RiskControlException:
|
||||
raise
|
||||
except Exception as e:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 提取详情失败: {str(e)}", thread_id=thread_id)
|
||||
return None
|
||||
|
||||
def _extract_count(self, content: str, keywords: List[str]) -> int:
|
||||
"""从页面内容提取计数"""
|
||||
import re
|
||||
for keyword in keywords:
|
||||
# 匹配数字
|
||||
pattern = f"{keyword}[\\s:：]*([\\d.]+)"
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
count_str = match.group(1)
|
||||
# 处理万、k等单位
|
||||
if '万' in count_str:
|
||||
return int(float(count_str.replace('万', '')) * 10000)
|
||||
elif 'k' in count_str.lower():
|
||||
return int(float(count_str.lower().replace('k', '')) * 1000)
|
||||
else:
|
||||
return int(count_str)
|
||||
return 0
|
||||
|
||||
async def save_user_data(self, user_id: str, new_notes: List[Dict]):
|
||||
"""保存用户数据（增量）"""
|
||||
file_path = os.path.join(config.DATA_DIR, f"{user_id}.json")
|
||||
|
||||
# 读取现有数据
|
||||
existing_notes = []
|
||||
if os.path.exists(file_path):
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
existing_notes = data.get('notes', [])
|
||||
except Exception as e:
|
||||
await self.emit_log("WARNING", f"读取现有数据失败: {e}")
|
||||
|
||||
# 提取现有 URL
|
||||
existing_urls = {note['note_url'] for note in existing_notes}
|
||||
|
||||
# 追加新笔记
|
||||
added_count = 0
|
||||
for note in new_notes:
|
||||
if note['note_url'] not in existing_urls:
|
||||
existing_notes.append(note)
|
||||
existing_urls.add(note['note_url'])
|
||||
added_count += 1
|
||||
|
||||
# 保存
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({
|
||||
"user_id": user_id,
|
||||
"last_updated": datetime.now().isoformat(),
|
||||
"total_notes": len(existing_notes),
|
||||
"notes": existing_notes
|
||||
}, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return added_count
|
||||
|
||||
async def process_user(self, context: BrowserContext, user_id: str, thread_id: str = "Main") -> int:
|
||||
"""
|
||||
处理单个用户
|
||||
返回新增笔记数
|
||||
"""
|
||||
await self.emit_user_status(user_id, "processing", thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "busy", user_id)
|
||||
|
||||
try:
|
||||
await self.emit_log("INFO", f"[{thread_id}] 开始处理用户: {user_id}", thread_id=thread_id)
|
||||
|
||||
# 创建新页面
|
||||
page = await context.new_page()
|
||||
|
||||
# 访问用户主页
|
||||
url = config.USER_PROFILE_URL_TEMPLATE.format(user_id=user_id)
|
||||
await self.emit_log("INFO", f"[{thread_id}] 访问用户主页: {url}", thread_id=thread_id)
|
||||
|
||||
try:
|
||||
await page.goto(url, timeout=30000)
|
||||
except PlaywrightTimeoutError:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 访问用户主页超时: {user_id}", thread_id=thread_id)
|
||||
await page.close()
|
||||
await self.emit_user_status(user_id, "failed", thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "idle", None)
|
||||
return 0
|
||||
|
||||
# 检查风控
|
||||
is_risk, reason = await self.check_risk_control(page)
|
||||
if is_risk:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 访问用户主页触发风控: {reason}", thread_id=thread_id)
|
||||
await page.close()
|
||||
await self.emit_user_status(user_id, "failed", thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "idle", None)
|
||||
raise RiskControlException(reason)
|
||||
|
||||
await asyncio.sleep(random.uniform(*config.PAGE_LOAD_DELAY))
|
||||
|
||||
# 滚动加载所有笔记
|
||||
await self.human_like_scroll(page, thread_id)
|
||||
|
||||
# 提取笔记卡片
|
||||
note_cards = await self.extract_note_cards(page, thread_id)
|
||||
|
||||
if not note_cards:
|
||||
await self.emit_log("WARNING", f"[{thread_id}] 未找到笔记，可能页面结构变化", thread_id=thread_id)
|
||||
await page.close()
|
||||
await self.emit_user_status(user_id, "completed", 0, thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "idle", None)
|
||||
return 0
|
||||
|
||||
# 采集每条笔记
|
||||
collected_notes = []
|
||||
for i, card in enumerate(note_cards):
|
||||
if self._stop_requested:
|
||||
await self.emit_log("WARNING", f"[{thread_id}] 收到停止信号", thread_id=thread_id)
|
||||
break
|
||||
|
||||
await self.emit_log("INFO", f"[{thread_id}] 采集笔记 {i+1}/{len(note_cards)}: {card['title']}", thread_id=thread_id)
|
||||
|
||||
try:
|
||||
# 访问详情页
|
||||
await page.goto(card['url'], timeout=30000)
|
||||
|
||||
# 模拟阅读停留
|
||||
await asyncio.sleep(random.uniform(*config.NOTE_READ_DELAY))
|
||||
|
||||
# 提取数据
|
||||
note_data = await self.extract_note_detail(page, thread_id)
|
||||
|
||||
if note_data:
|
||||
collected_notes.append(note_data)
|
||||
await self.emit_log("SUCCESS", f"[{thread_id}] 成功采集: {note_data['title'][:30]}", thread_id=thread_id)
|
||||
|
||||
# 返回列表页
|
||||
await page.go_back(url, wait_until="networkidle")
|
||||
|
||||
# 间隔
|
||||
await asyncio.sleep(random.uniform(*config.NOTE_INTERVAL_DELAY))
|
||||
|
||||
except RiskControlException:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 采集笔记触发风控", thread_id=thread_id)
|
||||
raise
|
||||
except Exception as e:
|
||||
await self.emit_log("WARNING", f"[{thread_id}] 采集笔记失败: {str(e)}", thread_id=thread_id)
|
||||
continue
|
||||
|
||||
await page.close()
|
||||
|
||||
# 保存数据
|
||||
if collected_notes:
|
||||
added_count = await self.save_user_data(user_id, collected_notes)
|
||||
await self.emit_log("SUCCESS", f"[{thread_id}] 用户 {user_id} 采集完成，新增 {added_count} 条笔记", thread_id=thread_id)
|
||||
await self.emit_user_status(user_id, "completed", len(collected_notes), thread_id=thread_id)
|
||||
else:
|
||||
await self.emit_log("INFO", f"[{thread_id}] 用户 {user_id} 无新笔记", thread_id=thread_id)
|
||||
await self.emit_user_status(user_id, "completed", 0, thread_id=thread_id)
|
||||
|
||||
await self.emit_thread_status(thread_id, "idle", None)
|
||||
return len(collected_notes)
|
||||
|
||||
except RiskControlException:
|
||||
await self.emit_user_status(user_id, "failed", thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "error", None)
|
||||
raise
|
||||
except Exception as e:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 处理用户 {user_id} 失败: {str(e)}", thread_id=thread_id)
|
||||
await self.emit_user_status(user_id, "failed", thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "error", None)
|
||||
return 0
|
||||
|
||||
async def run_single_thread(self, user_ids: List[str]):
|
||||
"""单线程模式"""
|
||||
await self.emit_log("INFO", "启动单线程采集模式")
|
||||
|
||||
# 创建 Context 并登录
|
||||
context = await self.browser.new_context(
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
|
||||
# 检查是否有保存的登录状态
|
||||
if os.path.exists(config.AUTH_STATE_FILE):
|
||||
await self.emit_log("INFO", "加载已保存的登录状态")
|
||||
await context.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
|
||||
else:
|
||||
# 需要登录
|
||||
login_success = await self.handle_login(context)
|
||||
if not login_success:
|
||||
await context.close()
|
||||
return
|
||||
|
||||
self.is_running = True
|
||||
|
||||
for i, user_id in enumerate(user_ids):
|
||||
if self._stop_requested:
|
||||
await self.emit_log("WARNING", "收到停止请求")
|
||||
break
|
||||
|
||||
await self.emit_log("INFO", f"处理用户 {i+1}/{len(user_ids)}: {user_id}")
|
||||
|
||||
try:
|
||||
await self.process_user(context, user_id, thread_id="Main")
|
||||
|
||||
# 用户切换延迟
|
||||
if i < len(user_ids) - 1:
|
||||
delay = random.uniform(*config.USER_SWITCH_DELAY)
|
||||
await self.emit_log("INFO", f"等待 {delay:.1f} 秒后处理下一个用户...")
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
except RiskControlException as e:
|
||||
await self.emit_log("ERROR", f"触发风控: {e.reason}，终止采集")
|
||||
break
|
||||
except Exception as e:
|
||||
await self.emit_log("ERROR", f"处理用户 {user_id} 失败: {str(e)}")
|
||||
continue
|
||||
|
||||
await context.close()
|
||||
self.is_running = False
|
||||
await self.emit_log("SUCCESS", "单线程采集完成")
|
||||
|
||||
async def worker(self, context: BrowserContext, task_queue: asyncio.Queue, thread_id: str):
|
||||
"""
|
||||
Worker 处理任务（多线程模式）
|
||||
"""
|
||||
# 线程启动时错开
|
||||
await asyncio.sleep(int(thread_id.split('-')[1]) * config.THREAD_OFFSET)
|
||||
|
||||
await self.emit_thread_status(thread_id, "idle", None)
|
||||
|
||||
while True:
|
||||
if self._stop_requested:
|
||||
break
|
||||
|
||||
try:
|
||||
# 获取任务
|
||||
user_id = await asyncio.wait_for(task_queue.get(), timeout=1.0)
|
||||
await self.emit_thread_status(thread_id, "busy", user_id)
|
||||
|
||||
try:
|
||||
# 线程间最小间隔
|
||||
await asyncio.sleep(random.uniform(
|
||||
config.THREAD_MIN_DELAY,
|
||||
config.THREAD_MIN_DELAY + 3
|
||||
))
|
||||
|
||||
await self.process_user(context, user_id, thread_id)
|
||||
|
||||
except RiskControlException as e:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 触发风控: {e.reason}，停止该线程", thread_id=thread_id)
|
||||
await self.emit_thread_status(thread_id, "error", None)
|
||||
break
|
||||
except Exception as e:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] 处理失败: {str(e)}", thread_id=thread_id)
|
||||
finally:
|
||||
task_queue.task_done()
|
||||
await self.emit_thread_status(thread_id, "idle", None)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception as e:
|
||||
await self.emit_log("ERROR", f"[{thread_id}] Worker 错误: {str(e)}", thread_id=thread_id)
|
||||
break
|
||||
|
||||
async def run_multi_thread(self, user_ids: List[str], max_threads: int = 3):
|
||||
"""多线程模式"""
|
||||
await self.emit_log("INFO", f"启动多线程采集模式，线程数: {max_threads}")
|
||||
|
||||
# 创建多个 Context
|
||||
contexts = []
|
||||
for i in range(max_threads):
|
||||
ctx = await self.browser.new_context(
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
|
||||
# 第一个 Context 登录，其他复制 Cookies
|
||||
if i == 0:
|
||||
if os.path.exists(config.AUTH_STATE_FILE):
|
||||
await self.emit_log("INFO", "加载已保存的登录状态")
|
||||
await ctx.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
|
||||
else:
|
||||
login_success = await self.handle_login(ctx)
|
||||
if not login_success:
|
||||
await ctx.close()
|
||||
return
|
||||
else:
|
||||
await self.copy_cookies(ctx, contexts[0])
|
||||
await self.emit_log("INFO", f"Thread-{i+1} 已复制登录状态")
|
||||
|
||||
contexts.append(ctx)
|
||||
|
||||
# 创建任务队列
|
||||
task_queue = asyncio.Queue()
|
||||
for user_id in user_ids:
|
||||
await task_queue.put(user_id)
|
||||
|
||||
self.is_running = True
|
||||
|
||||
# 创建 workers
|
||||
workers = [
|
||||
asyncio.create_task(
|
||||
self.worker(contexts[i], task_queue, f"Thread-{i+1}")
|
||||
)
|
||||
for i in range(max_threads)
|
||||
]
|
||||
|
||||
# 等待完成
|
||||
await task_queue.join()
|
||||
|
||||
# 取消 workers
|
||||
for worker in workers:
|
||||
worker.cancel()
|
||||
|
||||
# 关闭 contexts
|
||||
for ctx in contexts:
|
||||
await ctx.close()
|
||||
|
||||
self.is_running = False
|
||||
await self.emit_log("SUCCESS", "多线程采集完成")
|
||||
|
||||
def stop(self):
|
||||
"""请求停止采集"""
|
||||
self._stop_requested = True
|
||||
self.is_running = False
|
||||
|
||||
async def run(self, user_ids: List[str], mode: str = "single", max_threads: int = 3):
|
||||
"""
|
||||
运行采集
|
||||
:param user_ids: 用户 ID 列表
|
||||
:param mode: single / multi
|
||||
:param max_threads: 多线程模式下的线程数
|
||||
"""
|
||||
self._stop_requested = False
|
||||
|
||||
await self.emit_log("INFO", f"开始采集，共 {len(user_ids)} 个用户")
|
||||
|
||||
await self.init_browser()
|
||||
|
||||
try:
|
||||
if mode == "single":
|
||||
await self.run_single_thread(user_ids)
|
||||
else:
|
||||
await self.run_multi_thread(user_ids, max_threads)
|
||||
finally:
|
||||
await self.close_browser()
|
||||
|
||||
await self.emit_log("SUCCESS", "所有采集任务完成")
|
||||
+516
@@ -0,0 +1,516 @@
|
||||
"""
|
||||
Web Backend - FastAPI 应用
|
||||
提供 API 和 SSE 事件推送
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
from fastapi import FastAPI, HTTPException, BackgroundTasks
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from fastapi.requests import Request
|
||||
from pydantic import BaseModel
|
||||
from typing import List as ListType
|
||||
|
||||
from app import config
|
||||
from app.log_manager import get_log_manager
|
||||
from app.scraper import XiaoHongShuScraper
|
||||
|
||||
|
||||
# ========== Pydantic 模型 ==========
|
||||
class StartRequest(BaseModel):
|
||||
mode: str = "single" # single / multi
|
||||
max_threads: int = 3
|
||||
|
||||
|
||||
class StatusResponse(BaseModel):
|
||||
is_running: bool
|
||||
mode: str
|
||||
max_threads: int
|
||||
total_users: int
|
||||
completed_users: int
|
||||
failed_users: int
|
||||
active_threads: int
|
||||
users_status: Dict[str, Dict]
|
||||
|
||||
|
||||
# ========== 全局状态 ==========
|
||||
class AppState:
|
||||
def __init__(self):
|
||||
self.is_running = False
|
||||
self.mode = "single"
|
||||
self.max_threads = 1
|
||||
self.total_users = 0
|
||||
self.completed_users = 0
|
||||
self.failed_users = 0
|
||||
self.active_threads = 0
|
||||
self.users_status: Dict[str, Dict] = {}
|
||||
self.thread_status: Dict[str, Dict] = {}
|
||||
self.event_queue: asyncio.Queue = None
|
||||
self.scraper: Optional[XiaoHongShuScraper] = None
|
||||
self.scraper_task: Optional[asyncio.Task] = None
|
||||
|
||||
|
||||
app_state = AppState()
|
||||
log_manager = get_log_manager()
|
||||
|
||||
# ========== FastAPI 应用 ==========
|
||||
app = FastAPI(title="小红书数据采集器", version="1.0.0")
|
||||
|
||||
# 挂载静态文件和模板
|
||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
# ========== 路由 ==========
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def index(request: Request):
|
||||
"""首页"""
|
||||
return templates.TemplateResponse("index.html", {"request": request})
|
||||
|
||||
|
||||
@app.get("/api/status", response_model=StatusResponse)
|
||||
async def get_status():
|
||||
"""获取当前状态"""
|
||||
return StatusResponse(
|
||||
is_running=app_state.is_running,
|
||||
mode=app_state.mode,
|
||||
max_threads=app_state.max_threads,
|
||||
total_users=app_state.total_users,
|
||||
completed_users=app_state.completed_users,
|
||||
failed_users=app_state.failed_users,
|
||||
active_threads=app_state.active_threads,
|
||||
users_status=app_state.users_status
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/start")
|
||||
async def start_scraping(request: StartRequest):
|
||||
"""启动采集"""
|
||||
if app_state.is_running:
|
||||
raise HTTPException(status_code=400, detail="采集任务正在运行中")
|
||||
|
||||
# 读取用户 ID 列表
|
||||
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
||||
if not os.path.exists(users_file):
|
||||
raise HTTPException(status_code=400, detail="用户文件不存在，请先创建 data/users.txt")
|
||||
|
||||
with open(users_file, 'r', encoding='utf-8') as f:
|
||||
user_ids = [line.strip() for line in f if line.strip()]
|
||||
|
||||
if not user_ids:
|
||||
raise HTTPException(status_code=400, detail="用户 ID 列表为空")
|
||||
|
||||
# 验证模式
|
||||
if request.mode not in ["single", "multi"]:
|
||||
raise HTTPException(status_code=400, detail="无效的模式，必须是 single 或 multi")
|
||||
|
||||
# 验证线程数
|
||||
if request.mode == "multi":
|
||||
if request.max_threads < 1 or request.max_threads > config.MAX_CONCURRENT_THREADS:
|
||||
raise HTTPException(status_code=400, detail=f"线程数必须在 1-{config.MAX_CONCURRENT_THREADS} 之间")
|
||||
|
||||
# 重置状态
|
||||
app_state.is_running = True
|
||||
app_state.mode = request.mode
|
||||
app_state.max_threads = request.max_threads if request.mode == "multi" else 1
|
||||
app_state.total_users = len(user_ids)
|
||||
app_state.completed_users = 0
|
||||
app_state.failed_users = 0
|
||||
app_state.active_threads = 0
|
||||
app_state.users_status = {}
|
||||
app_state.thread_status = {}
|
||||
|
||||
# 初始化用户状态
|
||||
for user_id in user_ids:
|
||||
app_state.users_status[user_id] = {
|
||||
"status": "pending",
|
||||
"note_count": 0,
|
||||
"thread_id": None
|
||||
}
|
||||
|
||||
# 创建事件队列
|
||||
app_state.event_queue = asyncio.Queue()
|
||||
|
||||
# 创建采集器
|
||||
app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager)
|
||||
|
||||
# 启动采集任务
|
||||
async def run_scraper():
|
||||
try:
|
||||
await app_state.scraper.run(
|
||||
user_ids=user_ids,
|
||||
mode=request.mode,
|
||||
max_threads=request.max_threads
|
||||
)
|
||||
finally:
|
||||
app_state.is_running = False
|
||||
await app_state.event_queue.put({
|
||||
"type": "finished"
|
||||
})
|
||||
|
||||
app_state.scraper_task = asyncio.create_task(run_scraper())
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"采集任务已启动，模式: {request.mode}, 用户数: {len(user_ids)}",
|
||||
"total_users": len(user_ids),
|
||||
"mode": request.mode,
|
||||
"max_threads": app_state.max_threads
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/stop")
|
||||
async def stop_scraping():
|
||||
"""停止采集"""
|
||||
if not app_state.is_running:
|
||||
raise HTTPException(status_code=400, detail="没有正在运行的采集任务")
|
||||
|
||||
if app_state.scraper:
|
||||
app_state.scraper.stop()
|
||||
|
||||
# 取消任务
|
||||
if app_state.scraper_task:
|
||||
app_state.scraper_task.cancel()
|
||||
try:
|
||||
await app_state.scraper_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
app_state.is_running = False
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": "采集任务已停止"
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/logs/clear")
|
||||
async def clear_logs():
|
||||
"""清空日志"""
|
||||
log_manager.clear()
|
||||
return {
|
||||
"success": True,
|
||||
"message": "日志已清空"
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/logs")
|
||||
async def get_logs(level: Optional[str] = None, limit: Optional[int] = None):
|
||||
"""获取日志"""
|
||||
logs = log_manager.get_logs(level=level, limit=limit)
|
||||
return {
|
||||
"logs": logs,
|
||||
"stats": log_manager.get_stats()
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/logs/search")
|
||||
async def search_logs(keyword: str, limit: Optional[int] = None):
|
||||
"""搜索日志"""
|
||||
logs = log_manager.search_logs(keyword, limit)
|
||||
return {
|
||||
"logs": logs,
|
||||
"count": len(logs)
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/logs/download")
|
||||
async def download_logs():
|
||||
"""下载日志文件"""
|
||||
text = log_manager.export_to_text()
|
||||
|
||||
return StreamingResponse(
|
||||
iter([text]),
|
||||
media_type="text/plain",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=xiaohongshu_logs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/users")
|
||||
async def get_users():
|
||||
"""获取用户列表"""
|
||||
users_file = os.path.join(config.DATA_DIR, "users.txt")
|
||||
|
||||
if not os.path.exists(users_file):
|
||||
return {"users": []}
|
||||
|
||||
with open(users_file, 'r', encoding='utf-8') as f:
|
||||
users = [{"id": line.strip()} for line in f if line.strip()]
|
||||
|
||||
# 添加采集状态
|
||||
for user in users:
|
||||
user_id = user["id"]
|
||||
if user_id in app_state.users_status:
|
||||
user.update(app_state.users_status[user_id])
|
||||
else:
|
||||
user["status"] = "pending"
|
||||
user["note_count"] = 0
|
||||
|
||||
return {"users": users}
|
||||
|
||||
|
||||
@app.get("/api/data/{user_id}")
|
||||
async def get_user_data(user_id: str):
|
||||
"""获取用户采集数据"""
|
||||
file_path = os.path.join(config.DATA_DIR, f"{user_id}.json")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=404, detail="用户数据不存在")
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@app.get("/api/events")
|
||||
async def event_stream():
|
||||
"""SSE 事件流"""
|
||||
async def event_generator():
|
||||
while True:
|
||||
try:
|
||||
# 等待事件，超时 1 秒发送心跳
|
||||
event = await asyncio.wait_for(
|
||||
app_state.event_queue.get(),
|
||||
timeout=1.0
|
||||
)
|
||||
|
||||
# 更新状态
|
||||
if event["type"] == "log":
|
||||
# 日志不需要更新状态
|
||||
pass
|
||||
elif event["type"] == "progress":
|
||||
app_state.completed_users = event.get("completed", app_state.completed_users)
|
||||
app_state.failed_users = event.get("failed", app_state.failed_users)
|
||||
app_state.active_threads = event.get("active_threads", app_state.active_threads)
|
||||
elif event["type"] == "user_status":
|
||||
if event["user_id"] in app_state.users_status:
|
||||
app_state.users_status[event["user_id"]]["status"] = event["status"]
|
||||
app_state.users_status[event["user_id"]]["note_count"] = event.get("note_count", 0)
|
||||
app_state.users_status[event["user_id"]]["thread_id"] = event.get("thread_id")
|
||||
elif event["type"] == "thread_status":
|
||||
app_state.thread_status[event["thread_id"]] = {
|
||||
"status": event["status"],
|
||||
"current_user": event.get("current_user")
|
||||
}
|
||||
# 更新活跃线程数
|
||||
app_state.active_threads = sum(
|
||||
1 for s in app_state.thread_status.values() if s.get("status") == "busy"
|
||||
)
|
||||
|
||||
# 发送事件
|
||||
yield f"data: {json.dumps(event)}\n\n"
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
# 发送心跳
|
||||
yield ": heartbeat\n\n"
|
||||
|
||||
except Exception as e:
|
||||
# 错误事件
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"Access-Control-Allow-Origin": "*"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ========== Git 相关 API ==========
|
||||
|
||||
@app.get("/api/git/status")
|
||||
async def git_status():
|
||||
"""获取 Git 状态"""
|
||||
if not os.path.exists("/app/.git"):
|
||||
return {
|
||||
"enabled": False,
|
||||
"message": "不是 Git 仓库"
|
||||
}
|
||||
|
||||
try:
|
||||
# 获取当前分支
|
||||
result = subprocess.run(
|
||||
["git", "branch", "--show-current"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
current_branch = result.stdout.strip()
|
||||
|
||||
# 获取最新提交
|
||||
result = subprocess.run(
|
||||
["git", "log", "-1", "--format=%h %s %cr"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
latest_commit = result.stdout.strip()
|
||||
|
||||
# 检查是否有未提交的更改
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
has_changes = len(result.stdout.strip()) > 0
|
||||
|
||||
return {
|
||||
"enabled": True,
|
||||
"branch": current_branch,
|
||||
"latest_commit": latest_commit,
|
||||
"has_changes": has_changes,
|
||||
"auto_pull_enabled": config.ENABLE_GIT_PULL,
|
||||
"auto_pull_interval": config.AUTO_PULL_INTERVAL
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"enabled": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/git/pull")
|
||||
async def git_pull():
|
||||
"""手动拉取最新代码"""
|
||||
if not os.path.exists("/app/.git"):
|
||||
raise HTTPException(status_code=400, detail="不是 Git 仓库")
|
||||
|
||||
try:
|
||||
# 获取当前分支
|
||||
result = subprocess.run(
|
||||
["git", "branch", "--show-current"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
current_branch = result.stdout.strip()
|
||||
|
||||
# 拉取最新代码
|
||||
result = subprocess.run(
|
||||
["git", "fetch", "origin", current_branch],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise HTTPException(status_code=500, detail=f"拉取失败: {result.stderr}")
|
||||
|
||||
# 检查是否有更新
|
||||
result = subprocess.run(
|
||||
["git", "rev-list", "HEAD..origin/" + current_branch, "--count"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
commits_behind = int(result.stdout.strip())
|
||||
|
||||
if commits_behind == 0:
|
||||
return {
|
||||
"success": True,
|
||||
"message": "已是最新代码",
|
||||
"needs_restart": False
|
||||
}
|
||||
|
||||
# 重置到最新代码
|
||||
result = subprocess.run(
|
||||
["git", "reset", "--hard", "origin/" + current_branch],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise HTTPException(status_code=500, detail=f"重置失败: {result.stderr}")
|
||||
|
||||
# 检查 Python 文件是否有变化
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", "HEAD@{1}", "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/app"
|
||||
)
|
||||
changed_files = result.stdout.strip().split('\n')
|
||||
python_changed = any(f.endswith('.py') for f in changed_files if f)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"已拉取 {commits_behind} 个新提交",
|
||||
"changed_files": changed_files,
|
||||
"needs_restart": python_changed,
|
||||
"restart_hint": "Python 代码有变化，建议重启容器以应用更新"
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"拉取失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/api/git/restart")
|
||||
async def restart_container():
|
||||
"""
|
||||
请求重启容器
|
||||
注意：这需要在容器外配置重启策略
|
||||
"""
|
||||
return {
|
||||
"success": True,
|
||||
"message": "重启请求已接收",
|
||||
"methods": [
|
||||
"docker restart xiaohongshu-scraper",
|
||||
"docker-compose restart"
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# ========== 启动事件 ==========
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""启动时初始化"""
|
||||
# 确保数据目录存在
|
||||
os.makedirs(config.DATA_DIR, exist_ok=True)
|
||||
os.makedirs(config.SCREENSHOT_DIR, exist_ok=True)
|
||||
|
||||
# 初始化事件队列
|
||||
app_state.event_queue = asyncio.Queue()
|
||||
|
||||
print("=" * 60)
|
||||
print("小红书数据采集器 Web 服务启动")
|
||||
print(f"访问地址: http://{config.WEB_HOST}:{config.WEB_PORT}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def shutdown_event():
|
||||
"""关闭时清理"""
|
||||
if app_state.scraper:
|
||||
app_state.scraper.stop()
|
||||
|
||||
if app_state.scraper_task:
|
||||
app_state.scraper_task.cancel()
|
||||
|
||||
print("Web 服务已关闭")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(
|
||||
"app.web:app",
|
||||
host=config.WEB_HOST,
|
||||
port=config.WEB_PORT,
|
||||
reload=True
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
# 小红书用户 ID 列表
|
||||
# 每行一个用户 ID，例如：
|
||||
# 5ef74296000000000100003a
|
||||
# 5b6c1f9e0000000001000388
|
||||
@@ -0,0 +1,23 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
xiaohongshu-scraper:
|
||||
build: .
|
||||
container_name: xiaohongshu-scraper
|
||||
ports:
|
||||
- "18080:18080"
|
||||
volumes:
|
||||
# 持久化数据目录（采集结果、日志、登录状态）
|
||||
- ./data:/app/data
|
||||
environment:
|
||||
- HOST=0.0.0.0
|
||||
- PORT=18080
|
||||
- HEADLESS=false
|
||||
- SCREENSHOT_ON_ERROR=true
|
||||
- MAX_THREADS=5
|
||||
# Git 自动更新配置（可选）
|
||||
# - ENABLE_GIT_PULL=true
|
||||
# - GIT_REPO_URL=https://github.com/yourusername/xiaohongshu-scraper.git
|
||||
# - GIT_BRANCH=main
|
||||
# - AUTO_PULL_INTERVAL=3600
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "================================"
|
||||
echo "小红书数据采集器启动脚本"
|
||||
echo "================================"
|
||||
|
||||
# Git 自动更新功能
|
||||
if [ "$ENABLE_GIT_PULL" = "true" ]; then
|
||||
echo "Git 自动更新已启用"
|
||||
|
||||
# 如果是首次运行且配置了仓库地址，则克隆仓库
|
||||
if [ ! -d "/app/.git" ] && [ -n "$GIT_REPO_URL" ]; then
|
||||
echo "克隆仓库: $GIT_REPO_URL"
|
||||
cd /tmp
|
||||
git clone "$GIT_REPO_URL" repo
|
||||
cd repo
|
||||
git checkout "${GIT_BRANCH:-main}"
|
||||
cp -r . /app/
|
||||
cd /app
|
||||
rm -rf /tmp/repo
|
||||
echo "仓库克隆完成"
|
||||
fi
|
||||
|
||||
# 启动 Git 自动拉取后台任务
|
||||
if [ -d "/app/.git" ]; then
|
||||
echo "启动 Git 自动拉取服务（间隔: ${AUTO_PULL_INTERVAL:-3600}秒）"
|
||||
|
||||
(
|
||||
while true; do
|
||||
sleep "${AUTO_PULL_INTERVAL:-3600}"
|
||||
|
||||
echo "[Git Pull] 检查更新..."
|
||||
cd /app
|
||||
|
||||
# 获取当前分支
|
||||
CURRENT_BRANCH=$(git branch --show-current)
|
||||
|
||||
# 拉取最新代码
|
||||
if git fetch origin "${CURRENT_BRANCH}" && git reset --hard "origin/${CURRENT_BRANCH}"; then
|
||||
echo "[Git Pull] 代码已更新"
|
||||
|
||||
# 检查是否有 Python 文件变化
|
||||
if git diff --name-only HEAD@{1} HEAD | grep -q '\.py$'; then
|
||||
echo "[Git Pull] Python 代码有变化，需要重启容器"
|
||||
# 可以在这里发送重启信号
|
||||
fi
|
||||
else
|
||||
echo "[Git Pull] 拉取失败"
|
||||
fi
|
||||
done
|
||||
) &
|
||||
fi
|
||||
else
|
||||
echo "Git 自动更新未启用"
|
||||
fi
|
||||
|
||||
echo "启动 Web 服务..."
|
||||
echo "端口: ${PORT:-18080}"
|
||||
echo "无头模式: ${HEADLESS:-false}"
|
||||
echo "================================"
|
||||
|
||||
# 启动应用
|
||||
exec python main.py
|
||||
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
小红书数据采集器 - 程序入口
|
||||
"""
|
||||
import uvicorn
|
||||
from app import config
|
||||
from app.web import app
|
||||
|
||||
|
||||
def main():
|
||||
"""启动 Web 服务"""
|
||||
print("=" * 60)
|
||||
print("小红书数据采集器")
|
||||
print("=" * 60)
|
||||
print(f"版本: 1.0.0")
|
||||
print(f"访问地址: http://{config.WEB_HOST}:{config.WEB_PORT}")
|
||||
print(f"数据目录: {config.DATA_DIR}")
|
||||
print("=" * 60)
|
||||
print("\n请在浏览器中打开上述地址，然后:")
|
||||
print("1. 在 data/users.txt 中添加要采集的用户 ID（每行一个）")
|
||||
print("2. 选择采集模式（单线程/多线程）")
|
||||
print("3. 点击「启动采集」")
|
||||
print("4. 在弹出的浏览器窗口中扫码登录")
|
||||
print("5. 实时监控采集进度\n")
|
||||
|
||||
uvicorn.run(
|
||||
app,
|
||||
host=config.WEB_HOST,
|
||||
port=config.WEB_PORT,
|
||||
log_level="info"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,14 @@
|
||||
# Web Framework
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.32.0
|
||||
jinja2==3.1.4
|
||||
python-multipart==0.0.12
|
||||
|
||||
# Browser Automation
|
||||
playwright==1.48.0
|
||||
|
||||
# Async Utilities
|
||||
aiofiles==24.1.0
|
||||
|
||||
# Utilities
|
||||
python-dotenv==1.0.1
|
||||
+586
@@ -0,0 +1,586 @@
|
||||
/**
|
||||
* 小红书采集器前端逻辑
|
||||
*/
|
||||
|
||||
// ========== 状态管理 ==========
|
||||
const state = {
|
||||
isRunning: false,
|
||||
mode: 'single',
|
||||
maxThreads: 3,
|
||||
totalUsers: 0,
|
||||
completedUsers: 0,
|
||||
failedUsers: 0,
|
||||
activeThreads: 0,
|
||||
usersStatus: {},
|
||||
threadStatus: {},
|
||||
eventSource: null,
|
||||
autoScrollLogs: true,
|
||||
logFilter: 'all',
|
||||
logs: []
|
||||
};
|
||||
|
||||
// ========== DOM 元素 ==========
|
||||
const elements = {
|
||||
startBtn: null,
|
||||
stopBtn: null,
|
||||
clearLogsBtn: null,
|
||||
exportDataBtn: null,
|
||||
themeToggle: null,
|
||||
themeIcon: null,
|
||||
modeRadios: null,
|
||||
threadControl: null,
|
||||
threadCount: null,
|
||||
progressBar: null,
|
||||
progressText: null,
|
||||
totalUsers: null,
|
||||
completedUsers: null,
|
||||
failedUsers: null,
|
||||
activeThreadsStat: null,
|
||||
activeThreads: null,
|
||||
userList: null,
|
||||
userCount: null,
|
||||
threadMonitorPanel: null,
|
||||
threadList: null,
|
||||
logContainer: null,
|
||||
logFilter: null,
|
||||
logSearch: null,
|
||||
screenshotPanel: null,
|
||||
screenshotImage: null,
|
||||
closeScreenshot: null
|
||||
};
|
||||
|
||||
// ========== 初始化 ==========
|
||||
function init() {
|
||||
// 获取 DOM 元素
|
||||
elements.startBtn = document.getElementById('startBtn');
|
||||
elements.stopBtn = document.getElementById('stopBtn');
|
||||
elements.clearLogsBtn = document.getElementById('clearLogsBtn');
|
||||
elements.exportDataBtn = document.getElementById('exportDataBtn');
|
||||
elements.themeToggle = document.getElementById('themeToggle');
|
||||
elements.themeIcon = document.getElementById('themeIcon');
|
||||
elements.modeRadios = document.querySelectorAll('input[name="mode"]');
|
||||
elements.threadControl = document.getElementById('threadControl');
|
||||
elements.threadCount = document.getElementById('threadCount');
|
||||
elements.progressBar = document.getElementById('progressBar');
|
||||
elements.progressText = document.getElementById('progressText');
|
||||
elements.totalUsers = document.getElementById('totalUsers');
|
||||
elements.completedUsers = document.getElementById('completedUsers');
|
||||
elements.failedUsers = document.getElementById('failedUsers');
|
||||
elements.activeThreadsStat = document.getElementById('activeThreadsStat');
|
||||
elements.activeThreads = document.getElementById('activeThreads');
|
||||
elements.userList = document.getElementById('userList');
|
||||
elements.userCount = document.getElementById('userCount');
|
||||
elements.threadMonitorPanel = document.getElementById('threadMonitorPanel');
|
||||
elements.threadList = document.getElementById('threadList');
|
||||
elements.logContainer = document.getElementById('logContainer');
|
||||
elements.logFilter = document.getElementById('logFilter');
|
||||
elements.logSearch = document.getElementById('logSearch');
|
||||
elements.screenshotPanel = document.getElementById('screenshotPanel');
|
||||
elements.screenshotImage = document.getElementById('screenshotImage');
|
||||
elements.closeScreenshot = document.getElementById('closeScreenshot');
|
||||
|
||||
// 加载主题
|
||||
loadTheme();
|
||||
|
||||
// 加载初始数据
|
||||
loadInitialData();
|
||||
|
||||
// 绑定事件
|
||||
bindEvents();
|
||||
|
||||
// 连接 SSE
|
||||
connectEventStream();
|
||||
}
|
||||
|
||||
// ========== 事件绑定 ==========
|
||||
function bindEvents() {
|
||||
// 启动按钮
|
||||
elements.startBtn.addEventListener('click', startScraping);
|
||||
|
||||
// 停止按钮
|
||||
elements.stopBtn.addEventListener('click', stopScraping);
|
||||
|
||||
// 清空日志
|
||||
elements.clearLogsBtn.addEventListener('click', clearLogs);
|
||||
|
||||
// 导出数据
|
||||
elements.exportDataBtn.addEventListener('click', exportData);
|
||||
|
||||
// 主题切换
|
||||
elements.themeToggle.addEventListener('click', toggleTheme);
|
||||
|
||||
// 模式切换
|
||||
elements.modeRadios.forEach(radio => {
|
||||
radio.addEventListener('change', (e) => {
|
||||
state.mode = e.target.value;
|
||||
updateModeUI();
|
||||
});
|
||||
});
|
||||
|
||||
// 线程数变化
|
||||
elements.threadCount.addEventListener('change', (e) => {
|
||||
state.maxThreads = parseInt(e.target.value);
|
||||
});
|
||||
|
||||
// 日志过滤
|
||||
elements.logFilter.addEventListener('change', (e) => {
|
||||
state.logFilter = e.target.value;
|
||||
renderLogs();
|
||||
});
|
||||
|
||||
// 日志搜索
|
||||
elements.logSearch.addEventListener('input', debounce((e) => {
|
||||
searchLogs(e.target.value);
|
||||
}, 300));
|
||||
|
||||
// 关闭截图
|
||||
elements.closeScreenshot.addEventListener('click', () => {
|
||||
elements.screenshotPanel.style.display = 'none';
|
||||
});
|
||||
|
||||
// 日志容器滚动监听（自动滚动控制）
|
||||
elements.logContainer.addEventListener('scroll', () => {
|
||||
const { scrollTop, scrollHeight, clientHeight } = elements.logContainer;
|
||||
state.autoScrollLogs = scrollTop + clientHeight >= scrollHeight - 10;
|
||||
});
|
||||
}
|
||||
|
||||
// ========== 加载初始数据 ==========
|
||||
async function loadInitialData() {
|
||||
try {
|
||||
// 加载状态
|
||||
const statusRes = await fetch('/api/status');
|
||||
const statusData = await statusRes.json();
|
||||
updateStatus(statusData);
|
||||
|
||||
// 加载用户列表
|
||||
const usersRes = await fetch('/api/users');
|
||||
const usersData = await usersRes.json();
|
||||
renderUserList(usersData.users);
|
||||
|
||||
// 加载历史日志
|
||||
const logsRes = await fetch('/api/logs?limit=100');
|
||||
const logsData = await logsRes.json();
|
||||
state.logs = logsData.logs;
|
||||
renderLogs();
|
||||
|
||||
} catch (error) {
|
||||
console.error('加载初始数据失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 启动采集 ==========
|
||||
async function startScraping() {
|
||||
try {
|
||||
const response = await fetch('/api/start', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
mode: state.mode,
|
||||
max_threads: state.maxThreads
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (response.ok) {
|
||||
state.isRunning = true;
|
||||
updateButtons();
|
||||
addLog('SUCCESS', data.message);
|
||||
} else {
|
||||
addLog('ERROR', data.detail || '启动失败');
|
||||
}
|
||||
} catch (error) {
|
||||
addLog('ERROR', `启动失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 停止采集 ==========
|
||||
async function stopScraping() {
|
||||
try {
|
||||
const response = await fetch('/api/stop', { method: 'POST' });
|
||||
const data = await response.json();
|
||||
|
||||
if (response.ok) {
|
||||
state.isRunning = false;
|
||||
updateButtons();
|
||||
addLog('WARNING', data.message);
|
||||
}
|
||||
} catch (error) {
|
||||
addLog('ERROR', `停止失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 清空日志 ==========
|
||||
async function clearLogs() {
|
||||
try {
|
||||
const response = await fetch('/api/logs/clear', { method: 'POST' });
|
||||
const data = await response.json();
|
||||
|
||||
if (response.ok) {
|
||||
state.logs = [];
|
||||
renderLogs();
|
||||
addLog('SUCCESS', '日志已清空');
|
||||
}
|
||||
} catch (error) {
|
||||
addLog('ERROR', `清空日志失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 导出数据 ==========
|
||||
async function exportData() {
|
||||
try {
|
||||
// 下载日志
|
||||
window.location.href = '/api/logs/download';
|
||||
addLog('SUCCESS', '正在下载日志文件...');
|
||||
} catch (error) {
|
||||
addLog('ERROR', `导出失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 搜索日志 ==========
|
||||
async function searchLogs(keyword) {
|
||||
if (!keyword) {
|
||||
// 加载所有日志
|
||||
const logsRes = await fetch('/api/logs?limit=100');
|
||||
const logsData = await logsRes.json();
|
||||
state.logs = logsData.logs;
|
||||
} else {
|
||||
// 搜索日志
|
||||
const searchRes = await fetch(`/api/logs/search?keyword=${encodeURIComponent(keyword)}&limit=100`);
|
||||
const searchData = await searchRes.json();
|
||||
state.logs = searchData.logs;
|
||||
}
|
||||
renderLogs();
|
||||
}
|
||||
|
||||
// ========== SSE 事件流 ==========
|
||||
function connectEventStream() {
|
||||
if (state.eventSource) {
|
||||
state.eventSource.close();
|
||||
}
|
||||
|
||||
state.eventSource = new EventSource('/api/events');
|
||||
|
||||
state.eventSource.onmessage = (event) => {
|
||||
if (event.data.startsWith(': heartbeat')) {
|
||||
return; // 忽略心跳
|
||||
}
|
||||
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
handleEvent(data);
|
||||
} catch (error) {
|
||||
console.error('解析事件失败:', error);
|
||||
}
|
||||
};
|
||||
|
||||
state.eventSource.onerror = (error) => {
|
||||
console.error('SSE 连接错误:', error);
|
||||
// 5秒后重连
|
||||
setTimeout(() => {
|
||||
if (!state.eventSource || state.eventSource.readyState === EventSource.CLOSED) {
|
||||
connectEventStream();
|
||||
}
|
||||
}, 5000);
|
||||
};
|
||||
}
|
||||
|
||||
// ========== 处理事件 ==========
|
||||
function handleEvent(event) {
|
||||
switch (event.type) {
|
||||
case 'log':
|
||||
addLog(event.level, event.message, event.thread_id, event.timestamp, false);
|
||||
break;
|
||||
|
||||
case 'progress':
|
||||
updateProgress(event);
|
||||
break;
|
||||
|
||||
case 'user_status':
|
||||
updateUserStatus(event);
|
||||
break;
|
||||
|
||||
case 'thread_status':
|
||||
updateThreadStatus(event);
|
||||
break;
|
||||
|
||||
case 'screenshot':
|
||||
showScreenshot(event.path);
|
||||
break;
|
||||
|
||||
case 'finished':
|
||||
state.isRunning = false;
|
||||
updateButtons();
|
||||
addLog('SUCCESS', '所有采集任务完成');
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
addLog('ERROR', event.message);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 更新状态 ==========
|
||||
function updateStatus(data) {
|
||||
state.isRunning = data.is_running;
|
||||
state.totalUsers = data.total_users;
|
||||
state.completedUsers = data.completed_users;
|
||||
state.failedUsers = data.failed_users;
|
||||
state.activeThreads = data.active_threads;
|
||||
state.usersStatus = data.users_status || {};
|
||||
|
||||
updateButtons();
|
||||
updateProgress();
|
||||
renderUserListFromStatus();
|
||||
}
|
||||
|
||||
// ========== 更新进度 ==========
|
||||
function updateProgress(data) {
|
||||
if (data) {
|
||||
state.completedUsers = data.completed || state.completedUsers;
|
||||
state.failedUsers = data.failed || state.failedUsers;
|
||||
state.activeThreads = data.active_threads || state.activeThreads;
|
||||
}
|
||||
|
||||
const total = state.totalUsers;
|
||||
const completed = state.completedUsers;
|
||||
const failed = state.failedUsers;
|
||||
const progress = total > 0 ? Math.round(((completed + failed) / total) * 100) : 0;
|
||||
|
||||
elements.progressBar.style.width = `${progress}%`;
|
||||
elements.progressText.textContent = `${progress}%`;
|
||||
elements.totalUsers.textContent = total;
|
||||
elements.completedUsers.textContent = completed;
|
||||
elements.failedUsers.textContent = failed;
|
||||
elements.activeThreads.textContent = state.activeThreads;
|
||||
|
||||
// 多线程模式显示活跃线程
|
||||
if (state.mode === 'multi') {
|
||||
elements.activeThreadsStat.style.display = 'flex';
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 更新用户状态 ==========
|
||||
function updateUserStatus(data) {
|
||||
state.usersStatus[data.user_id] = {
|
||||
status: data.status,
|
||||
note_count: data.note_count,
|
||||
thread_id: data.thread_id
|
||||
};
|
||||
|
||||
renderUserListFromStatus();
|
||||
}
|
||||
|
||||
// ========== 更新线程状态 ==========
|
||||
function updateThreadStatus(data) {
|
||||
state.threadStatus[data.thread_id] = {
|
||||
status: data.status,
|
||||
current_user: data.current_user
|
||||
};
|
||||
|
||||
renderThreadList();
|
||||
}
|
||||
|
||||
// ========== 渲染用户列表 ==========
|
||||
function renderUserList(users) {
|
||||
if (!users || users.length === 0) {
|
||||
elements.userList.innerHTML = `
|
||||
<div class="empty-state">
|
||||
<p>暂无用户数据</p>
|
||||
<p class="hint">请在 data/users.txt 中添加用户 ID</p>
|
||||
</div>
|
||||
`;
|
||||
elements.userCount.textContent = '0';
|
||||
return;
|
||||
}
|
||||
|
||||
elements.userCount.textContent = users.length;
|
||||
|
||||
elements.userList.innerHTML = users.map(user => {
|
||||
const status = user.status || 'pending';
|
||||
const noteCount = user.note_count || 0;
|
||||
|
||||
return `
|
||||
<div class="user-item" data-user-id="${user.id}">
|
||||
<div class="user-item-header">
|
||||
<span class="user-id">${user.id}</span>
|
||||
<div class="user-status">
|
||||
<span class="status-icon status-${status}"></span>
|
||||
<span>${getStatusText(status)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="user-meta">
|
||||
${noteCount > 0 ? `已采集 ${noteCount} 条笔记` : '等待采集'}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ========== 从状态渲染用户列表 ==========
|
||||
function renderUserListFromStatus() {
|
||||
const users = Object.keys(state.usersStatus).map(userId => ({
|
||||
id: userId,
|
||||
...state.usersStatus[userId]
|
||||
}));
|
||||
|
||||
renderUserList(users);
|
||||
}
|
||||
|
||||
// ========== 渲染线程列表 ==========
|
||||
function renderThreadList() {
|
||||
const threads = Object.entries(state.threadStatus);
|
||||
|
||||
if (threads.length === 0) {
|
||||
elements.threadList.innerHTML = `
|
||||
<div class="empty-state">
|
||||
<p>暂无线程活动</p>
|
||||
</div>
|
||||
`;
|
||||
return;
|
||||
}
|
||||
|
||||
elements.threadList.innerHTML = threads.map(([threadId, status]) => {
|
||||
const statusClass = status.status;
|
||||
const statusText = status.status === 'busy' ? '处理中' :
|
||||
status.status === 'idle' ? '空闲' : '错误';
|
||||
const currentUser = status.current_user || '-';
|
||||
|
||||
return `
|
||||
<div class="thread-item">
|
||||
<div class="thread-header">
|
||||
<span class="thread-name">${threadId}</span>
|
||||
<span class="thread-status ${statusClass}">${statusText}</span>
|
||||
</div>
|
||||
<div class="thread-current">${currentUser}</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ========== 添加日志 ==========
|
||||
function addLog(level, message, threadId = null, timestamp = null, save = true) {
|
||||
const logEntry = {
|
||||
timestamp: timestamp || new Date().toISOString(),
|
||||
level,
|
||||
message,
|
||||
thread_id: threadId
|
||||
};
|
||||
|
||||
state.logs.push(logEntry);
|
||||
|
||||
// 限制日志数量
|
||||
if (state.logs.length > 1000) {
|
||||
state.logs = state.logs.slice(-1000);
|
||||
}
|
||||
|
||||
renderLogs();
|
||||
}
|
||||
|
||||
// ========== 渲染日志 ==========
|
||||
function renderLogs() {
|
||||
let logs = state.logs;
|
||||
|
||||
// 过滤
|
||||
if (state.logFilter !== 'all') {
|
||||
logs = logs.filter(log => log.level === state.logFilter);
|
||||
}
|
||||
|
||||
elements.logContainer.innerHTML = logs.map(log => {
|
||||
const time = formatTime(log.timestamp);
|
||||
const thread = log.thread_id ? `<span class="log-thread">[${log.thread_id}]</span>` : '';
|
||||
|
||||
return `
|
||||
<div class="log-entry">
|
||||
<span class="log-time">${time}</span>
|
||||
<span class="log-level ${log.level}">${log.level}</span>
|
||||
<span class="log-message">${thread}${escapeHtml(log.message)}</span>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
|
||||
// 自动滚动到底部
|
||||
if (state.autoScrollLogs) {
|
||||
elements.logContainer.scrollTop = elements.logContainer.scrollHeight;
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 显示截图 ==========
|
||||
function showScreenshot(path) {
|
||||
elements.screenshotImage.src = path;
|
||||
elements.screenshotPanel.style.display = 'block';
|
||||
addLog('WARNING', `错误截图已生成: ${path}`);
|
||||
}
|
||||
|
||||
// ========== 更新按钮状态 ==========
|
||||
function updateButtons() {
|
||||
elements.startBtn.disabled = state.isRunning;
|
||||
elements.stopBtn.disabled = !state.isRunning;
|
||||
}
|
||||
|
||||
// ========== 更新模式 UI ==========
|
||||
function updateModeUI() {
|
||||
if (state.mode === 'multi') {
|
||||
elements.threadControl.style.display = 'flex';
|
||||
elements.threadMonitorPanel.style.display = 'block';
|
||||
document.querySelector('.main-content').classList.add('with-threads');
|
||||
} else {
|
||||
elements.threadControl.style.display = 'none';
|
||||
elements.threadMonitorPanel.style.display = 'none';
|
||||
document.querySelector('.main-content').classList.remove('with-threads');
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 主题相关 ==========
|
||||
function loadTheme() {
|
||||
const savedTheme = localStorage.getItem('theme') || 'light';
|
||||
if (savedTheme === 'dark') {
|
||||
document.body.classList.add('dark-theme');
|
||||
elements.themeIcon.textContent = '☀️';
|
||||
}
|
||||
}
|
||||
|
||||
function toggleTheme() {
|
||||
document.body.classList.toggle('dark-theme');
|
||||
const isDark = document.body.classList.contains('dark-theme');
|
||||
elements.themeIcon.textContent = isDark ? '☀️' : '🌙';
|
||||
localStorage.setItem('theme', isDark ? 'dark' : 'light');
|
||||
}
|
||||
|
||||
// ========== 工具函数 ==========
|
||||
function formatTime(isoString) {
|
||||
const date = new Date(isoString);
|
||||
return date.toLocaleTimeString('zh-CN', { hour12: false });
|
||||
}
|
||||
|
||||
function getStatusText(status) {
|
||||
const statusMap = {
|
||||
'pending': '待处理',
|
||||
'processing': '进行中',
|
||||
'completed': '已完成',
|
||||
'failed': '失败'
|
||||
};
|
||||
return statusMap[status] || status;
|
||||
}
|
||||
|
||||
function escapeHtml(text) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function debounce(func, wait) {
|
||||
let timeout;
|
||||
return function executedFunction(...args) {
|
||||
const later = () => {
|
||||
clearTimeout(timeout);
|
||||
func(...args);
|
||||
};
|
||||
clearTimeout(timeout);
|
||||
timeout = setTimeout(later, wait);
|
||||
};
|
||||
}
|
||||
|
||||
// ========== 启动 ==========
|
||||
document.addEventListener('DOMContentLoaded', init);
|
||||
@@ -0,0 +1,620 @@
|
||||
/* 小红书采集器样式表 */
|
||||
|
||||
/* ========== CSS 变量 ========== */
|
||||
:root {
|
||||
--bg-primary: #f5f7fa;
|
||||
--bg-secondary: #ffffff;
|
||||
--bg-tertiary: #e8ecf1;
|
||||
--text-primary: #2c3e50;
|
||||
--text-secondary: #5a6c7d;
|
||||
--text-muted: #8b95a5;
|
||||
--border-color: #d1d9e6;
|
||||
--primary-color: #ff2442; /* 小红书红 */
|
||||
--primary-hover: #e01f3a;
|
||||
--success-color: #10b981;
|
||||
--warning-color: #f59e0b;
|
||||
--error-color: #ef4444;
|
||||
--info-color: #3b82f6;
|
||||
--debug-color: #8b95a5;
|
||||
--shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
|
||||
--shadow-md: 0 4px 6px -1px rgba(0, 0, 0, 0.1);
|
||||
--shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.1);
|
||||
--radius-sm: 4px;
|
||||
--radius-md: 8px;
|
||||
--radius-lg: 12px;
|
||||
}
|
||||
|
||||
/* 深色主题 */
|
||||
body.dark-theme {
|
||||
--bg-primary: #1a1d23;
|
||||
--bg-secondary: #242830;
|
||||
--bg-tertiary: #2d333d;
|
||||
--text-primary: #e8eaed;
|
||||
--text-secondary: #9aa0a6;
|
||||
--text-muted: #5f6368;
|
||||
--border-color: #3c4043;
|
||||
--shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.3);
|
||||
--shadow-md: 0 4px 6px -1px rgba(0, 0, 0, 0.4);
|
||||
--shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
|
||||
/* ========== 基础重置 ========== */
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
|
||||
background: var(--bg-primary);
|
||||
color: var(--text-primary);
|
||||
line-height: 1.5;
|
||||
transition: background 0.3s, color 0.3s;
|
||||
}
|
||||
|
||||
/* ========== 布局 ========== */
|
||||
.container {
|
||||
max-width: 1600px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
/* ========== 头部 ========== */
|
||||
.header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 20px;
|
||||
padding: 20px;
|
||||
background: var(--bg-secondary);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 24px;
|
||||
font-weight: 600;
|
||||
color: var(--primary-color);
|
||||
}
|
||||
|
||||
.theme-toggle {
|
||||
background: var(--bg-tertiary);
|
||||
border: none;
|
||||
padding: 10px 16px;
|
||||
border-radius: var(--radius-md);
|
||||
cursor: pointer;
|
||||
font-size: 20px;
|
||||
transition: background 0.2s;
|
||||
}
|
||||
|
||||
.theme-toggle:hover {
|
||||
background: var(--border-color);
|
||||
}
|
||||
|
||||
/* ========== 控制面板 ========== */
|
||||
.control-panel {
|
||||
background: var(--bg-secondary);
|
||||
padding: 20px;
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.control-group {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.control-group:not(:last-child) {
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
/* ========== 按钮 ========== */
|
||||
.btn {
|
||||
padding: 10px 20px;
|
||||
border: none;
|
||||
border-radius: var(--radius-md);
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: var(--primary-color);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-primary:hover:not(:disabled) {
|
||||
background: var(--primary-hover);
|
||||
transform: translateY(-1px);
|
||||
box-shadow: var(--shadow-md);
|
||||
}
|
||||
|
||||
.btn-danger {
|
||||
background: var(--error-color);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-danger:hover:not(:disabled) {
|
||||
background: #dc2626;
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: var(--bg-tertiary);
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.btn-secondary:hover {
|
||||
background: var(--border-color);
|
||||
}
|
||||
|
||||
.btn-small {
|
||||
padding: 6px 12px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.btn-icon {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
/* ========== 单选按钮组 ========== */
|
||||
.radio-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.label-text {
|
||||
font-weight: 500;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.radio-group label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.radio-group input[type="radio"] {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* ========== 下拉选择 ========== */
|
||||
.select, .select-small {
|
||||
padding: 8px 12px;
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--bg-secondary);
|
||||
color: var(--text-primary);
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.select-small {
|
||||
padding: 4px 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* ========== 输入框 ========== */
|
||||
.input-small {
|
||||
padding: 4px 8px;
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--bg-secondary);
|
||||
color: var(--text-primary);
|
||||
font-size: 12px;
|
||||
width: 150px;
|
||||
}
|
||||
|
||||
/* ========== 线程控制 ========== */
|
||||
.thread-control {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.thread-control label {
|
||||
font-weight: 500;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
/* ========== 进度卡片 ========== */
|
||||
.progress-section {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.progress-card {
|
||||
background: var(--bg-secondary);
|
||||
padding: 20px;
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.progress-card h3 {
|
||||
margin-bottom: 16px;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.progress-bar-container {
|
||||
width: 100%;
|
||||
height: 24px;
|
||||
background: var(--bg-tertiary);
|
||||
border-radius: var(--radius-md);
|
||||
overflow: hidden;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, var(--primary-color), #ff6b8a);
|
||||
border-radius: var(--radius-md);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transition: width 0.3s ease;
|
||||
min-width: 40px;
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
color: white;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.progress-stats {
|
||||
display: flex;
|
||||
gap: 24px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.stat-item {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.stat-label {
|
||||
font-size: 12px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.stat-value {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.stat-success {
|
||||
color: var(--success-color);
|
||||
}
|
||||
|
||||
.stat-error {
|
||||
color: var(--error-color);
|
||||
}
|
||||
|
||||
.stat-info {
|
||||
color: var(--info-color);
|
||||
}
|
||||
|
||||
/* ========== 主内容区 ========== */
|
||||
.main-content {
|
||||
display: grid;
|
||||
grid-template-columns: 300px 1fr;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.main-content.with-threads {
|
||||
grid-template-columns: 300px 250px 1fr;
|
||||
}
|
||||
|
||||
/* ========== 面板 ========== */
|
||||
.panel {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
max-height: 500px;
|
||||
}
|
||||
|
||||
.panel-header {
|
||||
padding: 16px 20px;
|
||||
border-bottom: 1px solid var(--border-color);
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.panel-header h2 {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.badge {
|
||||
background: var(--primary-color);
|
||||
color: white;
|
||||
padding: 2px 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.panel-body {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
/* ========== 用户列表 ========== */
|
||||
.user-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.user-item {
|
||||
padding: 12px;
|
||||
background: var(--bg-tertiary);
|
||||
border-radius: var(--radius-md);
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.user-item:hover {
|
||||
background: var(--border-color);
|
||||
}
|
||||
|
||||
.user-item-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.user-id {
|
||||
font-family: monospace;
|
||||
font-size: 13px;
|
||||
color: var(--text-primary);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.user-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.status-icon {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
}
|
||||
|
||||
.status-pending { background: var(--text-muted); }
|
||||
.status-processing { background: var(--info-color); }
|
||||
.status-completed { background: var(--success-color); }
|
||||
.status-failed { background: var(--error-color); }
|
||||
|
||||
.user-meta {
|
||||
font-size: 12px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
text-align: center;
|
||||
padding: 40px 20px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.hint {
|
||||
font-size: 12px;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
/* ========== 线程监控 ========== */
|
||||
.thread-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.thread-item {
|
||||
padding: 12px;
|
||||
background: var(--bg-tertiary);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
|
||||
.thread-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.thread-name {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.thread-status {
|
||||
font-size: 11px;
|
||||
padding: 2px 6px;
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--bg-secondary);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.thread-status.busy {
|
||||
background: var(--info-color);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.thread-status.error {
|
||||
background: var(--error-color);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.thread-current {
|
||||
font-size: 12px;
|
||||
color: var(--text-muted);
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
/* ========== 日志面板 ========== */
|
||||
.log-controls {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.log-container {
|
||||
font-family: 'Consolas', 'Monaco', monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.log-entry {
|
||||
padding: 6px 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
margin-bottom: 4px;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.log-entry:hover {
|
||||
background: var(--bg-tertiary);
|
||||
}
|
||||
|
||||
.log-time {
|
||||
color: var(--text-muted);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.log-level {
|
||||
font-weight: 600;
|
||||
flex-shrink: 0;
|
||||
padding: 0 6px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 11px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.log-level.INFO {
|
||||
background: rgba(59, 130, 246, 0.15);
|
||||
color: var(--info-color);
|
||||
}
|
||||
|
||||
.log-level.SUCCESS {
|
||||
background: rgba(16, 185, 129, 0.15);
|
||||
color: var(--success-color);
|
||||
}
|
||||
|
||||
.log-level.WARNING {
|
||||
background: rgba(245, 158, 11, 0.15);
|
||||
color: var(--warning-color);
|
||||
}
|
||||
|
||||
.log-level.ERROR {
|
||||
background: rgba(239, 68, 68, 0.15);
|
||||
color: var(--error-color);
|
||||
}
|
||||
|
||||
.log-level.DEBUG {
|
||||
background: rgba(139, 149, 165, 0.15);
|
||||
color: var(--debug-color);
|
||||
}
|
||||
|
||||
.log-message {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.log-thread {
|
||||
color: var(--primary-color);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ========== 截图面板 ========== */
|
||||
.screenshot-panel {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
overflow: hidden;
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.screenshot-panel img {
|
||||
width: 100%;
|
||||
height: auto;
|
||||
display: block;
|
||||
}
|
||||
|
||||
/* ========== 滚动条样式 ========== */
|
||||
.panel-body::-webkit-scrollbar,
|
||||
.log-container::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
|
||||
.panel-body::-webkit-scrollbar-track,
|
||||
.log-container::-webkit-scrollbar-track {
|
||||
background: var(--bg-tertiary);
|
||||
}
|
||||
|
||||
.panel-body::-webkit-scrollbar-thumb,
|
||||
.log-container::-webkit-scrollbar-thumb {
|
||||
background: var(--border-color);
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.panel-body::-webkit-scrollbar-thumb:hover,
|
||||
.log-container::-webkit-scrollbar-thumb:hover {
|
||||
background: var(--text-muted);
|
||||
}
|
||||
|
||||
/* ========== 响应式 ========== */
|
||||
@media (max-width: 1200px) {
|
||||
.main-content {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.main-content.with-threads {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.thread-monitor-panel {
|
||||
display: none !important;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.control-group {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.progress-stats {
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.stat-value {
|
||||
font-size: 16px;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>小红书数据采集器</title>
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<!-- 顶部标题栏 -->
|
||||
<header class="header">
|
||||
<h1>小红书数据采集器</h1>
|
||||
<button class="theme-toggle" id="themeToggle" title="切换主题">
|
||||
<span id="themeIcon">🌙</span>
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<!-- 控制面板 -->
|
||||
<section class="control-panel">
|
||||
<div class="control-group">
|
||||
<button class="btn btn-primary" id="startBtn">
|
||||
<span class="btn-icon">▶</span> 启动采集
|
||||
</button>
|
||||
<button class="btn btn-danger" id="stopBtn" disabled>
|
||||
<span class="btn-icon">⏹</span> 停止
|
||||
</button>
|
||||
<button class="btn btn-secondary" id="clearLogsBtn">
|
||||
<span class="btn-icon">🗑</span> 清空日志
|
||||
</button>
|
||||
<button class="btn btn-secondary" id="exportDataBtn">
|
||||
<span class="btn-icon">📥</span> 导出数据
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
<label class="radio-group">
|
||||
<span class="label-text">模式:</span>
|
||||
<label>
|
||||
<input type="radio" name="mode" value="single" checked>
|
||||
<span>单线程</span>
|
||||
</label>
|
||||
<label>
|
||||
<input type="radio" name="mode" value="multi">
|
||||
<span>多线程</span>
|
||||
</label>
|
||||
</label>
|
||||
|
||||
<div class="thread-control" id="threadControl" style="display: none;">
|
||||
<label for="threadCount">线程数:</label>
|
||||
<select id="threadCount" class="select">
|
||||
<option value="1">1</option>
|
||||
<option value="2">2</option>
|
||||
<option value="3" selected>3</option>
|
||||
<option value="4">4</option>
|
||||
<option value="5">5</option>
|
||||
<option value="6">6</option>
|
||||
<option value="8">8</option>
|
||||
<option value="10">10</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- 进度概览 -->
|
||||
<section class="progress-section">
|
||||
<div class="progress-card">
|
||||
<h3>整体进度</h3>
|
||||
<div class="progress-bar-container">
|
||||
<div class="progress-bar" id="progressBar">
|
||||
<span class="progress-text" id="progressText">0%</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="progress-stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">总数</span>
|
||||
<span class="stat-value" id="totalUsers">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">已完成</span>
|
||||
<span class="stat-value stat-success" id="completedUsers">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">失败</span>
|
||||
<span class="stat-value stat-error" id="failedUsers">0</span>
|
||||
</div>
|
||||
<div class="stat-item" id="activeThreadsStat" style="display: none;">
|
||||
<span class="stat-label">活跃线程</span>
|
||||
<span class="stat-value stat-info" id="activeThreads">0</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- 主内容区 -->
|
||||
<div class="main-content">
|
||||
<!-- 左侧：用户列表 -->
|
||||
<section class="panel user-list-panel">
|
||||
<div class="panel-header">
|
||||
<h2>用户列表</h2>
|
||||
<span class="badge" id="userCount">0</span>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="user-list" id="userList">
|
||||
<!-- 用户项将动态添加 -->
|
||||
<div class="empty-state">
|
||||
<p>暂无用户数据</p>
|
||||
<p class="hint">请在 data/users.txt 中添加用户 ID</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- 中间：线程监控（多线程模式） -->
|
||||
<section class="panel thread-monitor-panel" id="threadMonitorPanel" style="display: none;">
|
||||
<div class="panel-header">
|
||||
<h2>线程监控</h2>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="thread-list" id="threadList">
|
||||
<!-- 线程状态将动态添加 -->
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- 右侧：日志面板 -->
|
||||
<section class="panel log-panel">
|
||||
<div class="panel-header">
|
||||
<h2>日志</h2>
|
||||
<div class="log-controls">
|
||||
<select id="logFilter" class="select-small">
|
||||
<option value="all">ALL</option>
|
||||
<option value="INFO">INFO</option>
|
||||
<option value="SUCCESS">SUCCESS</option>
|
||||
<option value="WARNING">WARNING</option>
|
||||
<option value="ERROR">ERROR</option>
|
||||
<option value="DEBUG">DEBUG</option>
|
||||
</select>
|
||||
<input type="text" id="logSearch" placeholder="搜索日志..." class="input-small">
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="log-container" id="logContainer">
|
||||
<!-- 日志将动态添加 -->
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<!-- 错误截图区 -->
|
||||
<section class="screenshot-panel" id="screenshotPanel" style="display: none;">
|
||||
<div class="panel-header">
|
||||
<h2>错误截图</h2>
|
||||
<button class="btn-small btn-secondary" id="closeScreenshot">关闭</button>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<img id="screenshotImage" src="" alt="错误截图">
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<script src="/static/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user