first commit

app/__init__.py

@@ -0,0 +1,2 @@
+# XiaoHongShu Scraper Package
+__version__ = "1.0.0"

app/config.py

@@ -0,0 +1,115 @@
+"""
+XiaoHongShu Scraper Configuration
+采集器配置文件
+"""
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+# ========== 基础配置 ==========
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DATA_DIR = os.path.join(BASE_DIR, "data")
+SCREENSHOT_DIR = os.path.join(DATA_DIR, "screenshots")
+
+# 确保目录存在
+os.makedirs(DATA_DIR, exist_ok=True)
+os.makedirs(SCREENSHOT_DIR, exist_ok=True)
+
+
+# ========== 采集字段配置 ==========
+FIELDS = ['title', 'publish_time', 'location', 'comment_count', 'like_count', 'collect_count', 'note_url']
+
+
+# ========== 延迟配置（秒）==========
+PAGE_LOAD_DELAY = (2, 5)        # 页面加载后等待
+NOTE_READ_DELAY = (3, 8)        # 笔记详情页停留（模拟阅读）
+NOTE_INTERVAL_DELAY = (3, 6)    # 采集完一个笔记后的间隔
+USER_SWITCH_DELAY = (30, 60)    # 切换用户时的间隔
+
+
+# ========== 多线程配置 ==========
+THREAD_MIN_DELAY = 2            # 线程间最小间隔（秒）
+THREAD_OFFSET = 5               # 线程启动偏移（秒）
+MAX_CONCURRENT_THREADS = 10     # 最大线程数
+
+
+# ========== 滚动配置 ==========
+SCROLL_STEP_MIN = 100           # 每次滚动最小像素
+SCROLL_STEP_MAX = 300           # 每次滚动最大像素
+SCROLL_PAUSE_MIN = 0.5          # 滚动后暂停最小时间（秒）
+SCROLL_PAUSE_MAX = 2.0          # 滚动后暂停最大时间（秒）
+MAX_SCROLL_COUNT = 50           # 最大滚动次数（防止无限滚动）
+
+
+# ========== 风控检测关键词 ==========
+RISK_KEYWORDS = [
+    '验证码',
+    '访问过于频繁',
+    '页面不存在',
+    '验证',
+    '流量异常',
+    '安全验证',
+    '人机验证',
+    '滑块验证'
+]
+
+
+# ========== 小红书 URL 配置 ==========
+XIAOHONGSHU_BASE_URL = "https://www.xiaohongshu.com"
+USER_PROFILE_URL_TEMPLATE = f"{XIAOHONGSHU_BASE_URL}/user/profile/{{user_id}}"
+
+
+# ========== 浏览器配置 ==========
+HEADLESS = os.getenv("HEADLESS", "false").lower() == "true"
+BROWSER_ARGS = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-blink-features=AutomationControlled'
+]
+
+# 视口配置
+VIEWPORT_WIDTH = 1920
+VIEWPORT_HEIGHT = 1080
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+
+
+# ========== 日志配置 ==========
+LOG_FILE = os.path.join(DATA_DIR, "logs.json")
+MAX_LOGS = 10000                # 最大日志条数
+
+
+# ========== Web 配置 ==========
+WEB_HOST = os.getenv("HOST", "0.0.0.0")
+WEB_PORT = int(os.getenv("PORT", "18080"))  # 默认端口 18080，范围 10000-50000
+SCREENSHOT_ON_ERROR = os.getenv("SCREENSHOT_ON_ERROR", "true").lower() == "true"
+
+# ========== Git 更新配置 ==========
+ENABLE_GIT_PULL = os.getenv("ENABLE_GIT_PULL", "false").lower() == "true"
+GIT_REPO_URL = os.getenv("GIT_REPO_URL", "")
+GIT_BRANCH = os.getenv("GIT_BRANCH", "main")
+AUTO_PULL_INTERVAL = int(os.getenv("AUTO_PULL_INTERVAL", "3600"))  # 自动拉取间隔（秒），默认1小时
+
+
+# ========== CSS 选择器（可能需要根据小红书页面结构调整）==========
+# 用户主页
+SELECTOR_USER_CONTAINER = "div.user-notes"
+SELECTOR_NOTE_CARD = "div[class*='note-item'], section[class*='note']"
+SELECTOR_NOTE_LINK = "a[href*='/explore/']"
+SELECTOR_NOTE_TITLE = "span[class*='title'], div[class*='title']"
+SELECTOR_NOTE_CONTENT = "div[class*='content'], span[class*='desc']"
+
+# 笔记详情页
+SELECTOR_DETAIL_TITLE = "div[class*='title'], span[class*='title']"
+SELECTOR_DETAIL_PUBLISH_TIME = "span[class*='time'], div[class*='time']"
+SELECTOR_DETAIL_LOCATION = "span[class*='location'], div[class*='location']"
+SELECTOR_DETAIL_COMMENT_COUNT = "span[class*='comment'], button[class*='comment']"
+SELECTOR_DETAIL_LIKE_COUNT = "span[class*='like'], button[class*='like']"
+SELECTOR_DETAIL_COLLECT_COUNT = "span[class*='collect'], button[class*='star']"
+
+
+# ========== 登录配置 ==========
+LOGIN_URL = f"{XIAOHONGSHU_BASE_URL}/login"
+AUTH_STATE_FILE = os.path.join(DATA_DIR, "auth_state.json")
+LOGIN_TIMEOUT = 300             # 登录等待超时（秒）

app/log_manager.py

@@ -0,0 +1,153 @@
+"""
+Log Manager - 持久化日志管理器
+支持日志加载、保存、追加、清空
+"""
+import json
+import os
+import threading
+from datetime import datetime
+from typing import List, Dict, Optional
+from app import config
+
+
+class LogManager:
+    """日志管理器 - 线程安全"""
+
+    def __init__(self, log_file: str = None, max_logs: int = None):
+        self.log_file = log_file or config.LOG_FILE
+        self.max_logs = max_logs or config.MAX_LOGS
+        self._lock = threading.Lock()
+        self.logs: List[Dict] = []
+        self._load_logs()
+
+    def _load_logs(self):
+        """启动时加载历史日志"""
+        with self._lock:
+            if os.path.exists(self.log_file):
+                try:
+                    with open(self.log_file, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                        self.logs = data.get('logs', [])
+                except (json.JSONDecodeError, IOError) as e:
+                    print(f"[LogManager] 加载日志文件失败: {e}")
+                    self.logs = []
+            else:
+                self.logs = []
+
+    def append(self, level: str, message: str, thread_id: Optional[str] = None) -> Dict:
+        """
+        追加新日志
+        返回日志条目
+        """
+        with self._lock:
+            log_entry = {
+                "timestamp": datetime.now().isoformat(),
+                "level": level,
+                "message": message,
+                "thread_id": thread_id
+            }
+            self.logs.append(log_entry)
+
+            # 限制日志数量（保留最新的 max_logs 条）
+            if len(self.logs) > self.max_logs:
+                self.logs = self.logs[-self.max_logs:]
+
+            # 异步保存到文件
+            self._save()
+
+            return log_entry
+
+    def _save(self):
+        """保存到文件（需在锁内调用）"""
+        try:
+            with open(self.log_file, 'w', encoding='utf-8') as f:
+                json.dump({
+                    "logs": self.logs,
+                    "last_updated": datetime.now().isoformat()
+                }, f, ensure_ascii=False, indent=2)
+        except IOError as e:
+            print(f"[LogManager] 保存日志文件失败: {e}")
+
+    def get_logs(self, level: Optional[str] = None, limit: Optional[int] = None) -> List[Dict]:
+        """
+        获取日志
+        :param level: 过滤级别 (None/INFO/SUCCESS/WARNING/ERROR/DEBUG)
+        :param limit: 限制返回数量
+        """
+        with self._lock:
+            if level:
+                filtered = [log for log in self.logs if log['level'] == level]
+            else:
+                filtered = self.logs
+
+            if limit:
+                return filtered[-limit:]
+            return filtered
+
+    def search_logs(self, keyword: str, limit: Optional[int] = None) -> List[Dict]:
+        """
+        搜索日志
+        :param keyword: 搜索关键词
+        :param limit: 限制返回数量
+        """
+        with self._lock:
+            filtered = [
+                log for log in self.logs
+                if keyword.lower() in log['message'].lower()
+            ]
+
+            if limit:
+                return filtered[-limit:]
+            return filtered
+
+    def clear(self):
+        """清空日志"""
+        with self._lock:
+            self.logs = []
+            self._save()
+
+    def get_stats(self) -> Dict:
+        """获取日志统计信息"""
+        with self._lock:
+            stats = {
+                "total": len(self.logs),
+                "by_level": {}
+            }
+
+            for log in self.logs:
+                level = log['level']
+                stats["by_level"][level] = stats["by_level"].get(level, 0) + 1
+
+            return stats
+
+    def export_to_text(self) -> str:
+        """导出为文本格式"""
+        with self._lock:
+            lines = []
+            lines.append("=" * 80)
+            lines.append("小红书采集器日志")
+            lines.append(f"导出时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+            lines.append(f"总日志数: {len(self.logs)}")
+            lines.append("=" * 80)
+            lines.append("")
+
+            for log in self.logs:
+                timestamp = log['timestamp'][:19]  # 去掉微秒
+                level = log['level'].ljust(8)
+                thread = f"[{log['thread_id']}]" if log.get('thread_id') else ""
+                message = log['message']
+                lines.append(f"{timestamp} | {level} | {thread} {message}")
+
+            return "\n".join(lines)
+
+
+# 全局日志管理器实例
+_log_manager_instance = None
+
+
+def get_log_manager() -> LogManager:
+    """获取全局日志管理器实例"""
+    global _log_manager_instance
+    if _log_manager_instance is None:
+        _log_manager_instance = LogManager()
+    return _log_manager_instance

app/scraper.py

@@ -0,0 +1,641 @@
+"""
+XiaoHongShu Scraper - 核心采集逻辑
+支持单线程和多线程模式
+"""
+import asyncio
+import json
+import os
+import random
+from datetime import datetime
+from typing import List, Dict, Optional, AsyncIterator
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page, TimeoutError as PlaywrightTimeoutError
+from app import config
+from app.log_manager import LogManager
+
+
+class RiskControlException(Exception):
+    """风控异常"""
+    def __init__(self, reason: str):
+        self.reason = reason
+        super().__init__(f"触发风控: {reason}")
+
+
+class XiaoHongShuScraper:
+    """小红书采集器"""
+
+    def __init__(self, event_queue: asyncio.Queue, log_manager: LogManager):
+        self.event_queue = event_queue
+        self.log_manager = log_manager
+        self.playwright = None
+        self.browser: Optional[Browser] = None
+        self.is_running = False
+        self._stop_requested = False
+
+    async def init_browser(self):
+        """初始化浏览器"""
+        self.playwright = await async_playwright().start()
+        self.browser = await self.playwright.chromium.launch(
+            headless=config.HEADLESS,
+            args=config.BROWSER_ARGS
+        )
+        await self.emit_log("SUCCESS", "浏览器初始化完成")
+
+    async def close_browser(self):
+        """关闭浏览器"""
+        if self.browser:
+            await self.browser.close()
+        if self.playwright:
+            await self.playwright.stop()
+        await self.emit_log("INFO", "浏览器已关闭")
+
+    async def emit_log(self, level: str, message: str, thread_id: Optional[str] = None):
+        """发送日志事件"""
+        # 持久化到日志管理器
+        log_entry = self.log_manager.append(level, message, thread_id)
+
+        # 推送到 Web 前端
+        await self.event_queue.put({
+            "type": "log",
+            **log_entry
+        })
+
+    async def emit_progress(self, **kwargs):
+        """发送进度更新"""
+        await self.event_queue.put({
+            "type": "progress",
+            **kwargs
+        })
+
+    async def emit_user_status(self, user_id: str, status: str, note_count: int = 0, thread_id: Optional[str] = None):
+        """发送用户状态"""
+        await self.event_queue.put({
+            "type": "user_status",
+            "user_id": user_id,
+            "status": status,
+            "note_count": note_count,
+            "thread_id": thread_id
+        })
+
+    async def emit_thread_status(self, thread_id: str, status: str, current_user: Optional[str] = None):
+        """发送线程状态"""
+        await self.event_queue.put({
+            "type": "thread_status",
+            "thread_id": thread_id,
+            "status": status,
+            "current_user": current_user
+        })
+
+    async def handle_login(self, context: BrowserContext) -> bool:
+        """
+        处理登录
+        返回是否已登录
+        """
+        page = await context.new_page()
+        await page.goto(config.LOGIN_URL)
+
+        await self.emit_log("WARNING", "请在浏览器中完成扫码登录...")
+        await self.emit_log("INFO", "等待登录确认...")
+
+        # 等待登录成功（检查 URL 变化或特定元素）
+        try:
+            # 等待跳转到首页或出现用户头像
+            await page.wait_for_url(
+                f"{config.XIAOHONGSHU_BASE_URL}/**",
+                timeout=config.LOGIN_TIMEOUT * 1000
+            )
+            await self.emit_log("SUCCESS", "登录成功！")
+
+            # 保存登录状态
+            await context.storage_state(path=config.AUTH_STATE_FILE)
+            await self.emit_log("INFO", "登录状态已保存")
+
+            await page.close()
+            return True
+
+        except PlaywrightTimeoutError:
+            await self.emit_log("ERROR", "登录超时，请重试")
+            await page.close()
+            return False
+
+    async def copy_cookies(self, target_context: BrowserContext, source_context: BrowserContext):
+        """复制 Cookies 到目标 Context"""
+        cookies = await source_context.cookies()
+        await target_context.add_cookies(cookies)
+
+    async def check_risk_control(self, page: Page) -> tuple[bool, str]:
+        """
+        检测页面是否触发风控
+        返回 (是否触发, 触发原因)
+        """
+        try:
+            content = await page.content()
+            for keyword in config.RISK_KEYWORDS:
+                if keyword in content:
+                    return True, f"检测到关键词: {keyword}"
+            return False, ""
+        except Exception as e:
+            return False, f"检测失败: {str(e)}"
+
+    async def human_like_scroll(self, page: Page, thread_id: Optional[str] = None):
+        """
+        模拟人类间歇性平滑滚动
+        """
+        await self.emit_log("DEBUG", f"[{thread_id}] 开始平滑滚动加载笔记...", thread_id=thread_id)
+
+        scroll_count = 0
+        last_height = 0
+
+        while scroll_count < config.MAX_SCROLL_COUNT:
+            # 获取当前页面高度
+            current_height = await page.evaluate("document.body.scrollHeight")
+
+            # 如果高度没有变化，可能到底了
+            if current_height == last_height:
+                await self.emit_log("DEBUG", f"[{thread_id}] 已滚动到底部", thread_id=thread_id)
+                break
+
+            last_height = current_height
+
+            # 随机滚动距离
+            scroll_step = random.randint(config.SCROLL_STEP_MIN, config.SCROLL_STEP_MAX)
+            scroll_position = await page.evaluate("window.scrollY")
+
+            # 平滑滚动
+            await page.evaluate(f'''
+                window.scrollTo({{
+                    top: {scroll_position + scroll_step},
+                    behavior: 'smooth'
+                }});
+            ''')
+
+            # 随机暂停
+            pause_time = random.uniform(config.SCROLL_PAUSE_MIN, config.SCROLL_PAUSE_MAX)
+            await asyncio.sleep(pause_time)
+
+            scroll_count += 1
+
+            # 每滚动几次检查一下风控
+            if scroll_count % 5 == 0:
+                is_risk, reason = await self.check_risk_control(page)
+                if is_risk:
+                    raise RiskControlException(reason)
+
+        await self.emit_log("DEBUG", f"[{thread_id}] 滚动完成，共滚动 {scroll_count} 次", thread_id=thread_id)
+
+    async def extract_note_cards(self, page: Page, thread_id: Optional[str] = None) -> List[Dict]:
+        """
+        从用户主页提取笔记卡片
+        返回 [{'url': '...', 'title': '...'}]
+        """
+        await self.emit_log("DEBUG", f"[{thread_id}] 提取笔记卡片...", thread_id=thread_id)
+
+        note_urls = []
+
+        try:
+            # 尝试多种选择器
+            selectors = [
+                "a[href*='/explore/']",
+                "section[class*='note'] a[href*='/explore/']",
+                "div[class*='note-item'] a[href*='/explore/']",
+            ]
+
+            links = []
+            for selector in selectors:
+                try:
+                    elements = await page.query_selector_all(selector)
+                    if elements:
+                        links = elements
+                        await self.emit_log("DEBUG", f"[{thread_id}] 使用选择器: {selector}", thread_id=thread_id)
+                        break
+                except Exception:
+                    continue
+
+            # 提取 URL
+            for link in links:
+                url = await link.get_attribute('href')
+                if url and '/explore/' in url:
+                    # 补全 URL
+                    if url.startswith('/'):
+                        url = f"{config.XIAOHONGSHU_BASE_URL}{url}"
+
+                    # 尝试获取标题
+                    try:
+                        title_elem = await link.query_selector("span, div")
+                        title = await title_elem.inner_text() if title_elem else "无标题"
+                    except Exception:
+                        title = "无标题"
+
+                    note_urls.append({
+                        'url': url,
+                        'title': title[:50]  # 限制标题长度
+                    })
+
+            await self.emit_log("INFO", f"[{thread_id}] 提取到 {len(note_urls)} 条笔记", thread_id=thread_id)
+
+            return note_urls
+
+        except Exception as e:
+            await self.emit_log("ERROR", f"[{thread_id}] 提取笔记卡片失败: {str(e)}", thread_id=thread_id)
+            return []
+
+    async def extract_note_detail(self, page: Page, thread_id: Optional[str] = None) -> Optional[Dict]:
+        """
+        从笔记详情页提取数据
+        """
+        try:
+            # 等待页面加载
+            await asyncio.sleep(random.uniform(*config.PAGE_LOAD_DELAY))
+
+            # 检查风控
+            is_risk, reason = await self.check_risk_control(page)
+            if is_risk:
+                raise RiskControlException(reason)
+
+            data = {}
+
+            # 提取标题
+            try:
+                title_elem = await page.query_selector(config.SELECTOR_DETAIL_TITLE)
+                data['title'] = await title_elem.inner_text() if title_elem else "无标题"
+            except Exception:
+                data['title'] = "无标题"
+
+            # 提取发布时间
+            try:
+                time_elem = await page.query_selector(config.SELECTOR_DETAIL_PUBLISH_TIME)
+                data['publish_time'] = await time_elem.inner_text() if time_elem else ""
+            except Exception:
+                data['publish_time'] = ""
+
+            # 提取发布地
+            try:
+                loc_elem = await page.query_selector(config.SELECTOR_DETAIL_LOCATION)
+                data['location'] = await loc_elem.inner_text() if loc_elem else ""
+            except Exception:
+                data['location'] = ""
+
+            # 提取互动数据（尝试从页面内容解析）
+            content = await page.content()
+
+            # 评论数
+            data['comment_count'] = self._extract_count(content, ['评论', 'comment'])
+
+            # 点赞数
+            data['like_count'] = self._extract_count(content, ['点赞', '赞', 'like'])
+
+            # 收藏数
+            data['collect_count'] = self._extract_count(content, ['收藏', 'star', 'collect'])
+
+            # 笔记链接
+            data['note_url'] = page.url
+
+            return data
+
+        except RiskControlException:
+            raise
+        except Exception as e:
+            await self.emit_log("ERROR", f"[{thread_id}] 提取详情失败: {str(e)}", thread_id=thread_id)
+            return None
+
+    def _extract_count(self, content: str, keywords: List[str]) -> int:
+        """从页面内容提取计数"""
+        import re
+        for keyword in keywords:
+            # 匹配数字
+            pattern = f"{keyword}[\\s:：]*([\\d.]+)"
+            match = re.search(pattern, content)
+            if match:
+                count_str = match.group(1)
+                # 处理万、k等单位
+                if '万' in count_str:
+                    return int(float(count_str.replace('万', '')) * 10000)
+                elif 'k' in count_str.lower():
+                    return int(float(count_str.lower().replace('k', '')) * 1000)
+                else:
+                    return int(count_str)
+        return 0
+
+    async def save_user_data(self, user_id: str, new_notes: List[Dict]):
+        """保存用户数据（增量）"""
+        file_path = os.path.join(config.DATA_DIR, f"{user_id}.json")
+
+        # 读取现有数据
+        existing_notes = []
+        if os.path.exists(file_path):
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    existing_notes = data.get('notes', [])
+            except Exception as e:
+                await self.emit_log("WARNING", f"读取现有数据失败: {e}")
+
+        # 提取现有 URL
+        existing_urls = {note['note_url'] for note in existing_notes}
+
+        # 追加新笔记
+        added_count = 0
+        for note in new_notes:
+            if note['note_url'] not in existing_urls:
+                existing_notes.append(note)
+                existing_urls.add(note['note_url'])
+                added_count += 1
+
+        # 保存
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump({
+                "user_id": user_id,
+                "last_updated": datetime.now().isoformat(),
+                "total_notes": len(existing_notes),
+                "notes": existing_notes
+            }, f, ensure_ascii=False, indent=2)
+
+        return added_count
+
+    async def process_user(self, context: BrowserContext, user_id: str, thread_id: str = "Main") -> int:
+        """
+        处理单个用户
+        返回新增笔记数
+        """
+        await self.emit_user_status(user_id, "processing", thread_id=thread_id)
+        await self.emit_thread_status(thread_id, "busy", user_id)
+
+        try:
+            await self.emit_log("INFO", f"[{thread_id}] 开始处理用户: {user_id}", thread_id=thread_id)
+
+            # 创建新页面
+            page = await context.new_page()
+
+            # 访问用户主页
+            url = config.USER_PROFILE_URL_TEMPLATE.format(user_id=user_id)
+            await self.emit_log("INFO", f"[{thread_id}] 访问用户主页: {url}", thread_id=thread_id)
+
+            try:
+                await page.goto(url, timeout=30000)
+            except PlaywrightTimeoutError:
+                await self.emit_log("ERROR", f"[{thread_id}] 访问用户主页超时: {user_id}", thread_id=thread_id)
+                await page.close()
+                await self.emit_user_status(user_id, "failed", thread_id=thread_id)
+                await self.emit_thread_status(thread_id, "idle", None)
+                return 0
+
+            # 检查风控
+            is_risk, reason = await self.check_risk_control(page)
+            if is_risk:
+                await self.emit_log("ERROR", f"[{thread_id}] 访问用户主页触发风控: {reason}", thread_id=thread_id)
+                await page.close()
+                await self.emit_user_status(user_id, "failed", thread_id=thread_id)
+                await self.emit_thread_status(thread_id, "idle", None)
+                raise RiskControlException(reason)
+
+            await asyncio.sleep(random.uniform(*config.PAGE_LOAD_DELAY))
+
+            # 滚动加载所有笔记
+            await self.human_like_scroll(page, thread_id)
+
+            # 提取笔记卡片
+            note_cards = await self.extract_note_cards(page, thread_id)
+
+            if not note_cards:
+                await self.emit_log("WARNING", f"[{thread_id}] 未找到笔记，可能页面结构变化", thread_id=thread_id)
+                await page.close()
+                await self.emit_user_status(user_id, "completed", 0, thread_id=thread_id)
+                await self.emit_thread_status(thread_id, "idle", None)
+                return 0
+
+            # 采集每条笔记
+            collected_notes = []
+            for i, card in enumerate(note_cards):
+                if self._stop_requested:
+                    await self.emit_log("WARNING", f"[{thread_id}] 收到停止信号", thread_id=thread_id)
+                    break
+
+                await self.emit_log("INFO", f"[{thread_id}] 采集笔记 {i+1}/{len(note_cards)}: {card['title']}", thread_id=thread_id)
+
+                try:
+                    # 访问详情页
+                    await page.goto(card['url'], timeout=30000)
+
+                    # 模拟阅读停留
+                    await asyncio.sleep(random.uniform(*config.NOTE_READ_DELAY))
+
+                    # 提取数据
+                    note_data = await self.extract_note_detail(page, thread_id)
+
+                    if note_data:
+                        collected_notes.append(note_data)
+                        await self.emit_log("SUCCESS", f"[{thread_id}] 成功采集: {note_data['title'][:30]}", thread_id=thread_id)
+
+                    # 返回列表页
+                    await page.go_back(url, wait_until="networkidle")
+
+                    # 间隔
+                    await asyncio.sleep(random.uniform(*config.NOTE_INTERVAL_DELAY))
+
+                except RiskControlException:
+                    await self.emit_log("ERROR", f"[{thread_id}] 采集笔记触发风控", thread_id=thread_id)
+                    raise
+                except Exception as e:
+                    await self.emit_log("WARNING", f"[{thread_id}] 采集笔记失败: {str(e)}", thread_id=thread_id)
+                    continue
+
+            await page.close()
+
+            # 保存数据
+            if collected_notes:
+                added_count = await self.save_user_data(user_id, collected_notes)
+                await self.emit_log("SUCCESS", f"[{thread_id}] 用户 {user_id} 采集完成，新增 {added_count} 条笔记", thread_id=thread_id)
+                await self.emit_user_status(user_id, "completed", len(collected_notes), thread_id=thread_id)
+            else:
+                await self.emit_log("INFO", f"[{thread_id}] 用户 {user_id} 无新笔记", thread_id=thread_id)
+                await self.emit_user_status(user_id, "completed", 0, thread_id=thread_id)
+
+            await self.emit_thread_status(thread_id, "idle", None)
+            return len(collected_notes)
+
+        except RiskControlException:
+            await self.emit_user_status(user_id, "failed", thread_id=thread_id)
+            await self.emit_thread_status(thread_id, "error", None)
+            raise
+        except Exception as e:
+            await self.emit_log("ERROR", f"[{thread_id}] 处理用户 {user_id} 失败: {str(e)}", thread_id=thread_id)
+            await self.emit_user_status(user_id, "failed", thread_id=thread_id)
+            await self.emit_thread_status(thread_id, "error", None)
+            return 0
+
+    async def run_single_thread(self, user_ids: List[str]):
+        """单线程模式"""
+        await self.emit_log("INFO", "启动单线程采集模式")
+
+        # 创建 Context 并登录
+        context = await self.browser.new_context(
+            viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
+            user_agent=config.USER_AGENT
+        )
+
+        # 检查是否有保存的登录状态
+        if os.path.exists(config.AUTH_STATE_FILE):
+            await self.emit_log("INFO", "加载已保存的登录状态")
+            await context.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
+        else:
+            # 需要登录
+            login_success = await self.handle_login(context)
+            if not login_success:
+                await context.close()
+                return
+
+        self.is_running = True
+
+        for i, user_id in enumerate(user_ids):
+            if self._stop_requested:
+                await self.emit_log("WARNING", "收到停止请求")
+                break
+
+            await self.emit_log("INFO", f"处理用户 {i+1}/{len(user_ids)}: {user_id}")
+
+            try:
+                await self.process_user(context, user_id, thread_id="Main")
+
+                # 用户切换延迟
+                if i < len(user_ids) - 1:
+                    delay = random.uniform(*config.USER_SWITCH_DELAY)
+                    await self.emit_log("INFO", f"等待 {delay:.1f} 秒后处理下一个用户...")
+                    await asyncio.sleep(delay)
+
+            except RiskControlException as e:
+                await self.emit_log("ERROR", f"触发风控: {e.reason}，终止采集")
+                break
+            except Exception as e:
+                await self.emit_log("ERROR", f"处理用户 {user_id} 失败: {str(e)}")
+                continue
+
+        await context.close()
+        self.is_running = False
+        await self.emit_log("SUCCESS", "单线程采集完成")
+
+    async def worker(self, context: BrowserContext, task_queue: asyncio.Queue, thread_id: str):
+        """
+        Worker 处理任务（多线程模式）
+        """
+        # 线程启动时错开
+        await asyncio.sleep(int(thread_id.split('-')[1]) * config.THREAD_OFFSET)
+
+        await self.emit_thread_status(thread_id, "idle", None)
+
+        while True:
+            if self._stop_requested:
+                break
+
+            try:
+                # 获取任务
+                user_id = await asyncio.wait_for(task_queue.get(), timeout=1.0)
+                await self.emit_thread_status(thread_id, "busy", user_id)
+
+                try:
+                    # 线程间最小间隔
+                    await asyncio.sleep(random.uniform(
+                        config.THREAD_MIN_DELAY,
+                        config.THREAD_MIN_DELAY + 3
+                    ))
+
+                    await self.process_user(context, user_id, thread_id)
+
+                except RiskControlException as e:
+                    await self.emit_log("ERROR", f"[{thread_id}] 触发风控: {e.reason}，停止该线程", thread_id=thread_id)
+                    await self.emit_thread_status(thread_id, "error", None)
+                    break
+                except Exception as e:
+                    await self.emit_log("ERROR", f"[{thread_id}] 处理失败: {str(e)}", thread_id=thread_id)
+                finally:
+                    task_queue.task_done()
+                    await self.emit_thread_status(thread_id, "idle", None)
+
+            except asyncio.TimeoutError:
+                continue
+            except Exception as e:
+                await self.emit_log("ERROR", f"[{thread_id}] Worker 错误: {str(e)}", thread_id=thread_id)
+                break
+
+    async def run_multi_thread(self, user_ids: List[str], max_threads: int = 3):
+        """多线程模式"""
+        await self.emit_log("INFO", f"启动多线程采集模式，线程数: {max_threads}")
+
+        # 创建多个 Context
+        contexts = []
+        for i in range(max_threads):
+            ctx = await self.browser.new_context(
+                viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
+                user_agent=config.USER_AGENT
+            )
+
+            # 第一个 Context 登录，其他复制 Cookies
+            if i == 0:
+                if os.path.exists(config.AUTH_STATE_FILE):
+                    await self.emit_log("INFO", "加载已保存的登录状态")
+                    await ctx.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
+                else:
+                    login_success = await self.handle_login(ctx)
+                    if not login_success:
+                        await ctx.close()
+                        return
+            else:
+                await self.copy_cookies(ctx, contexts[0])
+                await self.emit_log("INFO", f"Thread-{i+1} 已复制登录状态")
+
+            contexts.append(ctx)
+
+        # 创建任务队列
+        task_queue = asyncio.Queue()
+        for user_id in user_ids:
+            await task_queue.put(user_id)
+
+        self.is_running = True
+
+        # 创建 workers
+        workers = [
+            asyncio.create_task(
+                self.worker(contexts[i], task_queue, f"Thread-{i+1}")
+            )
+            for i in range(max_threads)
+        ]
+
+        # 等待完成
+        await task_queue.join()
+
+        # 取消 workers
+        for worker in workers:
+            worker.cancel()
+
+        # 关闭 contexts
+        for ctx in contexts:
+            await ctx.close()
+
+        self.is_running = False
+        await self.emit_log("SUCCESS", "多线程采集完成")
+
+    def stop(self):
+        """请求停止采集"""
+        self._stop_requested = True
+        self.is_running = False
+
+    async def run(self, user_ids: List[str], mode: str = "single", max_threads: int = 3):
+        """
+        运行采集
+        :param user_ids: 用户 ID 列表
+        :param mode: single / multi
+        :param max_threads: 多线程模式下的线程数
+        """
+        self._stop_requested = False
+
+        await self.emit_log("INFO", f"开始采集，共 {len(user_ids)} 个用户")
+
+        await self.init_browser()
+
+        try:
+            if mode == "single":
+                await self.run_single_thread(user_ids)
+            else:
+                await self.run_multi_thread(user_ids, max_threads)
+        finally:
+            await self.close_browser()
+
+        await self.emit_log("SUCCESS", "所有采集任务完成")

app/web.py

@@ -0,0 +1,516 @@
+"""
+Web Backend - FastAPI 应用
+提供 API 和 SSE 事件推送
+"""
+import asyncio
+import json
+import os
+import subprocess
+from datetime import datetime
+from typing import Dict, List, Optional
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from fastapi.requests import Request
+from pydantic import BaseModel
+from typing import List as ListType
+
+from app import config
+from app.log_manager import get_log_manager
+from app.scraper import XiaoHongShuScraper
+
+
+# ========== Pydantic 模型 ==========
+class StartRequest(BaseModel):
+    mode: str = "single"  # single / multi
+    max_threads: int = 3
+
+
+class StatusResponse(BaseModel):
+    is_running: bool
+    mode: str
+    max_threads: int
+    total_users: int
+    completed_users: int
+    failed_users: int
+    active_threads: int
+    users_status: Dict[str, Dict]
+
+
+# ========== 全局状态 ==========
+class AppState:
+    def __init__(self):
+        self.is_running = False
+        self.mode = "single"
+        self.max_threads = 1
+        self.total_users = 0
+        self.completed_users = 0
+        self.failed_users = 0
+        self.active_threads = 0
+        self.users_status: Dict[str, Dict] = {}
+        self.thread_status: Dict[str, Dict] = {}
+        self.event_queue: asyncio.Queue = None
+        self.scraper: Optional[XiaoHongShuScraper] = None
+        self.scraper_task: Optional[asyncio.Task] = None
+
+
+app_state = AppState()
+log_manager = get_log_manager()
+
+# ========== FastAPI 应用 ==========
+app = FastAPI(title="小红书数据采集器", version="1.0.0")
+
+# 挂载静态文件和模板
+app.mount("/static", StaticFiles(directory="static"), name="static")
+templates = Jinja2Templates(directory="templates")
+
+
+# ========== 路由 ==========
+
+@app.get("/", response_class=HTMLResponse)
+async def index(request: Request):
+    """首页"""
+    return templates.TemplateResponse("index.html", {"request": request})
+
+
+@app.get("/api/status", response_model=StatusResponse)
+async def get_status():
+    """获取当前状态"""
+    return StatusResponse(
+        is_running=app_state.is_running,
+        mode=app_state.mode,
+        max_threads=app_state.max_threads,
+        total_users=app_state.total_users,
+        completed_users=app_state.completed_users,
+        failed_users=app_state.failed_users,
+        active_threads=app_state.active_threads,
+        users_status=app_state.users_status
+    )
+
+
+@app.post("/api/start")
+async def start_scraping(request: StartRequest):
+    """启动采集"""
+    if app_state.is_running:
+        raise HTTPException(status_code=400, detail="采集任务正在运行中")
+
+    # 读取用户 ID 列表
+    users_file = os.path.join(config.DATA_DIR, "users.txt")
+    if not os.path.exists(users_file):
+        raise HTTPException(status_code=400, detail="用户文件不存在，请先创建 data/users.txt")
+
+    with open(users_file, 'r', encoding='utf-8') as f:
+        user_ids = [line.strip() for line in f if line.strip()]
+
+    if not user_ids:
+        raise HTTPException(status_code=400, detail="用户 ID 列表为空")
+
+    # 验证模式
+    if request.mode not in ["single", "multi"]:
+        raise HTTPException(status_code=400, detail="无效的模式，必须是 single 或 multi")
+
+    # 验证线程数
+    if request.mode == "multi":
+        if request.max_threads < 1 or request.max_threads > config.MAX_CONCURRENT_THREADS:
+            raise HTTPException(status_code=400, detail=f"线程数必须在 1-{config.MAX_CONCURRENT_THREADS} 之间")
+
+    # 重置状态
+    app_state.is_running = True
+    app_state.mode = request.mode
+    app_state.max_threads = request.max_threads if request.mode == "multi" else 1
+    app_state.total_users = len(user_ids)
+    app_state.completed_users = 0
+    app_state.failed_users = 0
+    app_state.active_threads = 0
+    app_state.users_status = {}
+    app_state.thread_status = {}
+
+    # 初始化用户状态
+    for user_id in user_ids:
+        app_state.users_status[user_id] = {
+            "status": "pending",
+            "note_count": 0,
+            "thread_id": None
+        }
+
+    # 创建事件队列
+    app_state.event_queue = asyncio.Queue()
+
+    # 创建采集器
+    app_state.scraper = XiaoHongShuScraper(app_state.event_queue, log_manager)
+
+    # 启动采集任务
+    async def run_scraper():
+        try:
+            await app_state.scraper.run(
+                user_ids=user_ids,
+                mode=request.mode,
+                max_threads=request.max_threads
+            )
+        finally:
+            app_state.is_running = False
+            await app_state.event_queue.put({
+                "type": "finished"
+            })
+
+    app_state.scraper_task = asyncio.create_task(run_scraper())
+
+    return {
+        "success": True,
+        "message": f"采集任务已启动，模式: {request.mode}, 用户数: {len(user_ids)}",
+        "total_users": len(user_ids),
+        "mode": request.mode,
+        "max_threads": app_state.max_threads
+    }
+
+
+@app.post("/api/stop")
+async def stop_scraping():
+    """停止采集"""
+    if not app_state.is_running:
+        raise HTTPException(status_code=400, detail="没有正在运行的采集任务")
+
+    if app_state.scraper:
+        app_state.scraper.stop()
+
+    # 取消任务
+    if app_state.scraper_task:
+        app_state.scraper_task.cancel()
+        try:
+            await app_state.scraper_task
+        except asyncio.CancelledError:
+            pass
+
+    app_state.is_running = False
+
+    return {
+        "success": True,
+        "message": "采集任务已停止"
+    }
+
+
+@app.post("/api/logs/clear")
+async def clear_logs():
+    """清空日志"""
+    log_manager.clear()
+    return {
+        "success": True,
+        "message": "日志已清空"
+    }
+
+
+@app.get("/api/logs")
+async def get_logs(level: Optional[str] = None, limit: Optional[int] = None):
+    """获取日志"""
+    logs = log_manager.get_logs(level=level, limit=limit)
+    return {
+        "logs": logs,
+        "stats": log_manager.get_stats()
+    }
+
+
+@app.get("/api/logs/search")
+async def search_logs(keyword: str, limit: Optional[int] = None):
+    """搜索日志"""
+    logs = log_manager.search_logs(keyword, limit)
+    return {
+        "logs": logs,
+        "count": len(logs)
+    }
+
+
+@app.get("/api/logs/download")
+async def download_logs():
+    """下载日志文件"""
+    text = log_manager.export_to_text()
+
+    return StreamingResponse(
+        iter([text]),
+        media_type="text/plain",
+        headers={
+            "Content-Disposition": f"attachment; filename=xiaohongshu_logs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
+        }
+    )
+
+
+@app.get("/api/users")
+async def get_users():
+    """获取用户列表"""
+    users_file = os.path.join(config.DATA_DIR, "users.txt")
+
+    if not os.path.exists(users_file):
+        return {"users": []}
+
+    with open(users_file, 'r', encoding='utf-8') as f:
+        users = [{"id": line.strip()} for line in f if line.strip()]
+
+    # 添加采集状态
+    for user in users:
+        user_id = user["id"]
+        if user_id in app_state.users_status:
+            user.update(app_state.users_status[user_id])
+        else:
+            user["status"] = "pending"
+            user["note_count"] = 0
+
+    return {"users": users}
+
+
+@app.get("/api/data/{user_id}")
+async def get_user_data(user_id: str):
+    """获取用户采集数据"""
+    file_path = os.path.join(config.DATA_DIR, f"{user_id}.json")
+
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="用户数据不存在")
+
+    with open(file_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    return data
+
+
+@app.get("/api/events")
+async def event_stream():
+    """SSE 事件流"""
+    async def event_generator():
+        while True:
+            try:
+                # 等待事件，超时 1 秒发送心跳
+                event = await asyncio.wait_for(
+                    app_state.event_queue.get(),
+                    timeout=1.0
+                )
+
+                # 更新状态
+                if event["type"] == "log":
+                    # 日志不需要更新状态
+                    pass
+                elif event["type"] == "progress":
+                    app_state.completed_users = event.get("completed", app_state.completed_users)
+                    app_state.failed_users = event.get("failed", app_state.failed_users)
+                    app_state.active_threads = event.get("active_threads", app_state.active_threads)
+                elif event["type"] == "user_status":
+                    if event["user_id"] in app_state.users_status:
+                        app_state.users_status[event["user_id"]]["status"] = event["status"]
+                        app_state.users_status[event["user_id"]]["note_count"] = event.get("note_count", 0)
+                        app_state.users_status[event["user_id"]]["thread_id"] = event.get("thread_id")
+                elif event["type"] == "thread_status":
+                    app_state.thread_status[event["thread_id"]] = {
+                        "status": event["status"],
+                        "current_user": event.get("current_user")
+                    }
+                    # 更新活跃线程数
+                    app_state.active_threads = sum(
+                        1 for s in app_state.thread_status.values() if s.get("status") == "busy"
+                    )
+
+                # 发送事件
+                yield f"data: {json.dumps(event)}\n\n"
+
+            except asyncio.TimeoutError:
+                # 发送心跳
+                yield ": heartbeat\n\n"
+
+            except Exception as e:
+                # 错误事件
+                yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Access-Control-Allow-Origin": "*"
+        }
+    )
+
+
+# ========== Git 相关 API ==========
+
+@app.get("/api/git/status")
+async def git_status():
+    """获取 Git 状态"""
+    if not os.path.exists("/app/.git"):
+        return {
+            "enabled": False,
+            "message": "不是 Git 仓库"
+        }
+
+    try:
+        # 获取当前分支
+        result = subprocess.run(
+            ["git", "branch", "--show-current"],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+        current_branch = result.stdout.strip()
+
+        # 获取最新提交
+        result = subprocess.run(
+            ["git", "log", "-1", "--format=%h %s %cr"],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+        latest_commit = result.stdout.strip()
+
+        # 检查是否有未提交的更改
+        result = subprocess.run(
+            ["git", "status", "--porcelain"],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+        has_changes = len(result.stdout.strip()) > 0
+
+        return {
+            "enabled": True,
+            "branch": current_branch,
+            "latest_commit": latest_commit,
+            "has_changes": has_changes,
+            "auto_pull_enabled": config.ENABLE_GIT_PULL,
+            "auto_pull_interval": config.AUTO_PULL_INTERVAL
+        }
+
+    except Exception as e:
+        return {
+            "enabled": False,
+            "error": str(e)
+        }
+
+
+@app.post("/api/git/pull")
+async def git_pull():
+    """手动拉取最新代码"""
+    if not os.path.exists("/app/.git"):
+        raise HTTPException(status_code=400, detail="不是 Git 仓库")
+
+    try:
+        # 获取当前分支
+        result = subprocess.run(
+            ["git", "branch", "--show-current"],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+        current_branch = result.stdout.strip()
+
+        # 拉取最新代码
+        result = subprocess.run(
+            ["git", "fetch", "origin", current_branch],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+
+        if result.returncode != 0:
+            raise HTTPException(status_code=500, detail=f"拉取失败: {result.stderr}")
+
+        # 检查是否有更新
+        result = subprocess.run(
+            ["git", "rev-list", "HEAD..origin/" + current_branch, "--count"],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+        commits_behind = int(result.stdout.strip())
+
+        if commits_behind == 0:
+            return {
+                "success": True,
+                "message": "已是最新代码",
+                "needs_restart": False
+            }
+
+        # 重置到最新代码
+        result = subprocess.run(
+            ["git", "reset", "--hard", "origin/" + current_branch],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+
+        if result.returncode != 0:
+            raise HTTPException(status_code=500, detail=f"重置失败: {result.stderr}")
+
+        # 检查 Python 文件是否有变化
+        result = subprocess.run(
+            ["git", "diff", "--name-only", "HEAD@{1}", "HEAD"],
+            capture_output=True,
+            text=True,
+            cwd="/app"
+        )
+        changed_files = result.stdout.strip().split('\n')
+        python_changed = any(f.endswith('.py') for f in changed_files if f)
+
+        return {
+            "success": True,
+            "message": f"已拉取 {commits_behind} 个新提交",
+            "changed_files": changed_files,
+            "needs_restart": python_changed,
+            "restart_hint": "Python 代码有变化，建议重启容器以应用更新"
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"拉取失败: {str(e)}")
+
+
+@app.post("/api/git/restart")
+async def restart_container():
+    """
+    请求重启容器
+    注意：这需要在容器外配置重启策略
+    """
+    return {
+        "success": True,
+        "message": "重启请求已接收",
+        "methods": [
+            "docker restart xiaohongshu-scraper",
+            "docker-compose restart"
+        ]
+    }
+
+
+# ========== 启动事件 ==========
+@app.on_event("startup")
+async def startup_event():
+    """启动时初始化"""
+    # 确保数据目录存在
+    os.makedirs(config.DATA_DIR, exist_ok=True)
+    os.makedirs(config.SCREENSHOT_DIR, exist_ok=True)
+
+    # 初始化事件队列
+    app_state.event_queue = asyncio.Queue()
+
+    print("=" * 60)
+    print("小红书数据采集器 Web 服务启动")
+    print(f"访问地址: http://{config.WEB_HOST}:{config.WEB_PORT}")
+    print("=" * 60)
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    """关闭时清理"""
+    if app_state.scraper:
+        app_state.scraper.stop()
+
+    if app_state.scraper_task:
+        app_state.scraper_task.cancel()
+
+    print("Web 服务已关闭")
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "app.web:app",
+        host=config.WEB_HOST,
+        port=config.WEB_PORT,
+        reload=True
+    )