ts

2026-01-12 18:58:23 +08:00
parent fb98a5d221
commit f2dcc4c0cf
13 changed files with 1183 additions and 1194 deletions
@@ -0,0 +1,171 @@
+
+"""auth_manager.py
+
+登录态管理（单账号）
+- 使用 Playwright storage_state 保存到 data/auth_state.json
+- 额外保存元数据到 data/auth_meta.json（登录时间/最后验证时间等）
+- 提供 verify_login / get_login_status 等方法
+
+设计目标：
+1) 不依赖“文件存在”判断登录，而是用 Playwright 打开页面做轻量验证
+2) 单账号长期复用：只要不掉线，就一直用
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Optional, Tuple
+
+from playwright.async_api import Browser, BrowserContext, TimeoutError as PlaywrightTimeoutError
+
+from app import config
+
+
+AUTH_META_FILE = os.path.join(config.DATA_DIR, "auth_meta.json")
+
+
+@dataclass
+class LoginStatus:
+    logged_in: bool
+    login_time: Optional[str] = None
+    last_verified_time: Optional[str] = None
+    message: Optional[str] = None
+
+
+def _utc_iso() -> str:
+    # 使用本地时间 ISO 字符串，方便 UI 展示
+    return datetime.now().isoformat(timespec="seconds")
+
+
+def load_auth_meta() -> dict:
+    if not os.path.exists(AUTH_META_FILE):
+        return {}
+    try:
+        with open(AUTH_META_FILE, "r", encoding="utf-8") as f:
+            return json.load(f) or {}
+    except Exception:
+        return {}
+
+
+def save_auth_meta(meta: dict) -> None:
+    tmp = AUTH_META_FILE + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(meta, f, ensure_ascii=False, indent=2)
+    os.replace(tmp, AUTH_META_FILE)
+
+
+async def _new_context(browser: Browser) -> BrowserContext:
+    """基于已保存的 storage_state 创建 context。"""
+    if os.path.exists(config.AUTH_STATE_FILE):
+        return await browser.new_context(storage_state=config.AUTH_STATE_FILE)
+    return await browser.new_context()
+
+
+async def open_login_page(browser: Browser) -> None:
+    """打开登录页面（通常需要用户在 VNC/有界面环境里扫码）。"""
+    context = await _new_context(browser)
+    page = await context.new_page()
+    await page.goto(config.LOGIN_URL, wait_until="domcontentloaded")
+    # 不关闭页面：让用户扫码
+    # 注意：context/page 生命周期由浏览器进程维护；任务结束时统一关闭 browser 即可
+
+
+async def verify_login(browser: Browser, timeout_sec: int = 20) -> Tuple[bool, str]:
+    """验证当前 storage_state 是否仍然有效。
+
+    策略（尽量稳，避免依赖某个易变 selector）：
+    1) 打开首页；
+    2) 若跳转到 /login 或出现明显登录按钮/文案，则判定未登录；
+    3) 若未跳登录页，并且存在一定数量 cookie，则判定登录有效（启发式）。
+    """
+    if not os.path.exists(config.AUTH_STATE_FILE):
+        return False, "未找到登录态文件，请先登录"
+
+    context = await _new_context(browser)
+    page = await context.new_page()
+
+    try:
+        await page.goto(config.XIAOHONGSHU_BASE_URL + "/", wait_until="domcontentloaded", timeout=timeout_sec * 1000)
+        await page.wait_for_timeout(800)
+
+        url = page.url or ""
+        if "/login" in url:
+            return False, "已跳转到登录页（登录态可能已失效）"
+
+        # 启发式：页面上是否出现明显“登录”入口
+        login_text_hits = await page.locator("text=登录").count()
+        if login_text_hits > 0:
+            # 有些已登录页面也会出现“登录”字样（比如弹窗），再做一次 cookie 辅助判断
+            cookies = await context.cookies()
+            if len(cookies) < 3:
+                return False, "检测到登录入口且 cookie 数量过少，疑似未登录"
+
+        cookies = await context.cookies()
+        if len(cookies) < 3:
+            return False, "cookie 数量过少，疑似未登录"
+
+        # 通过：写入 meta
+        meta = load_auth_meta()
+        if not meta.get("login_time"):
+            meta["login_time"] = _utc_iso()
+        meta["last_verified_time"] = _utc_iso()
+        save_auth_meta(meta)
+
+        # 刷新保存一次 state（让 cookie 续期信息尽可能落盘）
+        try:
+            await context.storage_state(path=config.AUTH_STATE_FILE)
+        except Exception:
+            # 不要因为保存失败影响验证结果
+            pass
+
+        return True, "ok"
+    except PlaywrightTimeoutError:
+        return False, "验证登录超时（网络或风控），请稍后重试"
+    except Exception as e:
+        return False, f"验证登录异常: {e}"
+    finally:
+        try:
+            await page.close()
+        except Exception:
+            pass
+        try:
+            await context.close()
+        except Exception:
+            pass
+
+
+async def get_login_status(browser: Optional[Browser]) -> LoginStatus:
+    """获取登录状态（如果 browser 可用则做轻量验证，否则仅返回本地 meta 状态）。"""
+    meta = load_auth_meta()
+    if not os.path.exists(config.AUTH_STATE_FILE):
+        return LoginStatus(logged_in=False, message="未登录")
+
+    if browser is None:
+        return LoginStatus(
+            logged_in=True if meta.get("login_time") else True,
+            login_time=meta.get("login_time"),
+            last_verified_time=meta.get("last_verified_time"),
+            message="仅本地状态（未进行在线验证）"
+        )
+
+    ok, msg = await verify_login(browser)
+    meta = load_auth_meta()
+    return LoginStatus(
+        logged_in=ok,
+        login_time=meta.get("login_time"),
+        last_verified_time=meta.get("last_verified_time"),
+        message=None if ok else msg
+    )
+
+
+def clear_login_state() -> None:
+    """清理登录态。"""
+    for p in [config.AUTH_STATE_FILE, AUTH_META_FILE]:
+        try:
+            if os.path.exists(p):
+                os.remove(p)
+        except Exception:
+            pass
@@ -1,153 +1,77 @@
+
+"""log_manager.py
+
+简单的线程安全日志管理器：
+- 追加日志（level/message/thread_id）
+- 持久化到 data/logs.json
+- 提供读取最近 N 条、清空
+
+说明：采集器是异步运行，但日志写入使用 threading.Lock 足够。
 """
-Log Manager - 持久化日志管理器
-支持日志加载、保存、追加、清空
-"""
+
+from __future__ import annotations
+
 import json
 import os
 import threading
 from datetime import datetime
 from typing import List, Dict, Optional
+
 from app import config


 class LogManager:
-    """日志管理器 - 线程安全"""
-
-    def __init__(self, log_file: str = None, max_logs: int = None):
-        self.log_file = log_file or config.LOG_FILE
-        self.max_logs = max_logs or config.MAX_LOGS
+    def __init__(self, log_file: Optional[str] = None, max_logs: int = 2000):
+        self.log_file = log_file or os.path.join(config.DATA_DIR, "logs.json")
+        self.max_logs = max_logs
        self._lock = threading.Lock()
        self.logs: List[Dict] = []
-        self._load_logs()
+        self._load()

-    def _load_logs(self):
-        """启动时加载历史日志"""
-        with self._lock:
-            if os.path.exists(self.log_file):
-                try:
-                    with open(self.log_file, 'r', encoding='utf-8') as f:
-                        data = json.load(f)
-                        self.logs = data.get('logs', [])
-                except (json.JSONDecodeError, IOError) as e:
-                    print(f"[LogManager] 加载日志文件失败: {e}")
-                    self.logs = []
-            else:
-                self.logs = []
+    def _load(self) -> None:
+        if not os.path.exists(self.log_file):
+            self.logs = []
+            return
+        try:
+            with open(self.log_file, "r", encoding="utf-8") as f:
+                data = json.load(f) or {}
+                self.logs = data.get("logs", []) or []
+        except Exception:
+            self.logs = []
+
+    def _save_locked(self) -> None:
+        os.makedirs(os.path.dirname(self.log_file), exist_ok=True)
+        tmp = self.log_file + ".tmp"
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(
+                {"logs": self.logs, "last_updated": datetime.now().isoformat(timespec="seconds")},
+                f,
+                ensure_ascii=False,
+                indent=2,
+            )
+        os.replace(tmp, self.log_file)

    def append(self, level: str, message: str, thread_id: Optional[str] = None) -> Dict:
-        """
-        追加新日志
-        返回日志条目
-        """
-        with self._lock:
-            log_entry = {
-                "timestamp": datetime.now().isoformat(),
-                "level": level,
-                "message": message,
-                "thread_id": thread_id
-            }
-            self.logs.append(log_entry)
+        entry = {
+            "time": datetime.now().isoformat(timespec="seconds"),
+            "level": level.upper(),
+            "message": message,
+        }
+        if thread_id:
+            entry["thread_id"] = thread_id

-            # 限制日志数量（保留最新的 max_logs 条）
+        with self._lock:
+            self.logs.append(entry)
            if len(self.logs) > self.max_logs:
-                self.logs = self.logs[-self.max_logs:]
+                self.logs = self.logs[-self.max_logs :]
+            self._save_locked()
+        return entry

-            # 异步保存到文件
-            self._save()
-
-            return log_entry
-
-    def _save(self):
-        """保存到文件（需在锁内调用）"""
-        try:
-            with open(self.log_file, 'w', encoding='utf-8') as f:
-                json.dump({
-                    "logs": self.logs,
-                    "last_updated": datetime.now().isoformat()
-                }, f, ensure_ascii=False, indent=2)
-        except IOError as e:
-            print(f"[LogManager] 保存日志文件失败: {e}")
-
-    def get_logs(self, level: Optional[str] = None, limit: Optional[int] = None) -> List[Dict]:
-        """
-        获取日志
-        :param level: 过滤级别 (None/INFO/SUCCESS/WARNING/ERROR/DEBUG)
-        :param limit: 限制返回数量
-        """
-        with self._lock:
-            if level:
-                filtered = [log for log in self.logs if log['level'] == level]
-            else:
-                filtered = self.logs
-
-            if limit:
-                return filtered[-limit:]
-            return filtered
-
-    def search_logs(self, keyword: str, limit: Optional[int] = None) -> List[Dict]:
-        """
-        搜索日志
-        :param keyword: 搜索关键词
-        :param limit: 限制返回数量
-        """
-        with self._lock:
-            filtered = [
-                log for log in self.logs
-                if keyword.lower() in log['message'].lower()
-            ]
-
-            if limit:
-                return filtered[-limit:]
-            return filtered
-
-    def clear(self):
-        """清空日志"""
+    def clear(self) -> None:
        with self._lock:
            self.logs = []
-            self._save()
+            self._save_locked()

-    def get_stats(self) -> Dict:
-        """获取日志统计信息"""
+    def get_latest(self, limit: int = 200) -> List[Dict]:
        with self._lock:
-            stats = {
-                "total": len(self.logs),
-                "by_level": {}
-            }
-
-            for log in self.logs:
-                level = log['level']
-                stats["by_level"][level] = stats["by_level"].get(level, 0) + 1
-
-            return stats
-
-    def export_to_text(self) -> str:
-        """导出为文本格式"""
-        with self._lock:
-            lines = []
-            lines.append("=" * 80)
-            lines.append("小红书采集器日志")
-            lines.append(f"导出时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-            lines.append(f"总日志数: {len(self.logs)}")
-            lines.append("=" * 80)
-            lines.append("")
-
-            for log in self.logs:
-                timestamp = log['timestamp'][:19]  # 去掉微秒
-                level = log['level'].ljust(8)
-                thread = f"[{log['thread_id']}]" if log.get('thread_id') else ""
-                message = log['message']
-                lines.append(f"{timestamp} | {level} | {thread} {message}")
-
-            return "\n".join(lines)
-
-
-# 全局日志管理器实例
-_log_manager_instance = None
-
-
-def get_log_manager() -> LogManager:
-    """获取全局日志管理器实例"""
-    global _log_manager_instance
-    if _log_manager_instance is None:
-        _log_manager_instance = LogManager()
-    return _log_manager_instance
+            return list(self.logs[-max(1, int(limit)) :])
@@ -467,16 +467,20 @@ class XiaoHongShuScraper:
        await self.emit_log("INFO", "启动单线程采集模式")

        # 创建 Context 并登录
-        context = await self.browser.new_context(
-            viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
-            user_agent=config.USER_AGENT
-        )
-
-        # 检查是否有保存的登录状态
+        
+        # 创建 Context（优先加载已保存的 storage_state）
        if os.path.exists(config.AUTH_STATE_FILE):
-            await self.emit_log("INFO", "加载已保存的登录状态")
-            await context.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
+            await self.emit_log("INFO", "加载已保存的登录状态 (storage_state)")
+            context = await self.browser.new_context(
+                storage_state=config.AUTH_STATE_FILE,
+                viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
+                user_agent=config.USER_AGENT
+            )
        else:
+            context = await self.browser.new_context(
+                viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
+                user_agent=config.USER_AGENT
+            )
            # 需要登录
            login_success = await self.handle_login(context)
            if not login_success:
@@ -559,29 +563,31 @@ class XiaoHongShuScraper:
        """多线程模式"""
        await self.emit_log("INFO", f"启动多线程采集模式，线程数: {max_threads}")

-        # 创建多个 Context
+
+        # 创建多个 Context（同一账号并发会增加风控风险，请合理控制线程数）
        contexts = []
-        for i in range(max_threads):
-            ctx = await self.browser.new_context(
+
+        # 确保已有登录态；若没有，先用第一个 context 登录并保存
+        if not os.path.exists(config.AUTH_STATE_FILE):
+            first = await self.browser.new_context(
                viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
                user_agent=config.USER_AGENT
            )
+            login_success = await self.handle_login(first)
+            if not login_success:
+                await first.close()
+                return
+            await first.close()

-            # 第一个 Context 登录，其他复制 Cookies
-            if i == 0:
-                if os.path.exists(config.AUTH_STATE_FILE):
-                    await self.emit_log("INFO", "加载已保存的登录状态")
-                    await ctx.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
-                else:
-                    login_success = await self.handle_login(ctx)
-                    if not login_success:
-                        await ctx.close()
-                        return
-            else:
-                await self.copy_cookies(ctx, contexts[0])
-                await self.emit_log("INFO", f"Thread-{i+1} 已复制登录状态")
-
+        # 基于 storage_state 为每个线程创建独立 context
+        for i in range(max_threads):
+            ctx = await self.browser.new_context(
+                storage_state=config.AUTH_STATE_FILE,
+                viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
+                user_agent=config.USER_AGENT
+            )
            contexts.append(ctx)
+            await self.emit_log("INFO", f"Thread-{i+1} context ready")

        # 创建任务队列
        task_queue = asyncio.Queue()
@@ -0,0 +1,153 @@
+
+"""user_resolver.py
+
+将“数字小红书号（例如 11432840954）”解析成 profile_id（/user/profile/{profile_id} 里的那串）。
+- 使用 Playwright 打开搜索结果页（更接近真实浏览器，风险更低）
+- 解析 HTML：根据 '小红书号：<keyword>' 定位，再提取附近的 profile href
+- 强制缓存：data/profile_map.json，避免重复搜索造成风控压力
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+import re
+from html import unescape
+from typing import Dict, Optional, Tuple
+
+from playwright.async_api import Browser
+
+from app import config
+
+PROFILE_MAP_FILE = os.path.join(config.DATA_DIR, "profile_map.json")
+
+
+def _load_map() -> Dict[str, str]:
+    if not os.path.exists(PROFILE_MAP_FILE):
+        return {}
+    try:
+        with open(PROFILE_MAP_FILE, "r", encoding="utf-8") as f:
+            return json.load(f) or {}
+    except Exception:
+        return {}
+
+
+def _save_map(m: Dict[str, str]) -> None:
+    tmp = PROFILE_MAP_FILE + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(m, f, ensure_ascii=False, indent=2)
+    os.replace(tmp, PROFILE_MAP_FILE)
+
+
+def extract_profile_id_from_search_html(html: str, keyword: str) -> Optional[str]:
+    """从 search_result HTML 中解析出对应 keyword 的 profile_id。"""
+    if not html:
+        return None
+
+    html = unescape(html)
+    marker = f"小红书号：{keyword}"
+    idx = html.find(marker)
+    if idx == -1:
+        return None
+
+    start = max(0, idx - 5000)
+    end = min(len(html), idx + 5000)
+    chunk = html[start:end]
+
+    m = re.search(r'href="\/user\/profile\/([0-9a-fA-F]+)', chunk)
+    if not m:
+        return None
+    return m.group(1)
+
+
+def parse_line_to_keyword_or_profile(line: str) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+    """解析一行 txt，返回 (keyword数字, profile_id, label备注) 三者之一。
+
+    允许格式：
+    - 11432840954
+    - 11432840954  张三
+    - https://www.xiaohongshu.com/user/profile/<profile_id> ...
+    - /user/profile/<profile_id> ...
+    - profile_id(十六进制串)
+    - 支持行尾 # 备注
+    """
+    raw = line.strip()
+    if not raw:
+        return None, None, None
+
+    # 整行注释
+    if raw.startswith("#") or raw.startswith("//"):
+        return None, None, None
+
+    # 拆备注：支持 # 以及连续空格
+    label = None
+    if "#" in raw:
+        raw, label = raw.split("#", 1)
+        raw = raw.strip()
+        label = label.strip() or None
+
+    parts = raw.split()
+    core = parts[0].strip() if parts else ""
+    if len(parts) > 1 and not label:
+        label = " ".join(parts[1:]).strip() or None
+
+    # URL/相对路径中直接提取 profile_id
+    m = re.search(r"/user/profile/([0-9a-fA-F]+)", core)
+    if m:
+        return None, m.group(1), label
+
+    # 可能直接就是 profile_id（十六进制串，长度>=8）
+    if re.fullmatch(r"[0-9a-fA-F]{8,}", core):
+        return None, core, label
+
+    # 纯数字：小红书号
+    if re.fullmatch(r"\d{5,20}", core):
+        return core, None, label
+
+    return None, None, label
+
+
+async def resolve_keyword_to_profile_id(browser: Browser, keyword: str, *, slow_mode: bool = True) -> Tuple[Optional[str], str]:
+    """将数字 keyword 解析成 profile_id，并写入缓存。返回 (profile_id, message)。"""
+    m = _load_map()
+    if keyword in m:
+        return m[keyword], "cache"
+
+    # 使用新的 context（不强制登录）
+    context = await browser.new_context()
+    page = await context.new_page()
+
+    try:
+        url = f"{config.XIAOHONGSHU_BASE_URL}/search_result?keyword={keyword}&source=unknown"
+        await page.goto(url, wait_until="domcontentloaded", timeout=20000)
+
+        # 轻微等待/滚动（更像真人浏览）
+        if slow_mode:
+            await page.wait_for_timeout(int(random.uniform(800, 1600)))
+            # 小幅滚动一下
+            try:
+                await page.mouse.wheel(0, random.randint(200, 600))
+            except Exception:
+                pass
+            await page.wait_for_timeout(int(random.uniform(400, 1000)))
+
+        html = await page.content()
+        pid = extract_profile_id_from_search_html(html, keyword)
+        if not pid:
+            return None, "not_found"
+
+        m[keyword] = pid
+        _save_map(m)
+        return pid, "ok"
+    except Exception as e:
+        return None, f"error:{e}"
+    finally:
+        try:
+            await page.close()
+        except Exception:
+            pass
+        try:
+            await context.close()
+        except Exception:
+            pass