From 65dfe594e0b8239b589dbd32977a2a629fc950d4 Mon Sep 17 00:00:00 2001 From: mzl Date: Sat, 10 Jan 2026 21:54:03 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B4=A6=E5=8F=B7=E8=A7=A3=E6=9E=90&=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E7=9B=B8=E5=BA=94=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/server.py | 73 +++++++++++++++++++++++++++++++++++++++++++-- xhs_crawler_core.py | 18 +++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/app/server.py b/app/server.py index f49ce87..be3d5a7 100644 --- a/app/server.py +++ b/app/server.py @@ -51,6 +51,50 @@ def require_token(token: Optional[str]): if ADMIN_TOKEN and token != ADMIN_TOKEN: raise HTTPException(status_code=401, detail="Unauthorized") +# ---- 内存缓存：提升 API 响应速度 ---- +class Cache: + def __init__(self): + self._profiles_cache = {"data": None, "ts": 0} + self._profiles_cache_ttl = 2 # 配置缓存 2 秒 + self._status_cache = {"data": None, "ts": 0} + self._status_cache_ttl = 1 # 状态缓存 1 秒 + + def get_profiles(self) -> Optional[List[Dict[str, Any]]]: + """获取缓存的 profiles，未过期则返回缓存""" + now = time.time() + if self._profiles_cache["data"] and (now - self._profiles_cache["ts"]) < self._profiles_cache_ttl: + return self._profiles_cache["data"] + return None + + def set_profiles(self, data: List[Dict[str, Any]]): + """更新 profiles 缓存""" + self._profiles_cache["data"] = data + self._profiles_cache["ts"] = time.time() + + def invalidate_profiles(self): + """使 profiles 缓存失效""" + self._profiles_cache["data"] = None + self._profiles_cache["ts"] = 0 + + def get_status(self) -> Optional[Dict[str, Any]]: + """获取缓存的状态""" + now = time.time() + if self._status_cache["data"] and (now - self._status_cache["ts"]) < self._status_cache_ttl: + return self._status_cache["data"] + return None + + def set_status(self, data: Dict[str, Any]): + """更新状态缓存""" + self._status_cache["data"] = data + self._status_cache["ts"] = time.time() + + def invalidate_status(self): + """使状态缓存失效""" + self._status_cache["data"] = None + self._status_cache["ts"] = 0 + +cache = Cache() + # ---- logging hub (queue + websocket broadcast) ---- class LogHub: def __init__(self): @@ -337,6 +381,8 @@ class TaskManager: self.stop_event.clear() self.running = True self.progress.update({"total": len(profiles), "done": 0, "current": "", "status": "running", "last_error": ""}) + # 使状态缓存失效 + cache.invalidate_status() def run(): try: @@ -367,6 +413,8 @@ class TaskManager: self._log(f"[任务][致命错误] {e}") finally: self.running = False + # 使状态缓存失效 + cache.invalidate_status() self.thread = threading.Thread(target=run, daemon=True) self.thread.start() @@ -403,6 +451,11 @@ def ping(): @app.get("/api/status") def status(token: Optional[str] = None): require_token(token) + # 尝试从缓存获取 + cached = cache.get_status() + if cached is not None: + return cached + cfg = taskmgr.load_config() storage_path = _resolve_storage_path(cfg) login_ok = False @@ -413,13 +466,16 @@ def status(token: Optional[str] = None): loghub.emit(f"[状态] 检查登录态异常: {e}") login_msg = str(e) - return { + result = { "running": taskmgr.running, "progress": taskmgr.progress, "login_ok": login_ok, "login_msg": login_msg, "storage_state_path": storage_path, } + # 更新缓存 + cache.set_status(result) + return result @app.get("/api/config") def get_config(token: Optional[str] = None): @@ -440,7 +496,14 @@ def set_config(payload: Dict[str, Any], token: Optional[str] = None): def get_profiles(token: Optional[str] = None): require_token(token) require_login_state() - return taskmgr.load_profiles() + # 尝试从缓存获取 + cached = cache.get_profiles() + if cached is not None: + return cached + # 缓存未命中，从文件加载 + data = taskmgr.load_profiles() + cache.set_profiles(data) + return data @app.post("/api/profiles") def set_profiles(payload: List[Dict[str, Any]], token: Optional[str] = None): @@ -448,6 +511,8 @@ def set_profiles(payload: List[Dict[str, Any]], token: Optional[str] = None): require_login_state() with open(PROFILES_PATH, "w", encoding="utf-8") as f: json.dump(payload, f, ensure_ascii=False, indent=2) + # 更新缓存 + cache.set_profiles(payload) loghub.emit(f"[Profiles] 已保存 profiles.json，共 {len(payload)} 条") return {"ok": True} @@ -491,6 +556,8 @@ def resolve_profile(payload: Dict[str, Any], token: Optional[str] = None): merged, added, updated = _merge_profile(existing, profile) with open(PROFILES_PATH, "w", encoding="utf-8") as f: json.dump(merged, f, ensure_ascii=False, indent=2) + # 更新缓存 + cache.set_profiles(merged) if added: loghub.emit(f"[Profiles] 解析新增账号：{profile.get('xhs_no') or profile.get('url')}") elif updated: @@ -662,6 +729,8 @@ def profiles_import_excel(payload: Dict[str, Any], token: Optional[str] = None): add_count += 1 with open(PROFILES_PATH, "w", encoding="utf-8") as f: json.dump(merged, f, ensure_ascii=False, indent=2) + # 更新缓存 + cache.set_profiles(merged) loghub.emit(f"[Profiles] Excel 导入完成：新增 {add_count} 条（总计 {len(merged)}）") return {"ok": True, "added": add_count, "total": len(merged), "profiles": merged} diff --git a/xhs_crawler_core.py b/xhs_crawler_core.py index 23eb01d..8c0bbf6 100644 --- a/xhs_crawler_core.py +++ b/xhs_crawler_core.py @@ -470,6 +470,15 @@ def resolve_profile_by_search( page.goto(search_url, wait_until="domcontentloaded", timeout=60000) page.wait_for_timeout(1200) + # 检查当前 URL，防止被重定向到个人主页 + current_url = page.url + _log(log, f"[账号] 当前页面：{current_url}") + if "/user/profile/" in current_url and current_url != search_url: + # 被重定向到某个用户主页，这可能是搜索机制导致的 + _log(log, f"[账号] 检测到重定向到用户主页，尝试返回搜索页") + page.goto(search_url, wait_until="domcontentloaded", timeout=60000) + page.wait_for_timeout(1500) + # ✅ 不再嵌套调用 check_login_state_with_storage（避免二次启动 playwright） for t in ["扫码登录", "安全验证", "请先登录", "登录后查看"]: try: @@ -490,14 +499,19 @@ def resolve_profile_by_search( try: loc = page.locator(sel) if loc.count() > 0: + _log(log, f"[账号] 找到用户标签：{sel}") loc.first.click(timeout=3000) + page.wait_for_timeout(500) return True except Exception: pass try: + _log(log, f"[账号] 尝试通过角色定位用户标签") page.get_by_role("tab", name="用户").click(timeout=3000) + page.wait_for_timeout(500) return True except Exception: + _log(log, f"[账号] 未找到用户标签，可能已经是用户搜索结果") return False def _wait_user_items() -> bool: @@ -546,6 +560,7 @@ def resolve_profile_by_search( items, use_anchor_only = _find_user_items() n = items.count() + _log(log, f"[账号] 找到 {n} 个用户结果，使用模式：{'锚点链接' if use_anchor_only else '结构化元素'}") if n == 0: context.close(); browser.close() raise RuntimeError("未找到用户结果（可能被拦截/需要更完整登录态）") @@ -555,6 +570,7 @@ def resolve_profile_by_search( for i in range(n): it = items.nth(i) + _log(log, f"[账号] 分析第 {i+1}/{n} 个结果...") if use_anchor_only: href = (it.get_attribute("href") or "").strip() nickname = "" @@ -618,6 +634,8 @@ def resolve_profile_by_search( if _extract_profile_id_from_href(href): score += 5 + _log(log, f"[账号] - 昵称: {nickname[:30] if len(nickname) > 30 else nickname}, 账号: {xhs_no}, 得分: {score}") + if score > best_score: best_score = score url = "https://www.xiaohongshu.com" + href if href.startswith("/") else href