xinyouhe/xhs_crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

账号解析&接口相应性能

Browse Source
This commit is contained in:
mzl
2026-01-10 21:54:03 +08:00
parent e69df2ebb6
commit 65dfe594e0
2 changed files with 89 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/server.py
+71 -2
View File
@@ -51,6 +51,50 @@ def require_token(token: Optional[str]):
if ADMIN_TOKEN and token != ADMIN_TOKEN:
raise HTTPException(status_code=401, detail="Unauthorized")
# ---- 内存缓存：提升 API 响应速度 ----
class Cache:
def __init__(self):
self._profiles_cache = {"data": None, "ts": 0}
self._profiles_cache_ttl = 2 # 配置缓存 2 秒
self._status_cache = {"data": None, "ts": 0}
self._status_cache_ttl = 1 # 状态缓存 1 秒
def get_profiles(self) -> Optional[List[Dict[str, Any]]]:
"""获取缓存的 profiles，未过期则返回缓存"""
now = time.time()
if self._profiles_cache["data"] and (now - self._profiles_cache["ts"]) < self._profiles_cache_ttl:
return self._profiles_cache["data"]
return None
def set_profiles(self, data: List[Dict[str, Any]]):
"""更新 profiles 缓存"""
self._profiles_cache["data"] = data
self._profiles_cache["ts"] = time.time()
def invalidate_profiles(self):
"""使 profiles 缓存失效"""
self._profiles_cache["data"] = None
self._profiles_cache["ts"] = 0
def get_status(self) -> Optional[Dict[str, Any]]:
"""获取缓存的状态"""
now = time.time()
if self._status_cache["data"] and (now - self._status_cache["ts"]) < self._status_cache_ttl:
return self._status_cache["data"]
return None
def set_status(self, data: Dict[str, Any]):
"""更新状态缓存"""
self._status_cache["data"] = data
self._status_cache["ts"] = time.time()
def invalidate_status(self):
"""使状态缓存失效"""
self._status_cache["data"] = None
self._status_cache["ts"] = 0
cache = Cache()
# ---- logging hub (queue + websocket broadcast) ----
class LogHub:
def __init__(self):
@@ -337,6 +381,8 @@ class TaskManager:
self.stop_event.clear()
self.running = True
self.progress.update({"total": len(profiles), "done": 0, "current": "", "status": "running", "last_error": ""})
# 使状态缓存失效
cache.invalidate_status()
def run():
try:
@@ -367,6 +413,8 @@ class TaskManager:
self._log(f"[任务][致命错误] {e}")
finally:
self.running = False
# 使状态缓存失效
cache.invalidate_status()
self.thread = threading.Thread(target=run, daemon=True)
self.thread.start()
@@ -403,6 +451,11 @@ def ping():
@app.get("/api/status")
def status(token: Optional[str] = None):
require_token(token)
# 尝试从缓存获取
cached = cache.get_status()
if cached is not None:
return cached
cfg = taskmgr.load_config()
storage_path = _resolve_storage_path(cfg)
login_ok = False
@@ -413,13 +466,16 @@ def status(token: Optional[str] = None):
loghub.emit(f"[状态] 检查登录态异常: {e}")
login_msg = str(e)
return {
result = {
"running": taskmgr.running,
"progress": taskmgr.progress,
"login_ok": login_ok,
"login_msg": login_msg,
"storage_state_path": storage_path,
}
# 更新缓存
cache.set_status(result)
return result
@app.get("/api/config")
def get_config(token: Optional[str] = None):
@@ -440,7 +496,14 @@ def set_config(payload: Dict[str, Any], token: Optional[str] = None):
def get_profiles(token: Optional[str] = None):
require_token(token)
require_login_state()
return taskmgr.load_profiles()
# 尝试从缓存获取
cached = cache.get_profiles()
if cached is not None:
return cached
# 缓存未命中，从文件加载
data = taskmgr.load_profiles()
cache.set_profiles(data)
return data
@app.post("/api/profiles")
def set_profiles(payload: List[Dict[str, Any]], token: Optional[str] = None):
@@ -448,6 +511,8 @@ def set_profiles(payload: List[Dict[str, Any]], token: Optional[str] = None):
require_login_state()
with open(PROFILES_PATH, "w", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False, indent=2)
# 更新缓存
cache.set_profiles(payload)
loghub.emit(f"[Profiles] 已保存 profiles.json，共 {len(payload)} 条")
return {"ok": True}
@@ -491,6 +556,8 @@ def resolve_profile(payload: Dict[str, Any], token: Optional[str] = None):
merged, added, updated = _merge_profile(existing, profile)
with open(PROFILES_PATH, "w", encoding="utf-8") as f:
json.dump(merged, f, ensure_ascii=False, indent=2)
# 更新缓存
cache.set_profiles(merged)
if added:
loghub.emit(f"[Profiles] 解析新增账号：{profile.get('xhs_no') or profile.get('url')}")
elif updated:
@@ -662,6 +729,8 @@ def profiles_import_excel(payload: Dict[str, Any], token: Optional[str] = None):
add_count += 1
with open(PROFILES_PATH, "w", encoding="utf-8") as f:
json.dump(merged, f, ensure_ascii=False, indent=2)
# 更新缓存
cache.set_profiles(merged)
loghub.emit(f"[Profiles] Excel 导入完成：新增 {add_count} 条（总计 {len(merged)}）")
return {"ok": True, "added": add_count, "total": len(merged), "profiles": merged}
xhs_crawler_core.py
+18
View File
@@ -470,6 +470,15 @@ def resolve_profile_by_search(
page.goto(search_url, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(1200)
# 检查当前 URL，防止被重定向到个人主页
current_url = page.url
_log(log, f"[账号] 当前页面：{current_url}")
if "/user/profile/" in current_url and current_url != search_url:
# 被重定向到某个用户主页，这可能是搜索机制导致的
_log(log, f"[账号] 检测到重定向到用户主页，尝试返回搜索页")
page.goto(search_url, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(1500)
# ✅ 不再嵌套调用 check_login_state_with_storage（避免二次启动 playwright）
for t in ["扫码登录", "安全验证", "请先登录", "登录后查看"]:
try:
@@ -490,14 +499,19 @@ def resolve_profile_by_search(
try:
loc = page.locator(sel)
if loc.count() > 0:
_log(log, f"[账号] 找到用户标签：{sel}")
loc.first.click(timeout=3000)
page.wait_for_timeout(500)
return True
except Exception:
pass
try:
_log(log, f"[账号] 尝试通过角色定位用户标签")
page.get_by_role("tab", name="用户").click(timeout=3000)
page.wait_for_timeout(500)
return True
except Exception:
_log(log, f"[账号] 未找到用户标签，可能已经是用户搜索结果")
return False
def _wait_user_items() -> bool:
@@ -546,6 +560,7 @@ def resolve_profile_by_search(
items, use_anchor_only = _find_user_items()
n = items.count()
_log(log, f"[账号] 找到 {n} 个用户结果，使用模式：{'锚点链接' if use_anchor_only else '结构化元素'}")
if n == 0:
context.close(); browser.close()
raise RuntimeError("未找到用户结果（可能被拦截/需要更完整登录态）")
@@ -555,6 +570,7 @@ def resolve_profile_by_search(
for i in range(n):
it = items.nth(i)
_log(log, f"[账号] 分析第 {i+1}/{n} 个结果...")
if use_anchor_only:
href = (it.get_attribute("href") or "").strip()
nickname = ""
@@ -618,6 +634,8 @@ def resolve_profile_by_search(
if _extract_profile_id_from_href(href):
score += 5
_log(log, f"[账号] - 昵称: {nickname[:30] if len(nickname) > 30 else nickname}, 账号: {xhs_no}, 得分: {score}")
if score > best_score:
best_score = score
url = "https://www.xiaohongshu.com" + href if href.startswith("/") else href