From 281cedbed1eaf4208bd2804d0c609f91e721181f Mon Sep 17 00:00:00 2001 From: mzl Date: Sun, 11 Jan 2026 11:06:07 +0800 Subject: [PATCH] ts --- xhs_crawler_core.py | 90 ++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 55 deletions(-) diff --git a/xhs_crawler_core.py b/xhs_crawler_core.py index f4fe757..c7d2d95 100644 --- a/xhs_crawler_core.py +++ b/xhs_crawler_core.py @@ -2,15 +2,13 @@ # 核心功能: # - login_mode_auto(): 打开浏览器手动登录,检测到已登录后自动保存 storage_state.json # - check_login_state_with_storage(): 用 storage_state.json 检查登录是否有效 -# - resolve_profile_by_search(): 通过“搜索页-用户”解析出个人主页 url + 昵称 + 小红书号(需要登录态) +# - resolve_profile_by_search(): 通过"搜索页-用户"解析出个人主页 url + 昵称 + 小红书号(需要登录态) # - crawl_detail(): 详情弹框采集(标题/时间地点/赞藏评),防漏采滚动策略(需要登录态) import re import json import random import urllib.parse -import os -import sys import shutil # Excel 导出(可选) @@ -74,36 +72,9 @@ USER_DESC_SELECTOR = ".user-desc" # 未登录/拦截时常见文本(兜底判断) BLOCK_TEXTS_STRONG = ["扫码登录", "安全验证", "请先登录", "登录后查看"] -def _winreg_app_path(exe_name: str) -> str: - """Windows: 从注册表 App Paths 获取浏览器路径(支持非 C 盘安装)""" - if sys.platform != "win32": - return "" - try: - import winreg - except Exception: - return "" - subkey = fr"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\{exe_name}" - for root in (getattr(winreg, "HKEY_CURRENT_USER", None), getattr(winreg, "HKEY_LOCAL_MACHINE", None)): - if root is None: - continue - try: - k = winreg.OpenKey(root, subkey) - try: - val, _ = winreg.QueryValueEx(k, "") - val = (val or "").strip() - if val and Path(val).exists(): - return val - finally: - try: - winreg.CloseKey(k) - except Exception: - pass - except Exception: - pass - return "" - def _which(cmd: str) -> str: + """在 Linux 系统上查找命令路径""" try: return shutil.which(cmd) or "" except Exception: @@ -111,30 +82,23 @@ def _which(cmd: str) -> str: def find_system_browser_executable() -> str: - """自动探测系统 Chrome/Edge 可执行文件路径""" - chrome = _winreg_app_path("chrome.exe") - if chrome: - return chrome - edge = _winreg_app_path("msedge.exe") - if edge: - return edge - - candidates = [ - r"C:\Program Files\Google\Chrome\Application\chrome.exe", - r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", - os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe"), - r"C:\Program Files\Microsoft\Edge\Application\msedge.exe", - r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe", - os.path.expandvars(r"%LOCALAPPDATA%\Microsoft\Edge\Application\msedge.exe"), + """ + 自动探测系统 Chrome/Chromium 可执行文件路径(Linux) + 常见安装路径: + - google-chrome: /usr/bin/google-chrome + - chromium: /usr/bin/chromium-browser + - chromium: /usr/bin/chromium + """ + # Linux 常见的 Chrome/Chromium 命令 + commands = [ + "google-chrome", + "google-chrome-stable", + "chromium-browser", + "chromium", + "chrome", ] - for p in candidates: - try: - if p and Path(p).exists(): - return p - except Exception: - pass - for cmd in ("chrome", "chrome.exe", "msedge", "msedge.exe"): + for cmd in commands: p = _which(cmd) if p: try: @@ -143,11 +107,27 @@ def find_system_browser_executable() -> str: except Exception: pass + # 常见路径(如果不在 PATH 中) + candidates = [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium-browser", + "/usr/bin/chromium", + "/opt/google/chrome/google-chrome", + "/snap/bin/chromium", + ] + for p in candidates: + try: + if Path(p).exists(): + return p + except Exception: + pass + return "" def pick_browser_executable(manual_path: str = "") -> str: - """优先使用手动路径;否则自动探测系统 Chrome/Edge;都没有则返回空字符串""" + """优先使用手动路径;否则自动探测系统 Chrome/Chromium;都没有则返回空字符串""" mp = (manual_path or "").strip().strip('"') if mp: try: @@ -1008,7 +988,7 @@ def crawl_detail( storage_state=str(pth), viewport={"width": 1366, "height": 820}, locale="zh-CN", - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", ) context.set_default_timeout(60000) page = context.new_page()