first commit

2026-01-12 12:04:05 +08:00
commit 8b6e431c12
17 changed files with 3358 additions and 0 deletions
@@ -0,0 +1,115 @@
+"""
+XiaoHongShu Scraper Configuration
+采集器配置文件
+"""
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+# ========== 基础配置 ==========
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DATA_DIR = os.path.join(BASE_DIR, "data")
+SCREENSHOT_DIR = os.path.join(DATA_DIR, "screenshots")
+
+# 确保目录存在
+os.makedirs(DATA_DIR, exist_ok=True)
+os.makedirs(SCREENSHOT_DIR, exist_ok=True)
+
+
+# ========== 采集字段配置 ==========
+FIELDS = ['title', 'publish_time', 'location', 'comment_count', 'like_count', 'collect_count', 'note_url']
+
+
+# ========== 延迟配置（秒）==========
+PAGE_LOAD_DELAY = (2, 5)        # 页面加载后等待
+NOTE_READ_DELAY = (3, 8)        # 笔记详情页停留（模拟阅读）
+NOTE_INTERVAL_DELAY = (3, 6)    # 采集完一个笔记后的间隔
+USER_SWITCH_DELAY = (30, 60)    # 切换用户时的间隔
+
+
+# ========== 多线程配置 ==========
+THREAD_MIN_DELAY = 2            # 线程间最小间隔（秒）
+THREAD_OFFSET = 5               # 线程启动偏移（秒）
+MAX_CONCURRENT_THREADS = 10     # 最大线程数
+
+
+# ========== 滚动配置 ==========
+SCROLL_STEP_MIN = 100           # 每次滚动最小像素
+SCROLL_STEP_MAX = 300           # 每次滚动最大像素
+SCROLL_PAUSE_MIN = 0.5          # 滚动后暂停最小时间（秒）
+SCROLL_PAUSE_MAX = 2.0          # 滚动后暂停最大时间（秒）
+MAX_SCROLL_COUNT = 50           # 最大滚动次数（防止无限滚动）
+
+
+# ========== 风控检测关键词 ==========
+RISK_KEYWORDS = [
+    '验证码',
+    '访问过于频繁',
+    '页面不存在',
+    '验证',
+    '流量异常',
+    '安全验证',
+    '人机验证',
+    '滑块验证'
+]
+
+
+# ========== 小红书 URL 配置 ==========
+XIAOHONGSHU_BASE_URL = "https://www.xiaohongshu.com"
+USER_PROFILE_URL_TEMPLATE = f"{XIAOHONGSHU_BASE_URL}/user/profile/{{user_id}}"
+
+
+# ========== 浏览器配置 ==========
+HEADLESS = os.getenv("HEADLESS", "false").lower() == "true"
+BROWSER_ARGS = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-blink-features=AutomationControlled'
+]
+
+# 视口配置
+VIEWPORT_WIDTH = 1920
+VIEWPORT_HEIGHT = 1080
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+
+
+# ========== 日志配置 ==========
+LOG_FILE = os.path.join(DATA_DIR, "logs.json")
+MAX_LOGS = 10000                # 最大日志条数
+
+
+# ========== Web 配置 ==========
+WEB_HOST = os.getenv("HOST", "0.0.0.0")
+WEB_PORT = int(os.getenv("PORT", "18080"))  # 默认端口 18080，范围 10000-50000
+SCREENSHOT_ON_ERROR = os.getenv("SCREENSHOT_ON_ERROR", "true").lower() == "true"
+
+# ========== Git 更新配置 ==========
+ENABLE_GIT_PULL = os.getenv("ENABLE_GIT_PULL", "false").lower() == "true"
+GIT_REPO_URL = os.getenv("GIT_REPO_URL", "")
+GIT_BRANCH = os.getenv("GIT_BRANCH", "main")
+AUTO_PULL_INTERVAL = int(os.getenv("AUTO_PULL_INTERVAL", "3600"))  # 自动拉取间隔（秒），默认1小时
+
+
+# ========== CSS 选择器（可能需要根据小红书页面结构调整）==========
+# 用户主页
+SELECTOR_USER_CONTAINER = "div.user-notes"
+SELECTOR_NOTE_CARD = "div[class*='note-item'], section[class*='note']"
+SELECTOR_NOTE_LINK = "a[href*='/explore/']"
+SELECTOR_NOTE_TITLE = "span[class*='title'], div[class*='title']"
+SELECTOR_NOTE_CONTENT = "div[class*='content'], span[class*='desc']"
+
+# 笔记详情页
+SELECTOR_DETAIL_TITLE = "div[class*='title'], span[class*='title']"
+SELECTOR_DETAIL_PUBLISH_TIME = "span[class*='time'], div[class*='time']"
+SELECTOR_DETAIL_LOCATION = "span[class*='location'], div[class*='location']"
+SELECTOR_DETAIL_COMMENT_COUNT = "span[class*='comment'], button[class*='comment']"
+SELECTOR_DETAIL_LIKE_COUNT = "span[class*='like'], button[class*='like']"
+SELECTOR_DETAIL_COLLECT_COUNT = "span[class*='collect'], button[class*='star']"
+
+
+# ========== 登录配置 ==========
+LOGIN_URL = f"{XIAOHONGSHU_BASE_URL}/login"
+AUTH_STATE_FILE = os.path.join(DATA_DIR, "auth_state.json")
+LOGIN_TIMEOUT = 300             # 登录等待超时（秒）