116 lines
4.0 KiB
Python
116 lines
4.0 KiB
Python
"""
|
|
XiaoHongShu Scraper Configuration
|
|
采集器配置文件
|
|
"""
|
|
import os
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
|
|
# ========== 基础配置 ==========
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
DATA_DIR = os.path.join(BASE_DIR, "data")
|
|
SCREENSHOT_DIR = os.path.join(DATA_DIR, "screenshots")
|
|
|
|
# 确保目录存在
|
|
os.makedirs(DATA_DIR, exist_ok=True)
|
|
os.makedirs(SCREENSHOT_DIR, exist_ok=True)
|
|
|
|
|
|
# ========== 采集字段配置 ==========
|
|
FIELDS = ['title', 'publish_time', 'location', 'comment_count', 'like_count', 'collect_count', 'note_url']
|
|
|
|
|
|
# ========== 延迟配置（秒）==========
|
|
PAGE_LOAD_DELAY = (2, 5) # 页面加载后等待
|
|
NOTE_READ_DELAY = (3, 8) # 笔记详情页停留（模拟阅读）
|
|
NOTE_INTERVAL_DELAY = (3, 6) # 采集完一个笔记后的间隔
|
|
USER_SWITCH_DELAY = (30, 60) # 切换用户时的间隔
|
|
|
|
|
|
# ========== 多线程配置 ==========
|
|
THREAD_MIN_DELAY = 2 # 线程间最小间隔（秒）
|
|
THREAD_OFFSET = 5 # 线程启动偏移（秒）
|
|
MAX_CONCURRENT_THREADS = 10 # 最大线程数
|
|
|
|
|
|
# ========== 滚动配置 ==========
|
|
SCROLL_STEP_MIN = 100 # 每次滚动最小像素
|
|
SCROLL_STEP_MAX = 300 # 每次滚动最大像素
|
|
SCROLL_PAUSE_MIN = 0.5 # 滚动后暂停最小时间（秒）
|
|
SCROLL_PAUSE_MAX = 2.0 # 滚动后暂停最大时间（秒）
|
|
MAX_SCROLL_COUNT = 50 # 最大滚动次数（防止无限滚动）
|
|
|
|
|
|
# ========== 风控检测关键词 ==========
|
|
RISK_KEYWORDS = [
|
|
'验证码',
|
|
'访问过于频繁',
|
|
'页面不存在',
|
|
'验证',
|
|
'流量异常',
|
|
'安全验证',
|
|
'人机验证',
|
|
'滑块验证'
|
|
]
|
|
|
|
|
|
# ========== 小红书 URL 配置 ==========
|
|
XIAOHONGSHU_BASE_URL = "https://www.xiaohongshu.com"
|
|
USER_PROFILE_URL_TEMPLATE = f"{XIAOHONGSHU_BASE_URL}/user/profile/{{user_id}}"
|
|
|
|
|
|
# ========== 浏览器配置 ==========
|
|
HEADLESS = os.getenv("HEADLESS", "false").lower() == "true"
|
|
BROWSER_ARGS = [
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-blink-features=AutomationControlled'
|
|
]
|
|
|
|
# 视口配置
|
|
VIEWPORT_WIDTH = 1920
|
|
VIEWPORT_HEIGHT = 1080
|
|
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
|
|
|
|
|
# ========== 日志配置 ==========
|
|
LOG_FILE = os.path.join(DATA_DIR, "logs.json")
|
|
MAX_LOGS = 10000 # 最大日志条数
|
|
|
|
|
|
# ========== Web 配置 ==========
|
|
WEB_HOST = os.getenv("HOST", "0.0.0.0")
|
|
WEB_PORT = int(os.getenv("PORT", "18085")) # 默认端口 18085，范围 10000-50000
|
|
SCREENSHOT_ON_ERROR = os.getenv("SCREENSHOT_ON_ERROR", "true").lower() == "true"
|
|
|
|
# ========== Git 更新配置 ==========
|
|
ENABLE_GIT_PULL = os.getenv("ENABLE_GIT_PULL", "false").lower() == "true"
|
|
GIT_REPO_URL = os.getenv("GIT_REPO_URL", "")
|
|
GIT_BRANCH = os.getenv("GIT_BRANCH", "main")
|
|
AUTO_PULL_INTERVAL = int(os.getenv("AUTO_PULL_INTERVAL", "3600")) # 自动拉取间隔（秒），默认1小时
|
|
|
|
|
|
# ========== CSS 选择器（可能需要根据小红书页面结构调整）==========
|
|
# 用户主页
|
|
SELECTOR_USER_CONTAINER = "div.user-notes"
|
|
SELECTOR_NOTE_CARD = "div[class*='note-item'], section[class*='note']"
|
|
SELECTOR_NOTE_LINK = "a[href*='/explore/']"
|
|
SELECTOR_NOTE_TITLE = "span[class*='title'], div[class*='title']"
|
|
SELECTOR_NOTE_CONTENT = "div[class*='content'], span[class*='desc']"
|
|
|
|
# 笔记详情页
|
|
SELECTOR_DETAIL_TITLE = "div[class*='title'], span[class*='title']"
|
|
SELECTOR_DETAIL_PUBLISH_TIME = "span[class*='time'], div[class*='time']"
|
|
SELECTOR_DETAIL_LOCATION = "span[class*='location'], div[class*='location']"
|
|
SELECTOR_DETAIL_COMMENT_COUNT = "span[class*='comment'], button[class*='comment']"
|
|
SELECTOR_DETAIL_LIKE_COUNT = "span[class*='like'], button[class*='like']"
|
|
SELECTOR_DETAIL_COLLECT_COUNT = "span[class*='collect'], button[class*='star']"
|
|
|
|
|
|
# ========== 登录配置 ==========
|
|
LOGIN_URL = f"{XIAOHONGSHU_BASE_URL}/login"
|
|
AUTH_STATE_FILE = os.path.join(DATA_DIR, "auth_state.json")
|
|
LOGIN_TIMEOUT = 300 # 登录等待超时（秒）
|