xhs_crawler_claude/app/config.py

"""
XiaoHongShu Scraper Configuration
采集器配置文件
"""
import os
from dotenv import load_dotenv

load_dotenv()


# ========== 基础配置 ==========
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, "data")
SCREENSHOT_DIR = os.path.join(DATA_DIR, "screenshots")

# 确保目录存在
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(SCREENSHOT_DIR, exist_ok=True)


# ========== 采集字段配置 ==========
FIELDS = ['title', 'publish_time', 'location', 'comment_count', 'like_count', 'collect_count', 'note_url']


# ========== 延迟配置（秒）==========
PAGE_LOAD_DELAY = (2, 5)        # 页面加载后等待
NOTE_READ_DELAY = (3, 8)        # 笔记详情页停留（模拟阅读）
NOTE_INTERVAL_DELAY = (3, 6)    # 采集完一个笔记后的间隔
USER_SWITCH_DELAY = (30, 60)    # 切换用户时的间隔


# ========== 多线程配置 ==========
THREAD_MIN_DELAY = 2            # 线程间最小间隔（秒）
THREAD_OFFSET = 5               # 线程启动偏移（秒）
MAX_CONCURRENT_THREADS = 10     # 最大线程数


# ========== 滚动配置 ==========
SCROLL_STEP_MIN = 100           # 每次滚动最小像素
SCROLL_STEP_MAX = 300           # 每次滚动最大像素
SCROLL_PAUSE_MIN = 0.5          # 滚动后暂停最小时间（秒）
SCROLL_PAUSE_MAX = 2.0          # 滚动后暂停最大时间（秒）
MAX_SCROLL_COUNT = 50           # 最大滚动次数（防止无限滚动）


# ========== 风控检测关键词 ==========
RISK_KEYWORDS = [
    '验证码',
    '访问过于频繁',
    '页面不存在',
    '验证',
    '流量异常',
    '安全验证',
    '人机验证',
    '滑块验证'
]


# ========== 小红书 URL 配置 ==========
XIAOHONGSHU_BASE_URL = "https://www.xiaohongshu.com"
USER_PROFILE_URL_TEMPLATE = f"{XIAOHONGSHU_BASE_URL}/user/profile/{{user_id}}"


# ========== 浏览器配置 ==========
HEADLESS = os.getenv("HEADLESS", "false").lower() == "true"
BROWSER_ARGS = [
    '--no-sandbox',
    '--disable-setuid-sandbox',
    '--disable-blink-features=AutomationControlled'
]

# 视口配置
VIEWPORT_WIDTH = 1920
VIEWPORT_HEIGHT = 1080
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"


# ========== 日志配置 ==========
LOG_FILE = os.path.join(DATA_DIR, "logs.json")
MAX_LOGS = 10000                # 最大日志条数


# ========== Web 配置 ==========
WEB_HOST = os.getenv("HOST", "0.0.0.0")
WEB_PORT = int(os.getenv("PORT", "18085"))  # 默认端口 18085，范围 10000-50000
SCREENSHOT_ON_ERROR = os.getenv("SCREENSHOT_ON_ERROR", "true").lower() == "true"

# ========== Git 更新配置 ==========
ENABLE_GIT_PULL = os.getenv("ENABLE_GIT_PULL", "false").lower() == "true"
GIT_REPO_URL = os.getenv("GIT_REPO_URL", "")
GIT_BRANCH = os.getenv("GIT_BRANCH", "main")
AUTO_PULL_INTERVAL = int(os.getenv("AUTO_PULL_INTERVAL", "3600"))  # 自动拉取间隔（秒），默认1小时


# ========== CSS 选择器（可能需要根据小红书页面结构调整）==========
# 用户主页
SELECTOR_USER_CONTAINER = "div.user-notes"
SELECTOR_NOTE_CARD = "div[class*='note-item'], section[class*='note']"
SELECTOR_NOTE_LINK = "a[href*='/explore/']"
SELECTOR_NOTE_TITLE = "span[class*='title'], div[class*='title']"
SELECTOR_NOTE_CONTENT = "div[class*='content'], span[class*='desc']"

# 笔记详情页
SELECTOR_DETAIL_TITLE = "div[class*='title'], span[class*='title']"
SELECTOR_DETAIL_PUBLISH_TIME = "span[class*='time'], div[class*='time']"
SELECTOR_DETAIL_LOCATION = "span[class*='location'], div[class*='location']"
SELECTOR_DETAIL_COMMENT_COUNT = "span[class*='comment'], button[class*='comment']"
SELECTOR_DETAIL_LIKE_COUNT = "span[class*='like'], button[class*='like']"
SELECTOR_DETAIL_COLLECT_COUNT = "span[class*='collect'], button[class*='star']"


# ========== 登录配置 ==========
LOGIN_URL = f"{XIAOHONGSHU_BASE_URL}/login"
AUTH_STATE_FILE = os.path.join(DATA_DIR, "auth_state.json")
LOGIN_TIMEOUT = 300             # 登录等待超时（秒）