This commit is contained in:
mzl
2026-01-12 18:58:23 +08:00
parent fb98a5d221
commit f2dcc4c0cf
13 changed files with 1183 additions and 1194 deletions
+31 -25
View File
@@ -467,16 +467,20 @@ class XiaoHongShuScraper:
await self.emit_log("INFO", "启动单线程采集模式")
# 创建 Context 并登录
context = await self.browser.new_context(
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
user_agent=config.USER_AGENT
)
# 检查是否有保存的登录状态
# 创建 Context(优先加载已保存的 storage_state)
if os.path.exists(config.AUTH_STATE_FILE):
await self.emit_log("INFO", "加载已保存的登录状态")
await context.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
await self.emit_log("INFO", "加载已保存的登录状态 (storage_state)")
context = await self.browser.new_context(
storage_state=config.AUTH_STATE_FILE,
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
user_agent=config.USER_AGENT
)
else:
context = await self.browser.new_context(
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
user_agent=config.USER_AGENT
)
# 需要登录
login_success = await self.handle_login(context)
if not login_success:
@@ -559,29 +563,31 @@ class XiaoHongShuScraper:
"""多线程模式"""
await self.emit_log("INFO", f"启动多线程采集模式,线程数: {max_threads}")
# 创建多个 Context
# 创建多个 Context(同一账号并发会增加风控风险,请合理控制线程数)
contexts = []
for i in range(max_threads):
ctx = await self.browser.new_context(
# 确保已有登录态;若没有,先用第一个 context 登录并保存
if not os.path.exists(config.AUTH_STATE_FILE):
first = await self.browser.new_context(
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
user_agent=config.USER_AGENT
)
login_success = await self.handle_login(first)
if not login_success:
await first.close()
return
await first.close()
# 第一个 Context 登录,其他复制 Cookies
if i == 0:
if os.path.exists(config.AUTH_STATE_FILE):
await self.emit_log("INFO", "加载已保存的登录状态")
await ctx.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
else:
login_success = await self.handle_login(ctx)
if not login_success:
await ctx.close()
return
else:
await self.copy_cookies(ctx, contexts[0])
await self.emit_log("INFO", f"Thread-{i+1} 已复制登录状态")
# 基于 storage_state 为每个线程创建独立 context
for i in range(max_threads):
ctx = await self.browser.new_context(
storage_state=config.AUTH_STATE_FILE,
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
user_agent=config.USER_AGENT
)
contexts.append(ctx)
await self.emit_log("INFO", f"Thread-{i+1} context ready")
# 创建任务队列
task_queue = asyncio.Queue()