ts
This commit is contained in:
+31
-25
@@ -467,16 +467,20 @@ class XiaoHongShuScraper:
|
||||
await self.emit_log("INFO", "启动单线程采集模式")
|
||||
|
||||
# 创建 Context 并登录
|
||||
context = await self.browser.new_context(
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
|
||||
# 检查是否有保存的登录状态
|
||||
|
||||
# 创建 Context(优先加载已保存的 storage_state)
|
||||
if os.path.exists(config.AUTH_STATE_FILE):
|
||||
await self.emit_log("INFO", "加载已保存的登录状态")
|
||||
await context.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
|
||||
await self.emit_log("INFO", "加载已保存的登录状态 (storage_state)")
|
||||
context = await self.browser.new_context(
|
||||
storage_state=config.AUTH_STATE_FILE,
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
else:
|
||||
context = await self.browser.new_context(
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
# 需要登录
|
||||
login_success = await self.handle_login(context)
|
||||
if not login_success:
|
||||
@@ -559,29 +563,31 @@ class XiaoHongShuScraper:
|
||||
"""多线程模式"""
|
||||
await self.emit_log("INFO", f"启动多线程采集模式,线程数: {max_threads}")
|
||||
|
||||
# 创建多个 Context
|
||||
|
||||
# 创建多个 Context(同一账号并发会增加风控风险,请合理控制线程数)
|
||||
contexts = []
|
||||
for i in range(max_threads):
|
||||
ctx = await self.browser.new_context(
|
||||
|
||||
# 确保已有登录态;若没有,先用第一个 context 登录并保存
|
||||
if not os.path.exists(config.AUTH_STATE_FILE):
|
||||
first = await self.browser.new_context(
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
login_success = await self.handle_login(first)
|
||||
if not login_success:
|
||||
await first.close()
|
||||
return
|
||||
await first.close()
|
||||
|
||||
# 第一个 Context 登录,其他复制 Cookies
|
||||
if i == 0:
|
||||
if os.path.exists(config.AUTH_STATE_FILE):
|
||||
await self.emit_log("INFO", "加载已保存的登录状态")
|
||||
await ctx.add_cookies([json.load(open(config.AUTH_STATE_FILE))])
|
||||
else:
|
||||
login_success = await self.handle_login(ctx)
|
||||
if not login_success:
|
||||
await ctx.close()
|
||||
return
|
||||
else:
|
||||
await self.copy_cookies(ctx, contexts[0])
|
||||
await self.emit_log("INFO", f"Thread-{i+1} 已复制登录状态")
|
||||
|
||||
# 基于 storage_state 为每个线程创建独立 context
|
||||
for i in range(max_threads):
|
||||
ctx = await self.browser.new_context(
|
||||
storage_state=config.AUTH_STATE_FILE,
|
||||
viewport={'width': config.VIEWPORT_WIDTH, 'height': config.VIEWPORT_HEIGHT},
|
||||
user_agent=config.USER_AGENT
|
||||
)
|
||||
contexts.append(ctx)
|
||||
await self.emit_log("INFO", f"Thread-{i+1} context ready")
|
||||
|
||||
# 创建任务队列
|
||||
task_queue = asyncio.Queue()
|
||||
|
||||
Reference in New Issue
Block a user