diff --git a/xhs_crawler_core.py b/xhs_crawler_core.py index c7d2d95..78b8292 100644 --- a/xhs_crawler_core.py +++ b/xhs_crawler_core.py @@ -556,31 +556,15 @@ def resolve_profile_by_search( xhs_no = "" try: full_text = (it.inner_text() or "").strip() - _log(log, f"[调试] 原始文本: {full_text[:100]}") - - # 首先提取小红书号(尝试多种模式) - patterns = [ - r'小红书号[::]\s*([A-Za-z0-9]+)', - r'小红书号[::]\s*([A-Za-z0-9]+)', - r'ID[::]\s*([A-Za-z0-9]+)', - r'账号[::]\s*([A-Za-z0-9]+)', - ] - for pattern in patterns: - m = re.search(pattern, full_text) - if m: - xhs_no = m.group(1).strip() - _log(log, f"[调试] 匹配到小红书号: {xhs_no} (pattern: {pattern})") - break - - if not xhs_no: - _log(log, f"[调试] 未能匹配小红书号") + # 提取小红书号 + m = re.search(r'小红书号[::]\s*([A-Za-z0-9]+)', full_text) + if m: + xhs_no = m.group(1).strip() # 提取昵称:从开头去除干扰信息 lines = full_text.split('\n') if lines: main_text = lines[0].strip() - _log(log, f"[调试] 第一行文本: {main_text}") - # 移除小红书号及其后面的所有内容 main_text = re.sub(r'小红书号[::].*$', '', main_text).strip() # 移除时间信息 @@ -592,11 +576,9 @@ def resolve_profile_by_search( # 移除分隔符 main_text = re.sub(r'\s*[・·]\s*', ' ', main_text).strip() nickname = re.sub(r'\s+', ' ', main_text).strip() - _log(log, f"[调试] 清理后昵称: {nickname}") else: nickname = full_text - except Exception as e: - _log(log, f"[调试] 解析异常: {e}") + except Exception: nickname = "" xhs_no = "" else: @@ -777,12 +759,12 @@ def crawl_detail( slow_mo: int = 80, batch_min: int = 3, batch_max: int = 7, - open_wait: Tuple[int, int] = (1800, 3200), - between_wait: Tuple[int, int] = (1500, 3200), - scroll_wait: Tuple[int, int] = (1600, 2800), - long_break: Tuple[int, int] = (6000, 12000), + open_wait: Tuple[int, int] = (1200, 2000), # 风控安全:适度等待 + between_wait: Tuple[int, int] = (1000, 1800), # 风控安全:笔记间等待 + scroll_wait: Tuple[int, int] = (1000, 1800), # 风控安全:滚动等待 + long_break: Tuple[int, int] = (5000, 9000), # 风控安全:长休息时间 scroll_target_new: int = 2, - scroll_max_attempts: int = 10, + scroll_max_attempts: int = 8, # 风控安全:适度滚动次数 out_file: str = "out/result_modal_details.json", stop_event=None, log: Optional[Callable[[str], None]] = None, @@ -955,7 +937,7 @@ def crawl_detail( """) except Exception: pass - page.wait_for_timeout(random.randint(2000, 3500)) + page.wait_for_timeout(random.randint(1500, 2500)) # 风控安全:适度等待 after2 = count_visible_unprocessed(page, processed) if after2 > before: return True @@ -994,8 +976,8 @@ def crawl_detail( page = context.new_page() page.goto(profile_url, wait_until="domcontentloaded", timeout=60000) - page.wait_for_selector(NOTE_CONTAINER_SELECTOR, timeout=30000) - _wait_ms(page, (1200, 2200)) + page.wait_for_selector(NOTE_CONTAINER_SELECTOR, timeout=25000) # 风控安全:适度等待 + _wait_ms(page, (800, 1500)) # 风控安全:适度等待 safety_round = 0 while len(results) < limit and safety_round < 2000: @@ -1020,7 +1002,7 @@ def crawl_detail( item.scroll_into_view_if_needed() except Exception: pass - _wait_ms(page, (700, 1300)) + _wait_ms(page, (500, 900)) # 风控安全:适度等待 try: opener.click(timeout=6000) @@ -1031,14 +1013,14 @@ def crawl_detail( continue try: - # 关键:不要用 #noteContainer 这种“永远可见”的元素判断弹框是否打开 - page.wait_for_selector(DETAIL_TITLE_SELECTOR, state="visible", timeout=25000) + # 关键:不要用 #noteContainer 这种"永远可见"的元素判断弹框是否打开 + page.wait_for_selector(DETAIL_TITLE_SELECTOR, state="visible", timeout=15000) # 优化:从 25 秒降至 15 秒 except PWTimeoutError: - # 先做一次“强制点击+更长等待”的快速重试,尽量别直接跳过 + # 先做一次"强制点击+更长等待"的快速重试,尽量别直接跳过 retry_ok = False try: opener.click(timeout=6000, force=True) - page.wait_for_selector(DETAIL_TITLE_SELECTOR, state="visible", timeout=30000) + page.wait_for_selector(DETAIL_TITLE_SELECTOR, state="visible", timeout=15000) # 优化:从 30 秒降至 15 秒 retry_ok = True except Exception: retry_ok = False