v10.5: 触底检测(.recommend-indicator'已经到底啦')→立即刷新页面取新数据, 不等200人累计阈值
This commit is contained in:
@@ -386,6 +386,18 @@ async function scrapeZhaopin(maxCount = 20, fullMode = false, scoreOnly = false)
|
||||
if (currentDataIndex === null || processed.has(currentDataIndex)) {
|
||||
await page.evaluate(() => window.scrollBy(0, window.innerHeight * 0.8));
|
||||
await randomSleep(3, 5);
|
||||
// v10.5:触底检测——出现"已经到底啦"指示器说明本批数据加载完,直接刷新页面拿新数据(不等累计人数阈值)
|
||||
const hitBottom = await page.evaluate(() => !!document.querySelector('.recommend-indicator') &&
|
||||
/已经到底/.test(document.querySelector('.recommend-indicator')?.innerText || ''));
|
||||
if (hitBottom) {
|
||||
console.log(' ⤓ 页面已触底(本批数据加载完),刷新页面获取新数据...');
|
||||
await page.reload({ waitUntil: 'domcontentloaded', timeout: 60000 }).catch(e => console.log(`刷新失败: ${e.message}`));
|
||||
await randomSleep(5, 10);
|
||||
processed = new Set();
|
||||
noCardCount = 0;
|
||||
currentDataIndex = null; // 重新从头扫描(启动跳过逻辑会略过已处理的卡片)
|
||||
continue;
|
||||
}
|
||||
const idx = await page.evaluate(() => document.querySelector('.recommend-item')?.getAttribute('data-index'));
|
||||
if (idx === null && ++noCardCount >= 5) { console.log('连续无新卡片,结束'); break; }
|
||||
currentDataIndex = idx; continue;
|
||||
|
||||
Reference in New Issue
Block a user