fix(scraper): 页面抓取器支持员工列表翻页(之前只抓第一页10个)

This commit is contained in:
浅念
2026-08-14 09:25:10 +08:00
parent 004e6e4d6b
commit bead7fdf43
+35 -4
View File
@@ -209,11 +209,42 @@ class PageScraper:
return result;
}""")
# Also paginate through staff list if needed
# (for now, first page of 50 should cover most)
# Paginate through ALL staff list pages
all_employees = list(employees)
page = 1
while await self.click_next_page():
page += 1
await self.page.wait_for_timeout(2000)
more = await self.page.evaluate("""() => {
const rows = document.querySelectorAll('.ant-table-row');
const result = [];
for (const tr of rows) {
const friendLink = tr.querySelector('a[href*="wechatdetail/1/"]');
const groupLink = tr.querySelector('a[href*="wechatdetail/2/"]');
if (friendLink) {
const href = friendLink.getAttribute('href');
const wxId = href.match(/wechatdetail\\/1\\/([^?]+)/)?.[1];
const friendCount = parseInt(friendLink.textContent) || 0;
const groupHref = groupLink?.getAttribute('href');
const groupCount = parseInt(groupLink?.textContent) || 0;
result.push({ wxId, friendCount, groupCount, friendHref: href, groupHref });
}
}
return result;
}""")
all_employees.extend(more)
logger.info(f" staff page {page}: +{len(more)} employees (total: {len(all_employees)})")
logger.info(f"Found {len(employees)} employees with friend links")
return employees
# Dedup by wxId
seen = set()
deduped = []
for emp in all_employees:
if emp["wxId"] not in seen:
seen.add(emp["wxId"])
deduped.append(emp)
logger.info(f"Found {len(deduped)} unique employees with friend links ({page} pages)")
return deduped
async def scrape_friends(cookie_str: str, max_employees: int = 100):