fix(scraper): 页面抓取器支持员工列表翻页(之前只抓第一页10个)
This commit is contained in:
@@ -209,11 +209,42 @@ class PageScraper:
|
||||
return result;
|
||||
}""")
|
||||
|
||||
# Also paginate through staff list if needed
|
||||
# (for now, first page of 50 should cover most)
|
||||
# Paginate through ALL staff list pages
|
||||
all_employees = list(employees)
|
||||
page = 1
|
||||
while await self.click_next_page():
|
||||
page += 1
|
||||
await self.page.wait_for_timeout(2000)
|
||||
more = await self.page.evaluate("""() => {
|
||||
const rows = document.querySelectorAll('.ant-table-row');
|
||||
const result = [];
|
||||
for (const tr of rows) {
|
||||
const friendLink = tr.querySelector('a[href*="wechatdetail/1/"]');
|
||||
const groupLink = tr.querySelector('a[href*="wechatdetail/2/"]');
|
||||
if (friendLink) {
|
||||
const href = friendLink.getAttribute('href');
|
||||
const wxId = href.match(/wechatdetail\\/1\\/([^?]+)/)?.[1];
|
||||
const friendCount = parseInt(friendLink.textContent) || 0;
|
||||
const groupHref = groupLink?.getAttribute('href');
|
||||
const groupCount = parseInt(groupLink?.textContent) || 0;
|
||||
result.push({ wxId, friendCount, groupCount, friendHref: href, groupHref });
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}""")
|
||||
all_employees.extend(more)
|
||||
logger.info(f" staff page {page}: +{len(more)} employees (total: {len(all_employees)})")
|
||||
|
||||
logger.info(f"Found {len(employees)} employees with friend links")
|
||||
return employees
|
||||
# Dedup by wxId
|
||||
seen = set()
|
||||
deduped = []
|
||||
for emp in all_employees:
|
||||
if emp["wxId"] not in seen:
|
||||
seen.add(emp["wxId"])
|
||||
deduped.append(emp)
|
||||
|
||||
logger.info(f"Found {len(deduped)} unique employees with friend links ({page} pages)")
|
||||
return deduped
|
||||
|
||||
|
||||
async def scrape_friends(cookie_str: str, max_employees: int = 100):
|
||||
|
||||
Reference in New Issue
Block a user