fix(scraper): 页面抓取器支持员工列表翻页(之前只抓第一页10个)
This commit is contained in:
@@ -209,11 +209,42 @@ class PageScraper:
|
|||||||
return result;
|
return result;
|
||||||
}""")
|
}""")
|
||||||
|
|
||||||
# Also paginate through staff list if needed
|
# Paginate through ALL staff list pages
|
||||||
# (for now, first page of 50 should cover most)
|
all_employees = list(employees)
|
||||||
|
page = 1
|
||||||
|
while await self.click_next_page():
|
||||||
|
page += 1
|
||||||
|
await self.page.wait_for_timeout(2000)
|
||||||
|
more = await self.page.evaluate("""() => {
|
||||||
|
const rows = document.querySelectorAll('.ant-table-row');
|
||||||
|
const result = [];
|
||||||
|
for (const tr of rows) {
|
||||||
|
const friendLink = tr.querySelector('a[href*="wechatdetail/1/"]');
|
||||||
|
const groupLink = tr.querySelector('a[href*="wechatdetail/2/"]');
|
||||||
|
if (friendLink) {
|
||||||
|
const href = friendLink.getAttribute('href');
|
||||||
|
const wxId = href.match(/wechatdetail\\/1\\/([^?]+)/)?.[1];
|
||||||
|
const friendCount = parseInt(friendLink.textContent) || 0;
|
||||||
|
const groupHref = groupLink?.getAttribute('href');
|
||||||
|
const groupCount = parseInt(groupLink?.textContent) || 0;
|
||||||
|
result.push({ wxId, friendCount, groupCount, friendHref: href, groupHref });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}""")
|
||||||
|
all_employees.extend(more)
|
||||||
|
logger.info(f" staff page {page}: +{len(more)} employees (total: {len(all_employees)})")
|
||||||
|
|
||||||
logger.info(f"Found {len(employees)} employees with friend links")
|
# Dedup by wxId
|
||||||
return employees
|
seen = set()
|
||||||
|
deduped = []
|
||||||
|
for emp in all_employees:
|
||||||
|
if emp["wxId"] not in seen:
|
||||||
|
seen.add(emp["wxId"])
|
||||||
|
deduped.append(emp)
|
||||||
|
|
||||||
|
logger.info(f"Found {len(deduped)} unique employees with friend links ({page} pages)")
|
||||||
|
return deduped
|
||||||
|
|
||||||
|
|
||||||
async def scrape_friends(cookie_str: str, max_employees: int = 100):
|
async def scrape_friends(cookie_str: str, max_employees: int = 100):
|
||||||
|
|||||||
Reference in New Issue
Block a user