diff --git a/yunke-server/scripts/scrape_via_page.py b/yunke-server/scripts/scrape_via_page.py index ef16181..7636b73 100644 --- a/yunke-server/scripts/scrape_via_page.py +++ b/yunke-server/scripts/scrape_via_page.py @@ -209,11 +209,42 @@ class PageScraper: return result; }""") - # Also paginate through staff list if needed - # (for now, first page of 50 should cover most) + # Paginate through ALL staff list pages + all_employees = list(employees) + page = 1 + while await self.click_next_page(): + page += 1 + await self.page.wait_for_timeout(2000) + more = await self.page.evaluate("""() => { + const rows = document.querySelectorAll('.ant-table-row'); + const result = []; + for (const tr of rows) { + const friendLink = tr.querySelector('a[href*="wechatdetail/1/"]'); + const groupLink = tr.querySelector('a[href*="wechatdetail/2/"]'); + if (friendLink) { + const href = friendLink.getAttribute('href'); + const wxId = href.match(/wechatdetail\\/1\\/([^?]+)/)?.[1]; + const friendCount = parseInt(friendLink.textContent) || 0; + const groupHref = groupLink?.getAttribute('href'); + const groupCount = parseInt(groupLink?.textContent) || 0; + result.push({ wxId, friendCount, groupCount, friendHref: href, groupHref }); + } + } + return result; + }""") + all_employees.extend(more) + logger.info(f" staff page {page}: +{len(more)} employees (total: {len(all_employees)})") - logger.info(f"Found {len(employees)} employees with friend links") - return employees + # Dedup by wxId + seen = set() + deduped = [] + for emp in all_employees: + if emp["wxId"] not in seen: + seen.add(emp["wxId"]) + deduped.append(emp) + + logger.info(f"Found {len(deduped)} unique employees with friend links ({page} pages)") + return deduped async def scrape_friends(cookie_str: str, max_employees: int = 100):