diff --git a/yunke-server/app/scraper/whatsapp.py b/yunke-server/app/scraper/whatsapp.py index d2d3a0b..5319b69 100644 --- a/yunke-server/app/scraper/whatsapp.py +++ b/yunke-server/app/scraper/whatsapp.py @@ -203,9 +203,26 @@ def _parse_wa_friend_rows(resp: Any) -> list[dict[str, Any]]: return rows if isinstance(rows, list) else [] -def _build_wa_friend_body(page_num: int, page_size: int) -> dict[str, Any]: - """Build the ``businessFriendList`` request body for a given page.""" - return {"pageNum": page_num, "pageSize": page_size} +def _build_wa_friend_body( + page_num: int, page_size: int, *, department_id: str = "" +) -> dict[str, Any]: + """Build the ``businessFriendList`` request body for a given page. + + Live capture: ``{"searchInfo":"","friendSearchInfo":"","page":1, + "pageSize":10,"departmentId":"","type":"0"}``. Note this endpoint + uses ``page`` (NOT ``pageNum`` like businessSalesList) and REQUIRES + ``departmentId`` (the company-root dept id) + ``type`` or it returns empty. + """ + body: dict[str, Any] = { + "searchInfo": "", + "friendSearchInfo": "", + "page": page_num, + "pageSize": page_size, + "type": "0", + } + if department_id: + body["departmentId"] = department_id + return body def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]: @@ -246,6 +263,7 @@ def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]: async def scrape_whatsapp_friends( client: YunkeClient, *, + department_id: str = "", page_size: int = 50, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized WhatsApp contact records from ``businessFriendList``. @@ -270,6 +288,7 @@ async def scrape_whatsapp_friends( parse_total=_parse_wa_friend_total, parse_rows=_parse_wa_friend_rows, build_body=_build_wa_friend_body, + department_id=department_id, ): yield _normalize_whatsapp_friend(record) diff --git a/yunke-server/scripts/seed_whatsapp.py b/yunke-server/scripts/seed_whatsapp.py new file mode 100644 index 0000000..f3d2b45 --- /dev/null +++ b/yunke-server/scripts/seed_whatsapp.py @@ -0,0 +1,58 @@ +"""Seed a slice of WhatsApp friends + one conversation's messages into smoke.db.""" +import asyncio +import time + +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.db import models # noqa: F401 +from app.scraper.wechat import scrape_departments +from app.scraper.whatsapp import scrape_whatsapp_friends, scrape_whatsapp_messages +from app.storage.repos import upsert_customer, upsert_messages +from scripts.smoke_wechat import _push_settings_into_environ, build_client_from_env + +# A known active WhatsApp conversation (Pasha-hesucar <-> Алекс). +UID = "15638182659@s.whatsapp.net" +FRIEND_UID = "375292581799@s.whatsapp.net" + + +async def _root_department_id(client) -> str | None: + async for d in scrape_departments(client): + if not d.get("parent_id"): + return d.get("dept_id") + return None + + +async def main() -> None: + _push_settings_into_environ() + client = build_client_from_env() + engine = create_async_engine("sqlite+aiosqlite:///./smoke.db") + sm = async_sessionmaker(engine, expire_on_commit=False) + + root_dept = await _root_department_id(client) + friends = [] + async for rec in scrape_whatsapp_friends( + client, department_id=root_dept or "", page_size=10 + ): + friends.append(rec) + if len(friends) >= 10: + break + + now = int(time.time() * 1000) + msgs = [] + async for m in scrape_whatsapp_messages(client, UID, FRIEND_UID, now - 60 * 86_400_000, now): + msgs.append(m) + if len(msgs) >= 40: + break + + async with sm() as session: + for rec in friends: + await upsert_customer(session, rec) + nmsg = await upsert_messages(session, msgs) + await session.commit() + print(f"upserted {len(friends)} whatsapp friends, {nmsg} whatsapp messages") + await client.aclose() + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/yunke-server/tests/test_scrape_whatsapp_friends.py b/yunke-server/tests/test_scrape_whatsapp_friends.py index ac9d810..f8d4a1b 100644 --- a/yunke-server/tests/test_scrape_whatsapp_friends.py +++ b/yunke-server/tests/test_scrape_whatsapp_friends.py @@ -130,7 +130,12 @@ async def test_scrape_whatsapp_friends_sends_pagination_body(): client = _primed_client() try: - results = [r async for r in scrape_whatsapp_friends(client, page_size=1)] + results = [ + r + async for r in scrape_whatsapp_friends( + client, department_id="ROOT_DEPT", page_size=1 + ) + ] finally: await client.aclose() @@ -138,11 +143,14 @@ async def test_scrape_whatsapp_friends_sends_pagination_body(): assert route.call_count == 2 body1 = _json.loads(route.calls[0].request.content.decode()) - assert body1["pageNum"] == 1 + # businessFriendList uses `page` (not pageNum) + departmentId + type. + assert body1["page"] == 1 assert body1["pageSize"] == 1 + assert body1["departmentId"] == "ROOT_DEPT" + assert body1["type"] == "0" body2 = _json.loads(route.calls[1].request.content.decode()) - assert body2["pageNum"] == 2 + assert body2["page"] == 2 assert body2["pageSize"] == 1