fix: whatsapp businessFriendList needs page+departmentId+type
Live validation: businessFriendList uses 'page' (not pageNum) and REQUIRES departmentId (root) + type or returns empty. Same class of bug as wxUser/list. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -203,9 +203,26 @@ def _parse_wa_friend_rows(resp: Any) -> list[dict[str, Any]]:
|
|||||||
return rows if isinstance(rows, list) else []
|
return rows if isinstance(rows, list) else []
|
||||||
|
|
||||||
|
|
||||||
def _build_wa_friend_body(page_num: int, page_size: int) -> dict[str, Any]:
|
def _build_wa_friend_body(
|
||||||
"""Build the ``businessFriendList`` request body for a given page."""
|
page_num: int, page_size: int, *, department_id: str = ""
|
||||||
return {"pageNum": page_num, "pageSize": page_size}
|
) -> dict[str, Any]:
|
||||||
|
"""Build the ``businessFriendList`` request body for a given page.
|
||||||
|
|
||||||
|
Live capture: ``{"searchInfo":"","friendSearchInfo":"","page":1,
|
||||||
|
"pageSize":10,"departmentId":"<root>","type":"0"}``. Note this endpoint
|
||||||
|
uses ``page`` (NOT ``pageNum`` like businessSalesList) and REQUIRES
|
||||||
|
``departmentId`` (the company-root dept id) + ``type`` or it returns empty.
|
||||||
|
"""
|
||||||
|
body: dict[str, Any] = {
|
||||||
|
"searchInfo": "",
|
||||||
|
"friendSearchInfo": "",
|
||||||
|
"page": page_num,
|
||||||
|
"pageSize": page_size,
|
||||||
|
"type": "0",
|
||||||
|
}
|
||||||
|
if department_id:
|
||||||
|
body["departmentId"] = department_id
|
||||||
|
return body
|
||||||
|
|
||||||
|
|
||||||
def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]:
|
def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]:
|
||||||
@@ -246,6 +263,7 @@ def _normalize_whatsapp_friend(record: dict[str, Any]) -> dict[str, Any]:
|
|||||||
async def scrape_whatsapp_friends(
|
async def scrape_whatsapp_friends(
|
||||||
client: YunkeClient,
|
client: YunkeClient,
|
||||||
*,
|
*,
|
||||||
|
department_id: str = "",
|
||||||
page_size: int = 50,
|
page_size: int = 50,
|
||||||
) -> AsyncIterator[dict[str, Any]]:
|
) -> AsyncIterator[dict[str, Any]]:
|
||||||
"""Yield normalized WhatsApp contact records from ``businessFriendList``.
|
"""Yield normalized WhatsApp contact records from ``businessFriendList``.
|
||||||
@@ -270,6 +288,7 @@ async def scrape_whatsapp_friends(
|
|||||||
parse_total=_parse_wa_friend_total,
|
parse_total=_parse_wa_friend_total,
|
||||||
parse_rows=_parse_wa_friend_rows,
|
parse_rows=_parse_wa_friend_rows,
|
||||||
build_body=_build_wa_friend_body,
|
build_body=_build_wa_friend_body,
|
||||||
|
department_id=department_id,
|
||||||
):
|
):
|
||||||
yield _normalize_whatsapp_friend(record)
|
yield _normalize_whatsapp_friend(record)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
"""Seed a slice of WhatsApp friends + one conversation's messages into smoke.db."""
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
from app.db import models # noqa: F401
|
||||||
|
from app.scraper.wechat import scrape_departments
|
||||||
|
from app.scraper.whatsapp import scrape_whatsapp_friends, scrape_whatsapp_messages
|
||||||
|
from app.storage.repos import upsert_customer, upsert_messages
|
||||||
|
from scripts.smoke_wechat import _push_settings_into_environ, build_client_from_env
|
||||||
|
|
||||||
|
# A known active WhatsApp conversation (Pasha-hesucar <-> Алекс).
|
||||||
|
UID = "15638182659@s.whatsapp.net"
|
||||||
|
FRIEND_UID = "375292581799@s.whatsapp.net"
|
||||||
|
|
||||||
|
|
||||||
|
async def _root_department_id(client) -> str | None:
|
||||||
|
async for d in scrape_departments(client):
|
||||||
|
if not d.get("parent_id"):
|
||||||
|
return d.get("dept_id")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
_push_settings_into_environ()
|
||||||
|
client = build_client_from_env()
|
||||||
|
engine = create_async_engine("sqlite+aiosqlite:///./smoke.db")
|
||||||
|
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
|
||||||
|
root_dept = await _root_department_id(client)
|
||||||
|
friends = []
|
||||||
|
async for rec in scrape_whatsapp_friends(
|
||||||
|
client, department_id=root_dept or "", page_size=10
|
||||||
|
):
|
||||||
|
friends.append(rec)
|
||||||
|
if len(friends) >= 10:
|
||||||
|
break
|
||||||
|
|
||||||
|
now = int(time.time() * 1000)
|
||||||
|
msgs = []
|
||||||
|
async for m in scrape_whatsapp_messages(client, UID, FRIEND_UID, now - 60 * 86_400_000, now):
|
||||||
|
msgs.append(m)
|
||||||
|
if len(msgs) >= 40:
|
||||||
|
break
|
||||||
|
|
||||||
|
async with sm() as session:
|
||||||
|
for rec in friends:
|
||||||
|
await upsert_customer(session, rec)
|
||||||
|
nmsg = await upsert_messages(session, msgs)
|
||||||
|
await session.commit()
|
||||||
|
print(f"upserted {len(friends)} whatsapp friends, {nmsg} whatsapp messages")
|
||||||
|
await client.aclose()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -130,7 +130,12 @@ async def test_scrape_whatsapp_friends_sends_pagination_body():
|
|||||||
|
|
||||||
client = _primed_client()
|
client = _primed_client()
|
||||||
try:
|
try:
|
||||||
results = [r async for r in scrape_whatsapp_friends(client, page_size=1)]
|
results = [
|
||||||
|
r
|
||||||
|
async for r in scrape_whatsapp_friends(
|
||||||
|
client, department_id="ROOT_DEPT", page_size=1
|
||||||
|
)
|
||||||
|
]
|
||||||
finally:
|
finally:
|
||||||
await client.aclose()
|
await client.aclose()
|
||||||
|
|
||||||
@@ -138,11 +143,14 @@ async def test_scrape_whatsapp_friends_sends_pagination_body():
|
|||||||
assert route.call_count == 2
|
assert route.call_count == 2
|
||||||
|
|
||||||
body1 = _json.loads(route.calls[0].request.content.decode())
|
body1 = _json.loads(route.calls[0].request.content.decode())
|
||||||
assert body1["pageNum"] == 1
|
# businessFriendList uses `page` (not pageNum) + departmentId + type.
|
||||||
|
assert body1["page"] == 1
|
||||||
assert body1["pageSize"] == 1
|
assert body1["pageSize"] == 1
|
||||||
|
assert body1["departmentId"] == "ROOT_DEPT"
|
||||||
|
assert body1["type"] == "0"
|
||||||
|
|
||||||
body2 = _json.loads(route.calls[1].request.content.decode())
|
body2 = _json.loads(route.calls[1].request.content.decode())
|
||||||
assert body2["pageNum"] == 2
|
assert body2["page"] == 2
|
||||||
assert body2["pageSize"] == 1
|
assert body2["pageSize"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user