浅念
0df53c6b1a
Backend (scrape_wechat_accounts): - Send wxUser/list as form-urlencoded (not JSON) with the REQUIRED filters isOnJob / departmentId / startdate / enddate + empty userId/salesname/ weChatNickName, matching the live request (empty fails with code -90000). - scrape_wechat_accounts now takes department_id (the company-root dept id) plus optional start/end defaulting to a recent 7-day UTC window. - paginate_by_page gains a form=True mode (data= + form content-type). - Orchestrator sync_wechat_accounts resolves the root dept from the Department table (or accepts one) and skips with a warning when unknown. - seed_accounts.py resolves the root via scrape_departments. Frontend (yunkes): - Add StaffList.vue (/staff) with is_on_job filter + pagination. - Add /staff route and a 员工 nav item between 客户 and 群. Co-Authored-By: Claude <noreply@anthropic.com>
254 lines
8.8 KiB
Python
254 lines
8.8 KiB
Python
"""Tests for the WeChat staff-account scraper (Task 2.1).
|
|
|
|
Drives the real page-number pagination + normalization logic through
|
|
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
|
|
login flow is skipped) against a mocked transport.
|
|
|
|
The fixture ships 4 records across 2 pages of size 2, plus a terminating
|
|
empty page (the documented primary stop signal for page-number endpoints):
|
|
|
|
* page 1 -> 2 records, ``total=4``
|
|
* page 2 -> 2 records, ``total=4``
|
|
* page 3 -> empty ``data`` array (no more rows)
|
|
|
|
The scraper must yield exactly the 4 records (normalized to ``WechatAccount``
|
|
snake_case columns + ``raw_json``), make exactly 3 calls, and stop without a
|
|
4th. The empty-page stop is the primary signal; ``total=4`` is only a
|
|
defensive over-fetch ceiling that must NOT trigger early (we yield exactly
|
|
4 == total, so the strict-``>`` ceiling never fires here).
|
|
|
|
The REQUEST body is form-urlencoded (NOT JSON) and must carry the required
|
|
``isOnJob`` / ``departmentId`` / ``startdate`` / ``enddate`` filters; the
|
|
"send pagination body" test asserts those keys land on the wire.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from urllib.parse import parse_qs
|
|
|
|
import httpx
|
|
import respx
|
|
|
|
from app.scraper.base import FORM_CONTENT_TYPE
|
|
from app.scraper.wechat import scrape_wechat_accounts
|
|
from app.session.yunke_client import YunkeClient
|
|
from tests.cassettes.wxUser_fixture import (
|
|
DEFAULT_PAGE_SIZE,
|
|
TOTAL,
|
|
WXUSER_RECORDS,
|
|
make_wxuser_list_response,
|
|
)
|
|
|
|
BASE = "https://crm2.yunkecn.com"
|
|
WXUSER_LIST_URL = f"{BASE}/wechatweb/wxUser/list"
|
|
|
|
USERID = "a" * 32
|
|
COMPANY = "8m0atd"
|
|
|
|
# The company-root department id used as the required ``departmentId`` filter.
|
|
# Mirrors the shape of the real root dept id (32 hex chars) without hardcoding
|
|
# the live value in the scraper itself.
|
|
ROOT_DEPARTMENT_ID = "0E9C4B803E134C7687CBA824886319DA"
|
|
|
|
|
|
def _primed_client() -> YunkeClient:
|
|
"""A YunkeClient already in a logged-in state (login flow skipped)."""
|
|
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
|
|
|
|
|
|
def _parse_form(request: httpx.Request) -> dict[str, str]:
|
|
"""Decode a form-urlencoded request body into a flat ``{key: value}`` dict.
|
|
|
|
``parse_qs`` returns lists per key; the wxUser body has no repeated keys,
|
|
so we collapse each to its single value for ergonomic assertions.
|
|
``keep_blank_values=True`` so the always-empty ``userId`` / ``salesname``
|
|
filters (sent as ``userId=``) survive decoding and can be asserted on.
|
|
"""
|
|
body = request.content.decode()
|
|
parsed = parse_qs(body, keep_blank_values=True)
|
|
return {k: v[0] for k, v in parsed.items()}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# End-to-end: pagination + field mapping + raw_json
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_accounts_yields_records_and_paginates():
|
|
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
|
|
page1 = make_wxuser_list_response(
|
|
WXUSER_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page2 = make_wxuser_list_response(
|
|
WXUSER_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
page3 = make_wxuser_list_response(
|
|
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
|
|
)
|
|
|
|
route = respx.post(WXUSER_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
httpx.Response(200, json=page3),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_accounts(
|
|
client,
|
|
department_id=ROOT_DEPARTMENT_ID,
|
|
page_size=DEFAULT_PAGE_SIZE,
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
|
|
assert route.call_count == 3
|
|
|
|
# The required form filters ride on every request (the live endpoint
|
|
# answers code -90000 without them).
|
|
for call in route.calls:
|
|
body = _parse_form(call.request)
|
|
assert body["departmentId"] == ROOT_DEPARTMENT_ID
|
|
assert body["isOnJob"] == "1"
|
|
assert body["startdate"]
|
|
assert body["enddate"]
|
|
assert call.request.headers["content-type"] == FORM_CONTENT_TYPE
|
|
|
|
# Exactly the 4 fixture records (not duplicated, not dropped).
|
|
assert len(results) == 4
|
|
assert {r["wx_id"] for r in results} == {
|
|
rec["wechatId"] for rec in WXUSER_RECORDS
|
|
}
|
|
|
|
# Field mapping for each record (compare against the source record).
|
|
for result, record in zip(results, WXUSER_RECORDS, strict=True):
|
|
# Identity.
|
|
assert result["wx_id"] == record["wechatId"]
|
|
|
|
# Naming.
|
|
assert result["nick_name"] == record["nickName"]
|
|
assert result["alias"] == record["alias"]
|
|
|
|
# Phones.
|
|
assert result["phone"] == record["phone"]
|
|
|
|
# Embedded staff fields (the owning employee).
|
|
assert result["user_id"] == record["userId"]
|
|
assert result["user_name"] == record["userName"]
|
|
|
|
# Counts.
|
|
assert result["contacts_num"] == record["contactsNum"]
|
|
assert result["group_num"] == record["groupNum"]
|
|
assert result["send_num"] == record["sendNum"]
|
|
assert result["receive_num"] == record["receiveNum"]
|
|
|
|
# Status / sizing / timestamps.
|
|
assert result["is_on_job"] == record["isOnJob"]
|
|
assert result["db_size"] == record["dbSize"]
|
|
assert result["online_status"] == record["onlineStatus"]
|
|
assert result["last_update"] == record["lastUpdate"]
|
|
|
|
# Version carried verbatim; raw_json preserves the original record.
|
|
assert result["version"] == record["version"]
|
|
assert result["raw_json"] == record
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pagination body: pageNum/pageSize advanced correctly
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_accounts_sends_pagination_body():
|
|
"""Outgoing form body carries pageNum/pageSize + the required filters; the
|
|
page number advances per request."""
|
|
page1 = make_wxuser_list_response(
|
|
WXUSER_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
|
|
)
|
|
page2 = make_wxuser_list_response(
|
|
[], total=TOTAL, page_num=2, page_size=1
|
|
)
|
|
route = respx.post(WXUSER_LIST_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=page1),
|
|
httpx.Response(200, json=page2),
|
|
]
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_accounts(
|
|
client,
|
|
department_id=ROOT_DEPARTMENT_ID,
|
|
page_size=1,
|
|
start="2026-07-01",
|
|
end="2026-07-08",
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
# One record on page 1, empty page 2 stops the loop.
|
|
assert len(results) == 1
|
|
assert route.call_count == 2
|
|
|
|
# Every request is form-urlencoded (NOT JSON) and carries the required
|
|
# filters the live endpoint demands (isOnJob / departmentId / startdate /
|
|
# enddate), plus the always-empty free-text filters from the real client.
|
|
for call in route.calls:
|
|
assert call.request.headers["content-type"] == FORM_CONTENT_TYPE
|
|
body = _parse_form(call.request)
|
|
assert body["isOnJob"] == "1"
|
|
assert body["departmentId"] == ROOT_DEPARTMENT_ID
|
|
assert body["startdate"] == "2026-07-01"
|
|
assert body["enddate"] == "2026-07-08"
|
|
assert body["userId"] == ""
|
|
assert body["salesname"] == ""
|
|
assert body["weChatNickName"] == ""
|
|
|
|
# pageNum/pageSize advance correctly across the form body.
|
|
body1 = _parse_form(route.calls[0].request)
|
|
assert body1["pageNum"] == "1"
|
|
assert body1["pageSize"] == "1"
|
|
|
|
body2 = _parse_form(route.calls[1].request)
|
|
assert body2["pageNum"] == "2"
|
|
assert body2["pageSize"] == "1"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Empty first page -> no yields, single call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@respx.mock
|
|
async def test_scrape_wechat_accounts_stops_on_empty_first_page():
|
|
"""If the very first page is empty, scraper stops without over-fetching."""
|
|
empty = make_wxuser_list_response([], total=0, page_num=1)
|
|
route = respx.post(WXUSER_LIST_URL).mock(
|
|
return_value=httpx.Response(200, json=empty)
|
|
)
|
|
|
|
client = _primed_client()
|
|
try:
|
|
results = [
|
|
r
|
|
async for r in scrape_wechat_accounts(
|
|
client, department_id=ROOT_DEPARTMENT_ID
|
|
)
|
|
]
|
|
finally:
|
|
await client.aclose()
|
|
|
|
assert results == []
|
|
assert route.call_count == 1
|