feat: department + wechat account/staff scrapers
M2 organization scrapers for yunke-server: - scrape_departments: POST /usercenter/dept/tree. Single request (no pagination). Defensively unwraps the envelope (data is node list OR an object wrapping nodes under children), flattens BOTH nested and flat shapes, and reconstructs the materialized path by walking the pId chain (ancestor names joined with /). Maps id/pId/name -> dept_id/parent_id/name. - scrape_wechat_accounts: POST /wechatweb/wxUser/list. Page-number pagination via the existing paginate_by_page helper. Envelope differs from friend_list (no basicResult wrapper): data.data/data.total. Maps wechatId/nickName/alias/phone/userId/userName/contactsNum/groupNum/ sendNum/receiveNum/isOnJob/dbSize/onlineStatus/lastUpdate/version to WechatAccount columns; carries embedded staff pair for the repo layer. Repos: - upsert_department: idempotent upsert into Department (PK dept_id). - upsert_staff: primitive idempotent staff upsert (PK user_id, is_on_job coerced to bool). - upsert_wechat_account: now ALSO upserts the embedded Staff row via upsert_staff_from_account (best-effort; skips when account has no user_id). - upsert_staff_from_account: refactored to delegate to upsert_staff and forward phone/sale_phone/is_on_job when present. Tests (TDD red->green): test_scrape_departments (nested + flat + wrapped-data + empty), test_scrape_wechat_accounts (2-page pagination + body advance + empty first page), test_repos extended (dept idempotency/root parent, direct staff upsert + bool coercion, account-also-upserts-staff, re-upsert updates both, no-user_id skips staff). Cassettes: dept_tree_fixture (same 4 nodes in both nested + flat shapes), wxUser_fixture (4 capture-shaped accounts incl. departed + orphan). Suite: 97 -> 111 passing. ruff check clean. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -22,13 +22,15 @@ import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db.base import Base
|
||||
from app.db.models import Customer, Message, Staff, WechatAccount
|
||||
from app.db.models import Customer, Department, Message, Staff, WechatAccount
|
||||
from app.storage.repos import (
|
||||
_coerce_bool,
|
||||
_coerce_int,
|
||||
upsert_customer,
|
||||
upsert_customers,
|
||||
upsert_department,
|
||||
upsert_messages,
|
||||
upsert_staff,
|
||||
upsert_staff_from_account,
|
||||
upsert_wechat_account,
|
||||
)
|
||||
@@ -426,3 +428,276 @@ async def test_upsert_staff_from_account_skips_when_no_user_id():
|
||||
assert rows == []
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Department upsert (Task 2.0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_upsert_department_is_idempotent_and_updates_path():
|
||||
"""Same dept_id twice with different path -> 1 row, path refreshed."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec1 = {
|
||||
"dept_id": "1.1",
|
||||
"parent_id": "1",
|
||||
"name": "East",
|
||||
"path": "Sales/East",
|
||||
}
|
||||
rec2 = {
|
||||
"dept_id": "1.1",
|
||||
"parent_id": "1",
|
||||
"name": "East",
|
||||
# Re-scrape reconstructs the same path; verify it is overwritten
|
||||
# rather than producing a duplicate row.
|
||||
"path": "Sales/East",
|
||||
"raw_json": {"id": "1.1", "pId": "1", "name": "East"},
|
||||
}
|
||||
|
||||
await upsert_department(session, rec1)
|
||||
await session.commit()
|
||||
await upsert_department(session, rec2)
|
||||
await session.commit()
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Department)
|
||||
.where(Department.dept_id == "1.1")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].dept_id == "1.1"
|
||||
assert rows[0].parent_id == "1"
|
||||
assert rows[0].name == "East"
|
||||
assert rows[0].path == "Sales/East"
|
||||
# raw_json from the second upsert wins (last write).
|
||||
assert rows[0].raw_json == rec2["raw_json"]
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_department_preserves_none_parent_for_root():
|
||||
"""A root dept (parent_id None) is stored as-is; no spurious coercion."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"dept_id": "1",
|
||||
"parent_id": None,
|
||||
"name": "Sales",
|
||||
"path": "Sales",
|
||||
}
|
||||
await upsert_department(session, rec)
|
||||
await session.commit()
|
||||
|
||||
fetched = (
|
||||
await session.execute(
|
||||
sa.select(Department).where(Department.dept_id == "1")
|
||||
)
|
||||
).scalar_one()
|
||||
assert fetched.parent_id is None
|
||||
assert fetched.name == "Sales"
|
||||
assert fetched.path == "Sales"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# upsert_staff — direct staff-shaped dict upsert (Task 2.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_upsert_staff_is_idempotent_and_updates_name():
|
||||
"""Same user_id twice with different name -> 1 row, name refreshed."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec1 = {"user_id": "u_new", "name": "Old Name"}
|
||||
rec2 = {"user_id": "u_new", "name": "New Name"}
|
||||
|
||||
await upsert_staff(session, rec1)
|
||||
await session.commit()
|
||||
await upsert_staff(session, rec2)
|
||||
await session.commit()
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Staff)
|
||||
.where(Staff.user_id == "u_new")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].name == "New Name"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_staff_coerces_is_on_job():
|
||||
"""is_on_job '1' -> True; '0' -> False; None preserved."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
await upsert_staff(
|
||||
session, {"user_id": "u_b1", "is_on_job": "1"}
|
||||
)
|
||||
await upsert_staff(
|
||||
session, {"user_id": "u_b0", "is_on_job": "0"}
|
||||
)
|
||||
await upsert_staff(
|
||||
session, {"user_id": "u_bn", "is_on_job": None}
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
on = (
|
||||
await session.execute(
|
||||
sa.select(Staff).where(Staff.user_id == "u_b1")
|
||||
)
|
||||
).scalar_one()
|
||||
off = (
|
||||
await session.execute(
|
||||
sa.select(Staff).where(Staff.user_id == "u_b0")
|
||||
)
|
||||
).scalar_one()
|
||||
none_ = (
|
||||
await session.execute(
|
||||
sa.select(Staff).where(Staff.user_id == "u_bn")
|
||||
)
|
||||
).scalar_one()
|
||||
assert on.is_on_job is True
|
||||
assert off.is_on_job is False
|
||||
assert none_.is_on_job is None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# upsert_wechat_account now ALSO upserts the embedded Staff (Task 2.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_upsert_wechat_account_also_upserts_embedded_staff():
|
||||
"""Account upsert creates BOTH the WechatAccount row and the Staff row."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"wx_id": "wxid_dup",
|
||||
"nick_name": "Dup WeChat",
|
||||
"user_id": "u_dup",
|
||||
"user_name": "Dup Real Name",
|
||||
"is_on_job": 1,
|
||||
"phone": "13800000099",
|
||||
}
|
||||
await upsert_wechat_account(session, rec)
|
||||
await session.commit()
|
||||
|
||||
acct = (
|
||||
await session.execute(
|
||||
sa.select(WechatAccount).where(WechatAccount.wx_id == "wxid_dup")
|
||||
)
|
||||
).scalar_one()
|
||||
assert acct.user_id == "u_dup"
|
||||
assert acct.user_name == "Dup Real Name"
|
||||
|
||||
staff = (
|
||||
await session.execute(
|
||||
sa.select(Staff).where(Staff.user_id == "u_dup")
|
||||
)
|
||||
).scalar_one()
|
||||
assert staff.name == "Dup Real Name"
|
||||
assert staff.is_on_job is True
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_wechat_account_reupsert_updates_embedded_staff():
|
||||
"""Re-upserting the account refreshes BOTH account + embedded staff rows."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec1 = {
|
||||
"wx_id": "wxid_re",
|
||||
"nick_name": "Old Nick",
|
||||
"user_id": "u_re",
|
||||
"user_name": "Old Name",
|
||||
"is_on_job": 1,
|
||||
}
|
||||
rec2 = {
|
||||
"wx_id": "wxid_re",
|
||||
"nick_name": "New Nick",
|
||||
"user_id": "u_re",
|
||||
"user_name": "New Name",
|
||||
"is_on_job": 0, # staff departed since last scrape
|
||||
}
|
||||
await upsert_wechat_account(session, rec1)
|
||||
await session.commit()
|
||||
await upsert_wechat_account(session, rec2)
|
||||
await session.commit()
|
||||
|
||||
# One account row, refreshed.
|
||||
acct_rows = (
|
||||
await session.execute(
|
||||
sa.select(WechatAccount)
|
||||
.where(WechatAccount.wx_id == "wxid_re")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(acct_rows) == 1
|
||||
assert acct_rows[0].nick_name == "New Nick"
|
||||
|
||||
# One staff row, refreshed (not duplicated).
|
||||
staff_rows = (
|
||||
await session.execute(
|
||||
sa.select(Staff)
|
||||
.where(Staff.user_id == "u_re")
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(staff_rows) == 1
|
||||
assert staff_rows[0].name == "New Name"
|
||||
assert staff_rows[0].is_on_job is False
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_wechat_account_without_user_id_skips_staff():
|
||||
"""Account with no user_id is written; embedded-staff upsert is skipped."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"wx_id": "wxid_no_staff",
|
||||
"nick_name": "Orphan WeChat",
|
||||
# no user_id / user_name
|
||||
}
|
||||
await upsert_wechat_account(session, rec)
|
||||
await session.commit()
|
||||
|
||||
# Account row written.
|
||||
acct = (
|
||||
await session.execute(
|
||||
sa.select(WechatAccount).where(
|
||||
WechatAccount.wx_id == "wxid_no_staff"
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
assert acct.nick_name == "Orphan WeChat"
|
||||
|
||||
# No staff rows at all.
|
||||
staff_rows = (
|
||||
await session.execute(sa.select(Staff))
|
||||
).scalars().all()
|
||||
assert staff_rows == []
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
Reference in New Issue
Block a user