diff --git a/yunke-server/app/session/__init__.py b/yunke-server/app/session/__init__.py new file mode 100644 index 0000000..b25114e --- /dev/null +++ b/yunke-server/app/session/__init__.py @@ -0,0 +1,29 @@ +"""会话层 (session layer) for the yunke CRM API. + +Public surface: + +* :class:`app.session.login.LoginResult` — parsed login outcome (userid, + company, cookies snapshot, raw payload). +* :func:`app.session.login.get_secure_key` — ``GET /newuc/login/getSecureKey``. +* :func:`app.session.login.pc_login` — ``POST /usercenter/login/pcLogin``. +* :func:`app.session.login.encrypt_password` — Phase-0 swappable password + encryption (see file for the open TODO). +* :class:`app.session.yunke_client.YunkeClient` — the authenticated HTTP + client every scraper uses. +""" + +from app.session.login import ( + LoginResult, + encrypt_password, + get_secure_key, + pc_login, +) +from app.session.yunke_client import YunkeClient + +__all__ = [ + "LoginResult", + "YunkeClient", + "encrypt_password", + "get_secure_key", + "pc_login", +] diff --git a/yunke-server/app/session/login.py b/yunke-server/app/session/login.py new file mode 100644 index 0000000..92daf7e --- /dev/null +++ b/yunke-server/app/session/login.py @@ -0,0 +1,167 @@ +"""yunke CRM login flow: ``getSecureKey`` -> ``pcLogin``. + +Real-API facts (from inspection — see design doc §6.1) +------------------------------------------------------ +* ``GET /newuc/login/getSecureKey`` kicks off the login chain. +* ``POST /usercenter/login/pcLogin`` exchanges credentials for a session. +* After login every data request to ``crm2.yunkecn.com`` carries the headers + ``userid`` / ``company`` / ``source=yunkepc`` / ``channel=4`` / + ``content-type=application/json`` plus the login session cookie (handled + by the httpx client's cookie jar, not by this module). + +Phase-0 unknowns (clearly marked) +--------------------------------- +The exact ``pcLogin`` request body, the password encryption scheme (almost +certainly RSA using the modulus/exponent from ``getSecureKey``), and the +precise response JSON shape were NOT captured. To keep the session layer +end-to-end runnable today: + +* :func:`encrypt_password` is a deterministic, documented placeholder. The + real RSA implementation drops in here later without touching call-sites. +* :func:`pc_login` parses ``userid`` / ``company`` out of a ``data`` object + with several reasonable key-name variants. When the real shape is captured + only this parser needs to change. + +Tests mock both endpoints, so the placeholders are exercised end-to-end. +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass, field +from typing import Any + +import httpx + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclass +class LoginResult: + """Outcome of a successful yunke login. + + Fields: + userid: 32-hex user id; sent as the ``userid`` header on every + subsequent request. + company: company code (e.g. ``"8m0atd"``); sent as the ``company`` + header. + cookies: Snapshot of the session cookies set by ``pcLogin``. The + live cookies live on the httpx client's cookie jar; this + snapshot is kept so the session can later be persisted + (encrypted) into ``YkAccount.session_blob_enc``. + raw: The full ``pcLogin`` JSON body, kept for diagnostics and + for future field additions without re-rolling the parser. + + Phase-0 assumption: ``userid`` and ``company`` come from the response + body's ``data`` object. Key-name variants tried in :func:`pc_login`: + ``userid`` / ``userId`` / ``user_id`` and ``company`` / ``companyCode`` / + ``company_code``. + """ + + userid: str + company: str + cookies: dict[str, str] = field(default_factory=dict) + raw: dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Password encryption — Phase-0 placeholder +# --------------------------------------------------------------------------- + + +def encrypt_password(plain: str, secure_key: dict[str, Any]) -> str: + """Encrypt ``plain`` using the key material in ``secure_key``. + + Phase-0 TODO: real RSA encryption per ``getSecureKey``. + + The browser-side crypto was not captured. Based on the endpoint name + (``getSecureKey``) and the typical Chinese-CRM login idiom, the real + implementation almost certainly RSA-encrypts the password using a + modulus/exponent (or PEM public key) returned in ``secure_key`` and + then base64-encodes the ciphertext. Until that is confirmed by a real + capture, this function returns base64(UTF-8(plain)) so the login flow + is end-to-end runnable and deterministic for tests. + + Drop-in contract for the real implementation: + def encrypt_password(plain: str, secure_key: dict[str, Any]) -> str: + # 1. Pull modulus/exponent (or PEM) out of secure_key. + # 2. RSA-OAEP (or PKCS#1 v1.5) encrypt UTF-8(plain). + # 3. Return base64(ciphertext)." + """ + # Phase 0 TODO: real RSA encryption per getSecureKey. + return base64.b64encode(plain.encode("utf-8")).decode("ascii") + + +# --------------------------------------------------------------------------- +# Endpoint wrappers +# --------------------------------------------------------------------------- + + +async def get_secure_key(client: httpx.AsyncClient) -> dict[str, Any]: + """Call ``GET {base}/newuc/login/getSecureKey`` and return the JSON body. + + The exact shape of the secure-key payload is a Phase-0 unknown; this + function returns the whole decoded JSON so :func:`encrypt_password` (and + eventually the real RSA step) can pick whatever fields it needs. + """ + resp = await client.get("/newuc/login/getSecureKey") + resp.raise_for_status() + return resp.json() + + +async def pc_login( + client: httpx.AsyncClient, + username: str, + password: str, + secure_key: dict[str, Any], +) -> LoginResult: + """Call ``POST {base}/usercenter/login/pcLogin`` and parse a LoginResult. + + Phase-0 assumptions about the request body (not captured): + ``{"account": , "password": }``. + Real key names (``account`` vs ``loginName`` vs ``username``) and any + additional required fields (captcha, deviceId, ...) are to be confirmed; + only this function needs to change when they are. + + Phase-0 assumptions about the response (not captured): JSON body with a + ``data`` object containing ``userid`` + ``company``. Several reasonable + key-name variants are tried (see :func:`_pick`). The session cookie is + read back from the httpx client's cookie jar — ``pcLogin``'s + ``Set-Cookie`` lands there automatically. + """ + body = { + "account": username, + "password": encrypt_password(password, secure_key), + } + resp = await client.post("/usercenter/login/pcLogin", json=body) + resp.raise_for_status() + payload = resp.json() + + data = payload.get("data") if isinstance(payload, dict) else None + data = data if isinstance(data, dict) else {} + + userid = _pick(data, "userid", "userId", "user_id") or _pick( + payload, "userid", "userId", "user_id" + ) + company = _pick(data, "company", "companyCode", "company_code") or _pick( + payload, "company", "companyCode", "company_code" + ) + + cookies = {c.name: c.value for c in client.cookies.jar} + return LoginResult( + userid=userid or "", + company=company or "", + cookies=cookies, + raw=payload if isinstance(payload, dict) else {"data": payload}, + ) + + +def _pick(d: dict[str, Any], *keys: str) -> str | None: + """Return the first present, truthy value among ``keys`` in ``d``.""" + for k in keys: + v = d.get(k) + if v: + return str(v) + return None diff --git a/yunke-server/app/session/yunke_client.py b/yunke-server/app/session/yunke_client.py new file mode 100644 index 0000000..6c3f1ec --- /dev/null +++ b/yunke-server/app/session/yunke_client.py @@ -0,0 +1,208 @@ +"""Authenticated HTTP client for the yunke CRM API. + +:class:`YunkeClient` is the single chokepoint every scraper uses to call the +CRM (see design doc §6.1 会话层). It wraps an :class:`httpx.AsyncClient` and: + +* Auto-logs-in (``getSecureKey`` -> ``pcLogin``) on the first request when no + session is present, or when constructed with credentials. +* Injects the 5 known auth headers (``userid`` / ``company`` / ``source`` / + ``channel`` / ``content-type``) on every request. +* Detects HTTP 401 (or a body-level session-expired marker), re-logs-in + *once*, and retries. A persistent 401 raises after the single retry — no + infinite loop. + +Phase-0 note: the session is held in-memory only. Future work will persist +the encrypted session blob to ``YkAccount.session_blob_enc`` (see +``app.db.models.YkAccount``) so scraper processes can resume an existing +session instead of always logging in fresh. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import httpx + +from app.session.login import ( + LoginResult, + get_secure_key, + pc_login, +) + +# The 4 constant auth headers. ``userid`` + ``company`` come from the +# LoginResult and are merged in separately. +_STATIC_AUTH_HEADERS: dict[str, str] = { + "source": "yunkepc", + "channel": "4", + "content-type": "application/json", +} + + +class YunkeClient: + """Authenticated async HTTP client for the yunke CRM API.""" + + def __init__( + self, + base_url: str, + *, + username: str | None = None, + password: str | None = None, + userid: str | None = None, + company: str | None = None, + cookies: Mapping[str, str] | None = None, + client: httpx.AsyncClient | None = None, + ) -> None: + """Construct a client either from credentials or an existing session. + + Credentials mode (auto-login capable): + Pass ``username`` + ``password``; ``login()`` works and 401s will + trigger automatic re-login. + + Existing-session mode (no re-login possible): + Pass ``userid`` + ``company`` (optionally ``cookies``). ``login()`` + raises if called — the caller has elected to skip credentials, so + nothing can re-login on a 401. + + ``client`` may be supplied to inject a pre-configured + :class:`httpx.AsyncClient` (e.g. for custom transports). When omitted + a new client bound to ``base_url`` is created and owned by this + instance (closed by :meth:`aclose`). + """ + has_creds = username is not None and password is not None + has_session = userid is not None and company is not None + if not has_creds and not has_session: + raise ValueError( + "YunkeClient requires either (username + password) for" + " auto-login or (userid + company) for an existing session." + ) + + self.base_url = base_url + self.username = username + self.password = password + self.login_result: LoginResult | None = None + + self._owns_client = client is None + self._client = client or httpx.AsyncClient(base_url=base_url) + + if has_session: + assert userid is not None and company is not None # for mypy + self.login_result = LoginResult( + userid=userid, company=company, cookies=dict(cookies or {}) + ) + for name, value in (cookies or {}).items(): + self._client.cookies.set(name, value) + + # -- properties --------------------------------------------------------- + + @property + def client(self) -> httpx.AsyncClient: + """The underlying httpx client (exposed for cookie inspection/tests).""" + return self._client + + @property + def logged_in(self) -> bool: + return self.login_result is not None + + # -- login -------------------------------------------------------------- + + async def login(self) -> LoginResult: + """Run ``getSecureKey`` -> ``pcLogin`` and store the LoginResult. + + Raises ``RuntimeError`` if this client was constructed without + credentials (existing-session mode) — there is nothing to log in + with. + """ + if self.username is None or self.password is None: + raise RuntimeError( + "YunkeClient.login() requires username + password; this" + " client was constructed from an existing session." + ) + secure_key = await get_secure_key(self._client) + result = await pc_login( + self._client, self.username, self.password, secure_key + ) + self.login_result = result + return result + + # -- authed request ----------------------------------------------------- + + async def request( + self, + method: str, + path: str, + *, + headers: Mapping[str, str] | None = None, + _retried: bool = False, + **kwargs: Any, + ) -> httpx.Response: + """Send an authed request, auto-logging-in and re-logging-in on 401. + + Per-request ``headers`` override the auth defaults for this call + only. ``_retried`` is an internal guard against infinite re-login + loops and must not be set by callers. + """ + if self.login_result is None: + await self.login() + + user_headers = dict(headers or {}) + merged = {**self._auth_headers(), **user_headers} + + resp = await self._client.request(method, path, headers=merged, **kwargs) + + if _is_session_expired(resp): + if _retried: + # Already retried once after a re-login — give up so a + # persistently dead session cannot loop forever. + raise httpx.HTTPStatusError( + "yunke session expired after re-login retry", + request=resp.request, + response=resp, + ) + await self.login() + return await self.request( + method, path, headers=user_headers, _retried=True, **kwargs + ) + return resp + + async def get(self, path: str, **kwargs: Any) -> httpx.Response: + return await self.request("GET", path, **kwargs) + + async def post(self, path: str, **kwargs: Any) -> httpx.Response: + return await self.request("POST", path, **kwargs) + + # -- teardown ----------------------------------------------------------- + + async def aclose(self) -> None: + """Close the underlying httpx client if this instance owns it.""" + if self._owns_client: + await self._client.aclose() + + # -- internals ---------------------------------------------------------- + + def _auth_headers(self) -> dict[str, str]: + if self.login_result is None: + raise RuntimeError("Not logged in") # pragma: no cover + return { + "userid": self.login_result.userid, + "company": self.login_result.company, + **_STATIC_AUTH_HEADERS, + } + + +def _is_session_expired(resp: httpx.Response) -> bool: + """Heuristic: does this response indicate the session is dead? + + * HTTP 401 -> yes. + * HTTP 200 with a body-level ``code`` of ``401`` (a common Chinese-CRM + idiom) -> yes. We deliberately stay narrow here — only the explicit + auth-failed code, not generic non-zero codes, to avoid re-login storms + on ordinary business errors. + """ + if resp.status_code == 401: + return True + try: + body = resp.json() + except (ValueError, httpx.DecodingError): + return False + return isinstance(body, dict) and body.get("code") in (401, "401") diff --git a/yunke-server/tests/test_yunke_client.py b/yunke-server/tests/test_yunke_client.py new file mode 100644 index 0000000..4f036e6 --- /dev/null +++ b/yunke-server/tests/test_yunke_client.py @@ -0,0 +1,208 @@ +"""Tests for the yunke session layer: ``app.session.login`` + ``YunkeClient``. + +These tests use ``respx`` to mock the upstream httpx transport so we can +assert exactly what gets sent (headers, body) and drive the auto-login + +401-relogin behaviour without touching the real CRM. + +Phase-0 caveat: the real ``pcLogin`` request body / password encryption / +response shape were not captured. The mocks here encode our documented +assumptions (see ``app/session/login.py``): response JSON has a ``data`` +object with ``userid`` + ``company`` fields. When the real shapes are +confirmed, only the parsing inside ``pc_login`` needs to change — these +tests stay green. +""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from app.session.login import LoginResult +from app.session.yunke_client import YunkeClient + +BASE = "https://crm2.yunkecn.com" +USERID = "a" * 32 # 32-hex user id, as observed in real auth headers +COMPANY = "8m0atd" + + +@pytest.fixture() +async def client(): + """A YunkeClient wired for username/password login. + + The fixture yields the client and always closes it so respx mocks set up + inside individual tests don't leak cookies across tests. + """ + c = YunkeClient(base_url=BASE, username="tester", password="hunter2") + try: + yield c + finally: + await c.aclose() + + +def _stub_login_endpoints() -> tuple[respx.Route, respx.Route]: + """Register the getSecureKey + pcLogin mocks used across tests.""" + secure = respx.get(f"{BASE}/newuc/login/getSecureKey").mock( + return_value=httpx.Response( + 200, json={"data": {"modulus": "00ab", "exponent": "010001"}} + ) + ) + login = respx.post(f"{BASE}/usercenter/login/pcLogin").mock( + return_value=httpx.Response( + 200, + json={"code": 0, "data": {"userid": USERID, "company": COMPANY}}, + headers={"Set-Cookie": "JSESSIONID=deadbeef; Path=/"}, + ) + ) + return secure, login + + +# --------------------------------------------------------------------------- +# Test A: login() populates LoginResult + the httpx client cookie jar +# --------------------------------------------------------------------------- + + +@respx.mock +async def test_login_stores_userid_and_cookies(client: YunkeClient): + _, login_route = _stub_login_endpoints() + + result = await client.login() + + assert login_route.called + assert isinstance(result, LoginResult) + assert result.userid == USERID + assert result.company == COMPANY + # LoginResult is the same object stored on the client. + assert client.login_result is result + # The session cookie from the mocked Set-Cookie must land in the httpx + # client's cookie jar so subsequent requests carry it. + assert client.client.cookies.get("JSESSIONID") == "deadbeef" + # The snapshot on LoginResult should also contain it. + assert result.cookies.get("JSESSIONID") == "deadbeef" + + +# --------------------------------------------------------------------------- +# Test B: every request carries the auth headers +# --------------------------------------------------------------------------- + + +@respx.mock +async def test_post_injects_auth_headers(client: YunkeClient): + # Skip the login flow entirely — prime the client with a logged-in state. + client.login_result = LoginResult(userid=USERID, company=COMPANY) + client.client.cookies.set("JSESSIONID", "deadbeef") + + route = respx.post(f"{BASE}/wechatweb/wxFriends/friend_list/es").mock( + return_value=httpx.Response(200, json={"data": []}) + ) + + resp = await client.post( + "/wechatweb/wxFriends/friend_list/es", json={"pageNum": 1, "pageSize": 20} + ) + + assert resp.status_code == 200 + assert route.called + + sent = route.calls.last.request + # The 5 known auth headers must be on every outgoing request. + assert sent.headers["userid"] == USERID + assert sent.headers["company"] == COMPANY + assert sent.headers["source"] == "yunkepc" + assert sent.headers["channel"] == "4" + assert sent.headers["content-type"] == "application/json" + # And the session cookie travels along too. + assert sent.headers.get("cookie", "").find("JSESSIONID=deadbeef") != -1 + + +# --------------------------------------------------------------------------- +# Test C: 401 -> re-login once -> retry succeeds +# --------------------------------------------------------------------------- + + +@respx.mock +async def test_401_triggers_relogin_and_retry(client: YunkeClient): + _, login_route = _stub_login_endpoints() + data_route = respx.post( + f"{BASE}/wechatweb/wxFriends/friend_list/es" + ).mock( + side_effect=[ + httpx.Response(401, json={"msg": "session expired"}), + httpx.Response(200, json={"data": [{"id": 1}]}), + ] + ) + + # Prime as logged-in so the first call doesn't itself trigger an + # auto-login; the 401 is what should drive exactly one re-login. + client.login_result = LoginResult(userid=USERID, company=COMPANY) + + resp = await client.post("/wechatweb/wxFriends/friend_list/es", json={}) + + assert resp.status_code == 200 + assert resp.json() == {"data": [{"id": 1}]} + # Exactly one re-login + exactly one retry — no infinite loop. + assert login_route.call_count == 1 + assert data_route.call_count == 2 + + +# --------------------------------------------------------------------------- +# Test D: persistent 401 -> raise after the single retry (no loop) +# --------------------------------------------------------------------------- + + +@respx.mock +async def test_persistent_401_raises_after_one_retry(client: YunkeClient): + _, login_route = _stub_login_endpoints() + data_route = respx.post( + f"{BASE}/wechatweb/wxFriends/friend_list/es" + ).mock(return_value=httpx.Response(401, json={"msg": "still expired"})) + + client.login_result = LoginResult(userid=USERID, company=COMPANY) + + with pytest.raises(httpx.HTTPStatusError): + await client.post("/wechatweb/wxFriends/friend_list/es", json={}) + + # The single retry happened, then we gave up and raised. + assert login_route.call_count == 1 + assert data_route.call_count == 2 + + +# --------------------------------------------------------------------------- +# Test E: auto-login on first request when no session yet +# --------------------------------------------------------------------------- + + +@respx.mock +async def test_first_request_auto_logs_in(client: YunkeClient): + secure_route, login_route = _stub_login_endpoints() + data_route = respx.post(f"{BASE}/wechatweb/wxFriends/friend_list/es").mock( + return_value=httpx.Response(200, json={"data": []}) + ) + + # No priming — client should auto-login on first call. + resp = await client.post("/wechatweb/wxFriends/friend_list/es", json={}) + + assert resp.status_code == 200 + assert secure_route.called + assert login_route.called + assert data_route.call_count == 1 + assert client.login_result is not None + assert client.login_result.userid == USERID + + +# --------------------------------------------------------------------------- +# Test F: encrypt_password is a documented Phase-0 placeholder +# --------------------------------------------------------------------------- + + +def test_encrypt_password_is_swappable_phase0_placeholder(): + """Phase 0: encrypt_password must be deterministic and non-trivial but is + NOT the real RSA scheme. It must accept any secure-key shape and return a + str so the login flow is end-to-end runnable in tests. + """ + from app.session.login import encrypt_password + + out = encrypt_password("hunter2", {"modulus": "00ab", "exponent": "010001"}) + assert isinstance(out, str) + assert out # non-empty + # Deterministic for the same inputs (so tests / debugging are stable). + assert encrypt_password("hunter2", {"modulus": "00ab"}) == out