"""Tests for the anti-risk-control / stealth layer in :class:`YunkeClient`. These exercise the four stealth concerns that ``request()`` now centralizes: * **Browser-identical headers** on every request (UA / sec-ch-ua / accept / accept-language / referer) merged with the existing auth headers, with per-request overrides winning. * A shared jittered :class:`RateLimiter` awaited ahead of each upstream request. * **Exponential backoff + retry** on risk-control signals (HTTP 429, or a 200 body carrying a 风控/限流 marker such as 频繁/风控/限制/稍后) — distinct from 401 session-expiry, which keeps its existing relogin-once behaviour. * A hard per-run request cap that raises :class:`RequestCapReached`. Every test injects explicit stealth params (no-op throttle, small backoff) so the suite stays fast and deterministic regardless of the process-wide :class:`Settings` defaults (which are deliberately slow in production). """ from __future__ import annotations from unittest.mock import AsyncMock import httpx import pytest import respx from cryptography.hazmat.primitives.asymmetric import rsa from app.scraper.base import RateLimiter from app.session.yunke_client import RequestCapReached, YunkeClient BASE = "https://crm2.yunkecn.com" USERID = "a" * 32 COMPANY = "8m0atd" DATA_PATH = "/wechatweb/wxFriends/friend_list/es" DATA_URL = f"{BASE}{DATA_PATH}" # The exact browser-captured header values the client must reproduce. UA = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36" ) SEC_CH_UA = '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"' # Backoff schedule uses base=5.0 so expected delays are 5.0, 10.0, 20.0, ... BACKOFF_BASE = 5.0 # A real RSA keypair backing the mocked ``getSecureKey`` endpoint so the real # RSA step in ``pc_login`` has a valid key to encrypt against (mirrors the # pattern in tests/test_yunke_client.py). Generated once at import. _TEST_PRIVATE_KEY = rsa.generate_private_key(public_exponent=65537, key_size=2048) def _secure_key_from_test_public_key() -> dict[str, str]: nums = _TEST_PRIVATE_KEY.public_key().public_numbers() return { "public_exponent": format(nums.e, "x"), "modulus": format(nums.n, "x"), } _TEST_SECURE_KEY = _secure_key_from_test_public_key() def _patch_backoff_sleep(monkeypatch, sink: list[float]) -> None: """Replace ``asyncio.sleep`` in yunke_client with an instant recorder.""" async def fake_sleep(delay: float) -> None: sink.append(delay) monkeypatch.setattr("app.session.yunke_client.asyncio.sleep", fake_sleep) @pytest.fixture() async def client(): """A logged-in (session-mode) client with a no-op throttle. Session mode (userid + company) skips the login flow entirely. Tests that need relogin construct their own credentials-mode client inline. """ c = YunkeClient( base_url=BASE, userid=USERID, company=COMPANY, rate_limiter=RateLimiter(min_interval=0.0, jitter=0.0), ) try: yield c finally: await c.aclose() # --------------------------------------------------------------------------- # 1. Browser-identical headers on every request # --------------------------------------------------------------------------- @respx.mock async def test_browser_headers_present_on_every_request(client: YunkeClient): route = respx.post(DATA_URL).mock( return_value=httpx.Response(200, json={"ok": True}) ) await client.post(DATA_PATH, json={"pageNum": 1}) sent = route.calls.last.request.headers assert sent["user-agent"] == UA assert sent["accept"] == "application/json, text/plain, */*" assert sent["accept-language"] == "zh-CN" assert sent["sec-ch-ua"] == SEC_CH_UA assert sent["sec-ch-ua-mobile"] == "?0" assert sent["sec-ch-ua-platform"] == '"Windows"' assert sent["referer"] == "https://crm2.yunkecn.com/" # The existing auth headers still ride along unchanged. assert "source" not in sent # omitted: breaks msg/chats Integer binding assert sent["channel"] == "4" assert sent["userid"] == USERID assert sent["company"] == COMPANY assert sent["content-type"] == "application/json" @respx.mock async def test_per_request_header_overrides_browser_defaults(client: YunkeClient): """A caller-supplied header wins over the browser + auth defaults.""" route = respx.post(DATA_URL).mock( return_value=httpx.Response(200, json={"ok": True}) ) await client.post( DATA_PATH, json={}, headers={"referer": "https://crm2.yunkecn.com/cms/x"} ) assert route.calls.last.request.headers["referer"] == ( "https://crm2.yunkecn.com/cms/x" ) # --------------------------------------------------------------------------- # 2. Rate limiter awaited once per request # --------------------------------------------------------------------------- @respx.mock async def test_rate_limiter_wait_awaited_once_per_request(client: YunkeClient): respx.post(DATA_URL).mock(return_value=httpx.Response(200, json={"ok": True})) spy = AsyncMock() client._rate_limiter.wait = spy # type: ignore[method-assign] await client.post(DATA_PATH, json={}) assert spy.await_count == 1 # --------------------------------------------------------------------------- # 3. Risk-control signals -> exponential backoff + retry # --------------------------------------------------------------------------- @respx.mock async def test_429_triggers_backoff_retry_then_success( monkeypatch, client: YunkeClient ): delays: list[float] = [] _patch_backoff_sleep(monkeypatch, delays) route = respx.post(DATA_URL).mock( side_effect=[ httpx.Response(429, json={"msg": "请求过于频繁"}), httpx.Response(429, json={"msg": "请求过于频繁"}), httpx.Response(200, json={"data": [{"id": 1}]}), ] ) resp = await client.post(DATA_PATH, json={}) assert resp.status_code == 200 assert resp.json() == {"data": [{"id": 1}]} # Initial try + two retries after backoff. assert route.call_count == 3 # base * 2**0 = 5.0, base * 2**1 = 10.0 assert delays == [5.0, 10.0] @respx.mock async def test_200_with_risk_marker_triggers_backoff(monkeypatch, client: YunkeClient): """A 200 body carrying a 风控/限流 marker must also trigger backoff — many Chinese CRMs signal throttling inline rather than via a real 429.""" _patch_backoff_sleep(monkeypatch, []) route = respx.post(DATA_URL).mock( side_effect=[ httpx.Response(200, json={"code": 500, "msg": "操作过于频繁"}), httpx.Response(200, json={"data": []}), ] ) resp = await client.post(DATA_PATH, json={}) assert resp.status_code == 200 assert resp.json() == {"data": []} assert route.call_count == 2 @respx.mock async def test_persistent_429_raises_after_backoff_attempts(monkeypatch): """When the upstream keeps signaling risk past ``risk_backoff_attempts``, the client gives up and raises an :class:`httpx.HTTPStatusError`.""" delays: list[float] = [] _patch_backoff_sleep(monkeypatch, delays) c = YunkeClient( base_url=BASE, userid=USERID, company=COMPANY, rate_limiter=RateLimiter(min_interval=0.0, jitter=0.0), backoff_attempts=2, backoff_base=BACKOFF_BASE, ) try: respx.post(DATA_URL).mock( return_value=httpx.Response(429, json={"msg": "请求过于频繁"}) ) with pytest.raises(httpx.HTTPStatusError): await c.post(DATA_PATH, json={}) # 1 initial attempt + 2 retries = 3 upstream calls, 2 backoff sleeps. assert delays == [5.0, 10.0] finally: await c.aclose() # --------------------------------------------------------------------------- # 4. Volume cap -> RequestCapReached # --------------------------------------------------------------------------- @respx.mock async def test_volume_cap_raises_on_nth_request(): respx.post(DATA_URL).mock(return_value=httpx.Response(200, json={"ok": True})) capped = YunkeClient( base_url=BASE, userid=USERID, company=COMPANY, rate_limiter=RateLimiter(min_interval=0.0, jitter=0.0), max_requests_per_run=2, ) try: await capped.post(DATA_PATH, json={}) # 1 -> ok await capped.post(DATA_PATH, json={}) # 2 -> ok with pytest.raises(RequestCapReached): await capped.post(DATA_PATH, json={}) # 3 -> cap hit finally: await capped.aclose() @respx.mock async def test_volume_cap_zero_means_unlimited(client: YunkeClient): """``max_requests_per_run=0`` (the default) disables the cap entirely.""" respx.post(DATA_URL).mock(return_value=httpx.Response(200, json={"ok": True})) for _ in range(5): await client.post(DATA_PATH, json={}) # --------------------------------------------------------------------------- # 5. 401 session-expiry still relogs exactly once (existing behaviour preserved) # --------------------------------------------------------------------------- @respx.mock async def test_401_still_relogs_once_with_stealth_active(): """Stealth must not break the existing 401 -> relogin-once -> retry path. A 401 is a session-death (handled by relogin), NOT a risk-control signal, so it must not enter the backoff loop. """ secure = respx.get(f"{BASE}/newuc/login/getSecureKey").mock( return_value=httpx.Response(200, json={"data": _TEST_SECURE_KEY}) ) login = respx.post(f"{BASE}/usercenter/login/pcLogin").mock( return_value=httpx.Response( 200, json={"code": 0, "data": {"userid": USERID, "company": COMPANY}}, headers={"Set-Cookie": "JSESSIONID=deadbeef; Path=/"}, ) ) data = respx.post(DATA_URL).mock( side_effect=[ httpx.Response(401, json={"msg": "session expired"}), httpx.Response(200, json={"data": [{"id": 1}]}), ] ) c = YunkeClient( base_url=BASE, username="tester", password="hunter2", rate_limiter=RateLimiter(min_interval=0.0, jitter=0.0), ) # Prime logged-in so only the 401-driven relogin is counted (mirrors the # pattern in test_yunke_client.py). from app.session.login import LoginResult c.login_result = LoginResult(userid=USERID, company=COMPANY) try: resp = await c.post(DATA_PATH, json={}) assert resp.status_code == 200 assert resp.json() == {"data": [{"id": 1}]} assert secure.called assert login.call_count == 1 assert data.call_count == 2 finally: await c.aclose()