浅念
46e3880c0b
Centralize scraping stealth in YunkeClient.request() so automated traffic
resembles the normal CRM web app and is paced conservatively to avoid
triggering 云客 risk control (pulls M6 rate-limit work forward):
- app/config.py: add 6 tunable settings with conservative defaults
(request_min_interval=1.0, request_jitter=0.8, max_requests_per_run=0,
risk_backoff_{base=2.0, max=60.0, attempts=4}).
- app/scraper/base.py: RateLimiter gains optional jitter + a new wait()
method (fixed min+jitter per-call sleep); gap-based acquire() is unchanged
for backward compat (default jitter=0.0).
- app/session/yunke_client.py: every request now carries browser-identical
headers (UA/sec-ch-ua/accept/accept-language/referer) merged with the auth
headers (per-request overrides win); awaits the shared jittered limiter
before sending; enforces a per-run request cap (RequestCapReached); and
backs off exponentially on risk signals (HTTP 429 or 200 body carrying
频繁/风控/限制/稍后), distinct from the existing 401 relogin-once path.
- tests/conftest.py: zero the throttle env in the test suite so the
conservative prod defaults don't slow unit tests.
- tests/test_yunke_client_stealth.py: 9 new tests (browser headers,
override, limiter spy, 429 backoff, 200-marker backoff, persistent-429
raise, volume cap, unlimited, 401-relogin preserved). TDD red->green.
Suite: 171 -> 181 passing. ruff clean.
Co-Authored-By: Claude <noreply@anthropic.com>
88 lines
3.3 KiB
Python
88 lines
3.3 KiB
Python
"""Tests for app.config.get_settings — env-var binding and cache isolation."""
|
|
|
|
from pathlib import Path
|
|
|
|
from app.config import Settings, get_settings
|
|
|
|
|
|
def test_get_settings_reads_env_vars(monkeypatch):
|
|
"""get_settings() must reflect env vars set before it is called."""
|
|
# Clear any cached singleton from earlier tests / module imports.
|
|
get_settings.cache_clear()
|
|
|
|
monkeypatch.setenv("YUNKE_BASE_URL", "https://example.test/")
|
|
monkeypatch.setenv("YUNKE_USERNAME", "tester")
|
|
monkeypatch.setenv("YUNKE_PASSWORD", "shh")
|
|
monkeypatch.setenv(
|
|
"DB_URL",
|
|
"mysql+asyncmy://u:p@db.example:3306/yunke_test",
|
|
)
|
|
monkeypatch.setenv("REDIS_URL", "redis://redis.example:6379/2")
|
|
monkeypatch.setenv("MEDIA_DIR", "/tmp/yunke_media")
|
|
monkeypatch.setenv("ENCRYPTION_KEY", "abc123")
|
|
|
|
settings = get_settings()
|
|
|
|
assert settings.yunke_base_url == "https://example.test/"
|
|
assert settings.yunke_username == "tester"
|
|
assert settings.yunke_password == "shh"
|
|
assert settings.db_url == "mysql+asyncmy://u:p@db.example:3306/yunke_test"
|
|
assert settings.redis_url == "redis://redis.example:6379/2"
|
|
assert settings.media_dir == Path("/tmp/yunke_media")
|
|
assert settings.encryption_key == "abc123"
|
|
|
|
|
|
def test_get_settings_is_cached(monkeypatch):
|
|
"""get_settings() returns the same instance until the cache is cleared."""
|
|
get_settings.cache_clear()
|
|
|
|
first = get_settings()
|
|
second = get_settings()
|
|
assert first is second
|
|
|
|
# After clearing the cache, a new instance is built.
|
|
get_settings.cache_clear()
|
|
third = get_settings()
|
|
assert third is not first
|
|
|
|
|
|
def test_get_settings_picks_up_changes_after_cache_clear(monkeypatch):
|
|
"""After cache_clear(), the next call re-reads the environment."""
|
|
get_settings.cache_clear()
|
|
monkeypatch.setenv("DB_URL", "sqlite+aiosqlite:///:memory:")
|
|
initial = get_settings()
|
|
assert initial.db_url == "sqlite+aiosqlite:///:memory:"
|
|
|
|
monkeypatch.setenv("DB_URL", "mysql+asyncmy://other:3306/yunke")
|
|
# Cached value must NOT change just because env changed.
|
|
assert get_settings().db_url == "sqlite+aiosqlite:///:memory:"
|
|
|
|
# After clearing, the new env var is picked up.
|
|
get_settings.cache_clear()
|
|
assert get_settings().db_url == "mysql+asyncmy://other:3306/yunke"
|
|
|
|
|
|
def test_settings_constructable_without_env():
|
|
"""Settings() can be built with defaults (no env required)."""
|
|
s = Settings()
|
|
assert isinstance(s.db_url, str)
|
|
assert s.encryption_key == ""
|
|
|
|
|
|
def test_stealth_defaults_are_conservative():
|
|
"""The anti-risk-control settings declare conservative defaults.
|
|
|
|
Asserted via Pydantic field-introspection rather than ``Settings()`` so it
|
|
is immune to the test-env throttle-zeroing in conftest (which intentionally
|
|
pins ``REQUEST_MIN_INTERVAL``/``REQUEST_JITTER`` to 0 for suite speed).
|
|
Production deployments that don't set these env vars get the slow,
|
|
jittered defaults declared here.
|
|
"""
|
|
f = Settings.model_fields
|
|
assert f["request_min_interval"].default == 1.0
|
|
assert f["request_jitter"].default == 0.8
|
|
assert f["max_requests_per_run"].default == 0
|
|
assert f["risk_backoff_base"].default == 2.0
|
|
assert f["risk_backoff_max"].default == 60.0
|
|
assert f["risk_backoff_attempts"].default == 4
|