"""原始上报批次（raw_batch）：脱敏快照、拒批截断、重放留痕、API 分页、30 天清理。 设计契约（与用户确认）： - 敏感字段值替换为 [FILTERED]（结构保留）——sanitize_payload 复用 SENSITIVE_KEY_PATTERNS； - 被拒批次也存（body 截断 4KB）；invalid_json（解析失败）不存； - 原始表写入失败不得影响主采集流程（best-effort）； - 每次接收都记一行（"原始记录"语义，重放不合并）。 """ import json from datetime import timedelta from io import StringIO import pytest from django.contrib.auth import get_user_model from django.core.cache import cache from django.core.management import call_command from django.utils import timezone from rest_framework.test import APIClient from tracker.models import RawBatch, Site, Visitor User = get_user_model() NOW = timezone.now() @pytest.fixture(autouse=True) def _clear_cache(): cache.clear() # locmem 限流历史跨测试残留，会让压测 429（与 test_collect 同款） @pytest.fixture def client(): # 覆盖 pytest-django 的 Django Client——认证用例需要 credentials() return APIClient() @pytest.fixture def member_user(): return User.objects.create_user(username="member", password="memberpass123") def _auth(client, user): from rest_framework_simplejwt.tokens import RefreshToken token = RefreshToken.for_user(user) client.credentials(HTTP_AUTHORIZATION=f"Bearer {token.access_token}") def _post(client, body, origin="https://www.a.com"): return client.post("/api/collect/batch", data=json.dumps(body), content_type="text/plain", HTTP_ORIGIN=origin) def _body(n=1, sid="s1", vid="v1", events=None): return {"site_id": "site_x", "visitor_id": vid, "session_id": sid, "context": {"referrer_domain": "", "path": "/", "device_type": "PC"}, "events": events if events is not None else [{"event_id": f"e{i}", "event_name": "page_view", "occurred_at": NOW.isoformat(), "path": "/", "title": "t", "properties": {}} for i in range(n)]} def _site(): return Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x") # ---- a) 正常上报 ---- def test_raw_batch_accepted(db, client): _site() r = _post(client, _body(2, vid="v9")) assert r.status_code == 200 rb = RawBatch.objects.get() assert rb.status == "accepted" and rb.event_count == 2 and rb.rejected_count == 0 vis = Visitor.objects.get(visitor_id="v9") assert rb.visitor_id == vis.pk and rb.site_id == vis.site_id # 访客关联 body = json.loads(rb.body) assert body["visitor_id"] == "v9" # 完整结构在场 assert body["context"]["device_type"] == "PC" assert [e["path"] for e in body["events"]] == ["/", "/"] # 快照先于 ingest 就地清洗 # ---- b) 敏感键脱敏 + partial ---- def test_raw_batch_partial_and_filtered(db, client): _site() events = [ {"event_id": "ok1", "event_name": "page_view", "occurred_at": NOW.isoformat(), "path": "/", "title": "t", "properties": {}}, {"event_id": "bad1", "event_name": "page_view", "occurred_at": NOW.isoformat(), "path": "/login", "title": "t", "properties": {"user_password": "hunter2"}}, ] r = _post(client, _body(events=events, vid="v1")) assert r.json()["data"]["accepted"] == 1 # 事件层：敏感事件被拒 rb = RawBatch.objects.get() assert rb.status == "partial" and rb.event_count == 1 and rb.rejected_count == 1 body = json.loads(rb.body) assert body["events"][1]["properties"]["user_password"] == "[FILTERED]" # 值替换结构保留 assert body["events"][1]["properties"] != {"user_password": "hunter2"} # ---- c) 超批 rejected + 截断 ---- def test_raw_batch_rejected_truncated(db, client): _site() r = _post(client, _body(51, vid="v1")) # >50 事件整批拒 assert r.json()["data"]["accepted"] == 0 rb = RawBatch.objects.get() assert rb.status == "rejected" and rb.event_count == 0 and rb.rejected_count == 1 # 51 事件原始 JSON 远超 4KB → 截断到 4096 + 标记后缀（≤ ~4.1KB） assert len(rb.body) == 4096 + len("...[TRUNCATED]") assert rb.body.endswith("...[TRUNCATED]") assert rb.visitor is None or rb.visitor.visitor_id == "v1" # 超批不建访客，允许 None # ---- d) 重放每次都记 ---- def test_raw_batch_replay_two_rows(db, client): _site() _post(client, _body(1, vid="v1")) _post(client, _body(1, vid="v1")) assert RawBatch.objects.count() == 2 # 原始记录语义：每次接收一行 # ---- e) 管理台 API：分页与字段 ---- def test_raw_batches_api_pagination_and_fields(db, client, member_user): _auth(client, member_user) site = _site() v = Visitor.objects.create(site=site, visitor_id="v1") for i in range(15): rb = RawBatch.objects.create(site=site, visitor=v, batch_id=f"b{i:02d}", body='{"visitor_id":"v1"}', event_count=i % 3, rejected_count=0, status="accepted", ip="1.2.3.4") # received_at 是 auto_now_add，update 绕开制造时间差（与 test_visitors 同款手法） RawBatch.objects.filter(pk=rb.pk).update(received_at=NOW - timedelta(minutes=i)) r = client.get("/api/admin/visitors/v1/raw-batches") assert r.status_code == 200 d = r.json()["data"] assert d["total"] == 15 and d["page"] == 1 and len(d["list"]) == 10 # page_size 10 row = d["list"][0] assert set(row) == {"id", "batch_id", "status", "event_count", "rejected_count", "received_at", "body"} assert row["batch_id"] == "b00" # -received_at：最新在前 d2 = client.get("/api/admin/visitors/v1/raw-batches", {"page": "2"}).json()["data"] assert d2["page"] == 2 and len(d2["list"]) == 5 assert [x["batch_id"] for x in d2["list"]][-1] == "b14" r = client.get("/api/admin/visitors/nope/raw-batches") assert r.status_code == 404 body = r.json() assert body["code"] == 404 and body["data"] is None and body["message"] == "visitor_not_found" # ---- f) 清理命令 ---- def test_cleanup_raw_command(db): site = _site() v = Visitor.objects.create(site=site, visitor_id="v1") old = RawBatch.objects.create(site=site, visitor=v, body="{}") RawBatch.objects.filter(pk=old.pk).update(received_at=NOW - timedelta(days=31)) new = RawBatch.objects.create(site=site, visitor=v, body="{}") # 刚收到 out = StringIO() call_command("cleanup_raw", "--days", 30, stdout=out) assert not RawBatch.objects.filter(pk=old.pk).exists() # 30 天前删除 assert RawBatch.objects.filter(pk=new.pk).exists() # 新的保留 assert "1" in out.getvalue() # 输出删除数