feat: 原始上报数据存储（raw_batch 表/脱敏/访客关联/详情页 Tab/30天清理）

Co-Authored-By: Claude Code <noreply@anthropic.com>

server/tracker/api/collect.py

@@ -7,8 +7,9 @@ from rest_framework.permissions import AllowAny
 from rest_framework.throttling import AnonRateThrottle
 from rest_framework.parsers import BaseParser
 from rest_framework.settings import api_settings
-from tracker.models import Site
+from tracker.models import RawBatch, Site, Visitor
 from tracker.services.ingest import ingest_batch
+from tracker.services.validation import sanitize_payload
 
 class PlainJSONParser(BaseParser):
     media_type = "text/plain"
@@ -56,6 +57,9 @@ class CollectBatchView(APIView):
         if not host and not site.allow_no_origin:
             return self._deny(403, "not_allowed")
         payload = request.data if isinstance(request.data, dict) else {}
+        # 原始批次快照：须在 occurred_at 字符串转 datetime 之前深拷贝（datetime 不可 JSON 序列化），
+        # 且独立于 ingest/validate 的就地清洗（path 截断等）——存的是"到达时"的原文（脱敏后）。
+        snapshot = sanitize_payload(json.loads(json.dumps(payload, ensure_ascii=False)))
         for ev in payload.get("events") or []:
             if isinstance(ev, dict) and "occurred_at" in ev:
                 try:
@@ -66,6 +70,10 @@ class CollectBatchView(APIView):
             return self._deny(400, "invalid_payload")
         ip = self._client_ip(request)
         r = ingest_batch(site, payload, ip, request.headers.get("User-Agent", ""))
+        try:
+            self._save_raw_batch(site, payload, snapshot, r, ip)
+        except Exception:   # best-effort：原始表任何失败（含孤代理 encode 崩溃）不影响主采集响应
+            pass
         return Response({"code": 200,
             "data": {"accepted": r["accepted"], "duplicated": r["duplicated"]},
             "message": "ok"})
@@ -95,6 +103,26 @@ class CollectBatchView(APIView):
                              "message": response.data["detail"]}
         return response
 
+    def _save_raw_batch(self, site, payload, snapshot, r, ip):
+        """原始批次落库（每次接收一行，含被拒批次）。失败由调用方静默吞掉。
+
+        rejected（accepted=0 且有拒绝）整批截断 4KB——超批 51 事件的原文必然超限；
+        partial（部分拒绝）不截：正常批次本就受 256KB 请求体上限约束，TextField 可容。
+        visitor 允许 None：被拒批次（如超批）不会创建访客记录。
+        """
+        body = json.dumps(snapshot, ensure_ascii=False)
+        status = ("rejected" if not r["accepted"] and r["rejected"]
+                  else "partial" if r["rejected"] else "accepted")
+        if status == "rejected" and len(body) > 4096:
+            body = body[:4096] + "...[TRUNCATED]"
+        vis = Visitor.objects.filter(site=site, visitor_id=payload.get("visitor_id")).first()
+        RawBatch.objects.create(
+            site=site, visitor=vis,
+            batch_id=str(payload.get("batch_id", ""))[:64],
+            body=body, event_count=r["accepted"],
+            rejected_count=len(r["rejected"]) if isinstance(r["rejected"], list) else r["rejected"],
+            status=status, ip=ip)
+
     def _match_site(self, request):
         sid = (request.data or {}).get("site_id") if isinstance(request.data, dict) else None
         return Site.objects.filter(site_id=sid).first() if sid else None

server/tracker/api/visitors.py

@@ -13,9 +13,10 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from tracker.api.envelope import EnvelopeMixin, ok
-from tracker.models import Event, Site, VisitSession, Visitor
+from tracker.models import Event, RawBatch, Site, VisitSession, Visitor
 
 PAGE_SIZE = 20
+RAW_PAGE_SIZE = 10   # 原始批次 body 较大，页小一些
 TIMELINE_DEFAULT_DAYS = 30
 TIMELINE_MAX_DAYS = 90
 TIMELINE_MAX_EVENTS = 1000
@@ -246,3 +247,30 @@ class VisitorSessionView(EnvelopeMixin, APIView):
             return _not_found("session_not_found")
         events = Event.objects.filter(session=session).order_by("occurred_at", "pk")  # (session,occurred_at) 索引
         return ok({**_session_summary(session), "events": _merge_page_stay(events)})
+
+
+class VisitorRawBatchesView(EnvelopeMixin, APIView):
+    """访客原始上报批次（raw_batch）：排查"上报了为何没进报表"。
+
+    body 是脱敏后的整包 JSON 文本（敏感值已 [FILTERED]、被拒批次已截断），
+    前端以 <pre> 文本渲染（禁 v-html）。按 -received_at 倒序分页，走 (visitor,-received_at) 索引。
+    """
+
+    def get(self, request, visitor_id):
+        visitor = _get_visitor(visitor_id)
+        if not visitor:
+            return _not_found("visitor_not_found")
+        qs = RawBatch.objects.filter(visitor=visitor)
+        total = qs.count()
+        try:
+            page = int(request.query_params.get("page", 1))
+        except (TypeError, ValueError):
+            page = 1
+        page = max(1, page)
+        batches = qs.order_by("-received_at", "-pk")[
+            (page - 1) * RAW_PAGE_SIZE: page * RAW_PAGE_SIZE]
+        return ok({"list": [{"id": b.id, "batch_id": b.batch_id, "status": b.status,
+                             "event_count": b.event_count, "rejected_count": b.rejected_count,
+                             "received_at": _iso(b.received_at), "body": b.body}
+                            for b in batches],
+                   "total": total, "page": page})