feat: 原始上报数据存储（raw_batch 表/脱敏/访客关联/详情页 Tab/30天清理）
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -93,6 +93,26 @@ export interface VisitorListResp {
|
||||
page: number
|
||||
}
|
||||
|
||||
/** 原始上报批次行（visitors.py `VisitorRawBatchesView`） */
|
||||
export interface RawBatchRow {
|
||||
id: number
|
||||
batch_id: string
|
||||
/** accepted / partial / rejected */
|
||||
status: string
|
||||
event_count: number
|
||||
rejected_count: number
|
||||
received_at: string
|
||||
/** 脱敏后的整包 JSON 文本（敏感值已 [FILTERED]、被拒批次已截断）——只做文本渲染，禁 v-html */
|
||||
body: string
|
||||
}
|
||||
|
||||
/** 原始批次列表响应 */
|
||||
export interface RawBatchListResp {
|
||||
list: RawBatchRow[]
|
||||
total: number
|
||||
page: number
|
||||
}
|
||||
|
||||
/**
|
||||
* 访客列表筛选参数。
|
||||
* online/is_new/show_bot 是后端字符串哨兵（仅 "1" 生效，见 visitors.py `_page_visitors`）——
|
||||
@@ -135,3 +155,10 @@ export const getSession = (visitorId: string, sessionId: string) =>
|
||||
request.get<SessionSummary & { events: TimelineEvent[] }>({
|
||||
url: `/admin/visitors/${visitorId}/sessions/${sessionId}`
|
||||
})
|
||||
|
||||
/** 访客原始上报批次（page_size 10，按接收时间倒序；404 时后端返回 visitor_not_found） */
|
||||
export const getRawBatches = (visitorId: string, page = 1) =>
|
||||
request.get<RawBatchListResp>({
|
||||
url: `/admin/visitors/${visitorId}/raw-batches`,
|
||||
params: { page }
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<!-- 访客统计 · 访客详情（档案卡 + 跨天时间线 + 会话列表/抽屉） -->
|
||||
<!-- 访客统计 · 访客详情（档案卡 + 跨天时间线 + 会话列表/抽屉 + 原始上报批次） -->
|
||||
<!-- keepAlive=true 且路由 path 固定：访客切换复用同一实例，刷新依赖 onActivated + query watch -->
|
||||
<template>
|
||||
<div class="visitor-detail-page p-5">
|
||||
@@ -207,9 +207,105 @@
|
||||
</ElTable>
|
||||
</div>
|
||||
</ElTabPane>
|
||||
|
||||
<!-- Tab 3：原始上报批次（raw_batch，排查"上报了为何没进报表"） -->
|
||||
<ElTabPane label="原始数据" name="raw">
|
||||
<div class="art-card px-5 py-4">
|
||||
<div class="flex-c flex-wrap gap-2 mb-3">
|
||||
<h3 class="m-0 text-sm font-semibold">原始上报批次</h3>
|
||||
<span class="text-xs text-g-500">
|
||||
每次接收的原样存档（保留 30 天）；敏感字段已脱敏为 [FILTERED]，被拒批次超 4KB 截断
|
||||
</span>
|
||||
</div>
|
||||
<ElTable
|
||||
:data="rawBatches?.list ?? []"
|
||||
v-loading="rawLoading"
|
||||
empty-text="暂无原始上报数据"
|
||||
row-class-name="session-row"
|
||||
@row-click="openRawDrawer"
|
||||
>
|
||||
<ElTableColumn label="接收时间" width="170">
|
||||
<template #default="{ row }">{{ formatDateTime(row.received_at) }}</template>
|
||||
</ElTableColumn>
|
||||
<ElTableColumn label="批次 ID" min-width="150">
|
||||
<template #default="{ row }">
|
||||
<div v-if="row.batch_id" class="flex-c gap-1">
|
||||
<code class="session-id-code">{{ row.batch_id.slice(0, 8) }}</code>
|
||||
<ElButton
|
||||
link
|
||||
type="primary"
|
||||
size="small"
|
||||
@click.stop="copyBatchId(row.batch_id)"
|
||||
>
|
||||
复制
|
||||
</ElButton>
|
||||
</div>
|
||||
<span v-else>—</span>
|
||||
</template>
|
||||
</ElTableColumn>
|
||||
<ElTableColumn label="状态" width="100">
|
||||
<template #default="{ row }">
|
||||
<ElTag :type="rawStatus(row.status).type" size="small">
|
||||
{{ rawStatus(row.status).label }}
|
||||
</ElTag>
|
||||
</template>
|
||||
</ElTableColumn>
|
||||
<ElTableColumn prop="event_count" label="入库数" width="80" align="right" />
|
||||
<ElTableColumn prop="rejected_count" label="拒绝数" width="80" align="right" />
|
||||
</ElTable>
|
||||
<div class="flex-c justify-end mt-3">
|
||||
<ElPagination
|
||||
:total="rawBatches?.total ?? 0"
|
||||
:page-size="RAW_PAGE_SIZE"
|
||||
:current-page="rawPage"
|
||||
layout="total, prev, pager, next"
|
||||
@current-change="onRawPageChange"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</ElTabPane>
|
||||
</ElTabs>
|
||||
</template>
|
||||
|
||||
<!-- 原始批次详情抽屉：body 是服务端脱敏后的 JSON 文本，<pre> 文本插值渲染（禁 v-html） -->
|
||||
<ElDrawer v-model="rawDrawerVisible" title="原始上报批次" size="46%">
|
||||
<template v-if="rawCurrent">
|
||||
<div class="drawer-summary">
|
||||
<div class="flex-c flex-wrap gap-x-4 gap-y-1.5">
|
||||
<span class="text-g-600 text-sm">批次 ID</span>
|
||||
<template v-if="rawCurrent.batch_id">
|
||||
<code class="session-id-code">{{ rawCurrent.batch_id }}</code>
|
||||
<ElButton link type="primary" size="small" @click="copyBatchId(rawCurrent.batch_id)">
|
||||
复制
|
||||
</ElButton>
|
||||
</template>
|
||||
<span v-else class="text-sm">—</span>
|
||||
</div>
|
||||
<div class="summary-grid">
|
||||
<div class="flex-c gap-2">
|
||||
<span class="text-xs text-g-600">接收时间</span>
|
||||
<span class="text-sm">{{ formatDateTime(rawCurrent.received_at) }}</span>
|
||||
</div>
|
||||
<div class="flex-c gap-2">
|
||||
<span class="text-xs text-g-600">状态</span>
|
||||
<ElTag :type="rawStatus(rawCurrent.status).type" size="small">
|
||||
{{ rawStatus(rawCurrent.status).label }}
|
||||
</ElTag>
|
||||
</div>
|
||||
<div class="flex-c gap-2">
|
||||
<span class="text-xs text-g-600">入库数</span>
|
||||
<span class="text-sm">{{ rawCurrent.event_count }}</span>
|
||||
</div>
|
||||
<div class="flex-c gap-2">
|
||||
<span class="text-xs text-g-600">拒绝数</span>
|
||||
<span class="text-sm">{{ rawCurrent.rejected_count }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<pre class="raw-json-body">{{ prettyBody(rawCurrent.body) }}</pre>
|
||||
</template>
|
||||
</ElDrawer>
|
||||
|
||||
<!-- 会话详情抽屉 -->
|
||||
<ElDrawer v-model="drawerVisible" title="会话详情" size="34%">
|
||||
<div v-loading="sessionLoading" class="drawer-body">
|
||||
@@ -268,9 +364,11 @@
|
||||
<script setup lang="ts">
|
||||
import { Back as BackIcon } from '@element-plus/icons-vue'
|
||||
import {
|
||||
getRawBatches,
|
||||
getSession,
|
||||
getTimeline,
|
||||
getVisitor,
|
||||
type RawBatchRow,
|
||||
type SessionSummary,
|
||||
type TimelineEvent,
|
||||
type TimelineResp,
|
||||
@@ -346,7 +444,7 @@
|
||||
])
|
||||
|
||||
// ---------- 跨天时间线 ----------
|
||||
const activeTab = ref<'timeline' | 'sessions'>('timeline')
|
||||
const activeTab = ref<'timeline' | 'sessions' | 'raw'>('timeline')
|
||||
const days = ref(30)
|
||||
const timeline = ref<TimelineResp | null>(null)
|
||||
const timelineLoading = ref(false)
|
||||
@@ -408,6 +506,75 @@
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 原始上报批次（Tab 3） ----------
|
||||
const RAW_PAGE_SIZE = 10 // 与后端 visitors.py RAW_PAGE_SIZE 同步
|
||||
const rawBatches = ref<{ list: RawBatchRow[]; total: number; page: number } | null>(null)
|
||||
const rawLoading = ref(false)
|
||||
const rawPage = ref(1)
|
||||
let rawSeq = 0
|
||||
|
||||
const loadRawBatches = async (page = rawPage.value) => {
|
||||
const vid = loadedVid
|
||||
if (!vid) return
|
||||
const seq = ++rawSeq
|
||||
rawPage.value = page
|
||||
rawLoading.value = true
|
||||
try {
|
||||
const resp = await getRawBatches(vid, page)
|
||||
if (seq === rawSeq) rawBatches.value = resp
|
||||
} catch {
|
||||
if (seq === rawSeq) rawBatches.value = null
|
||||
} finally {
|
||||
if (seq === rawSeq) rawLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
const onRawPageChange = (page: number) => {
|
||||
loadRawBatches(page)
|
||||
}
|
||||
|
||||
// 首次切到原始 Tab 才拉取（懒加载）；失败置 null，切走再切回可重试
|
||||
watch(activeTab, (tab) => {
|
||||
if (tab === 'raw' && rawBatches.value === null && loadedVid) loadRawBatches()
|
||||
})
|
||||
|
||||
/** 状态三色映射（accepted/partial/rejected；未知值兜底 info） */
|
||||
const rawStatusMap: Record<
|
||||
string,
|
||||
{ label: string; type: 'success' | 'warning' | 'danger' | 'info' }
|
||||
> = {
|
||||
accepted: { label: '已入库', type: 'success' },
|
||||
partial: { label: '部分拒绝', type: 'warning' },
|
||||
rejected: { label: '已拒绝', type: 'danger' }
|
||||
}
|
||||
const rawStatus = (status: string) => rawStatusMap[status] ?? { label: status, type: 'info' }
|
||||
|
||||
/** 被拒批次 body 被 4KB 截断后不是合法 JSON——parse 失败原文直出 */
|
||||
const prettyBody = (body: string): string => {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(body), null, 2)
|
||||
} catch {
|
||||
return body
|
||||
}
|
||||
}
|
||||
|
||||
// 原始批次抽屉（body 随行带出，无需再请求）
|
||||
const rawDrawerVisible = ref(false)
|
||||
const rawCurrent = ref<RawBatchRow | null>(null)
|
||||
const openRawDrawer = (row: RawBatchRow) => {
|
||||
rawCurrent.value = row
|
||||
rawDrawerVisible.value = true
|
||||
}
|
||||
|
||||
const copyBatchId = async (batchId: string) => {
|
||||
try {
|
||||
await copy(batchId)
|
||||
ElMessage.success('批次 ID 已复制到剪贴板')
|
||||
} catch {
|
||||
ElMessage.error('复制失败，请手动选择复制')
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- keepAlive 访客切换 ----------
|
||||
/**
|
||||
* 路由 keepAlive 且 path 固定为 /tracker/visitor/detail：列表点不同访客进来复用同一实例。
|
||||
@@ -418,9 +585,14 @@
|
||||
const vid = routeVisitorId.value
|
||||
if (!vid || vid === loadedVid) return
|
||||
loadedVid = vid
|
||||
// 换访客：清档案/时间线/抽屉态，防旧访客 ID 在 loading 遮罩下闪现（days 保持所选范围重新拉取）
|
||||
// 换访客：清档案/时间线/原始批次/抽屉态，防旧访客 ID 在 loading 遮罩下闪现
|
||||
//（days 保持所选范围重新拉取；原始 Tab 懒加载——分页重置回第 1 页，切到 Tab 时再拉）
|
||||
detail.value = null
|
||||
timeline.value = null
|
||||
rawBatches.value = null
|
||||
rawPage.value = 1
|
||||
rawDrawerVisible.value = false
|
||||
rawCurrent.value = null
|
||||
drawerVisible.value = false
|
||||
sessionDetail.value = null
|
||||
loadDetail(vid)
|
||||
@@ -532,4 +704,19 @@
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 8px 12px;
|
||||
}
|
||||
|
||||
// 原始批次 JSON（脱敏后文本，pre 渲染）
|
||||
.raw-json-body {
|
||||
max-height: calc(100vh - 260px);
|
||||
padding: 12px 14px;
|
||||
margin: 12px 0 0;
|
||||
overflow: auto;
|
||||
font-family: var(--el-font-family-mono, monospace);
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
border-radius: 6px;
|
||||
background: var(--el-fill-color-lighter);
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -4,7 +4,8 @@ from tracker.api.auth import LoginView, RefreshView, MeView, UserCreateView
|
||||
from tracker.api.sites import SiteListView, SiteDetailView
|
||||
from tracker.api.dashboard import OverviewView
|
||||
from tracker.api.visitors import (VisitorDetailView, VisitorListView,
|
||||
VisitorSessionView, VisitorTimelineView)
|
||||
VisitorRawBatchesView, VisitorSessionView,
|
||||
VisitorTimelineView)
|
||||
|
||||
urlpatterns = [
|
||||
path("api/collect/batch", CollectBatchView.as_view()),
|
||||
@@ -23,4 +24,6 @@ urlpatterns += [
|
||||
path("api/admin/visitors/<str:visitor_id>/timeline", VisitorTimelineView.as_view()),
|
||||
path("api/admin/visitors/<str:visitor_id>/sessions/<str:session_id>",
|
||||
VisitorSessionView.as_view()),
|
||||
path("api/admin/visitors/<str:visitor_id>/raw-batches",
|
||||
VisitorRawBatchesView.as_view()),
|
||||
]
|
||||
|
||||
@@ -7,8 +7,9 @@ from rest_framework.permissions import AllowAny
|
||||
from rest_framework.throttling import AnonRateThrottle
|
||||
from rest_framework.parsers import BaseParser
|
||||
from rest_framework.settings import api_settings
|
||||
from tracker.models import Site
|
||||
from tracker.models import RawBatch, Site, Visitor
|
||||
from tracker.services.ingest import ingest_batch
|
||||
from tracker.services.validation import sanitize_payload
|
||||
|
||||
class PlainJSONParser(BaseParser):
|
||||
media_type = "text/plain"
|
||||
@@ -56,6 +57,9 @@ class CollectBatchView(APIView):
|
||||
if not host and not site.allow_no_origin:
|
||||
return self._deny(403, "not_allowed")
|
||||
payload = request.data if isinstance(request.data, dict) else {}
|
||||
# 原始批次快照：须在 occurred_at 字符串转 datetime 之前深拷贝（datetime 不可 JSON 序列化），
|
||||
# 且独立于 ingest/validate 的就地清洗（path 截断等）——存的是"到达时"的原文（脱敏后）。
|
||||
snapshot = sanitize_payload(json.loads(json.dumps(payload, ensure_ascii=False)))
|
||||
for ev in payload.get("events") or []:
|
||||
if isinstance(ev, dict) and "occurred_at" in ev:
|
||||
try:
|
||||
@@ -66,6 +70,10 @@ class CollectBatchView(APIView):
|
||||
return self._deny(400, "invalid_payload")
|
||||
ip = self._client_ip(request)
|
||||
r = ingest_batch(site, payload, ip, request.headers.get("User-Agent", ""))
|
||||
try:
|
||||
self._save_raw_batch(site, payload, snapshot, r, ip)
|
||||
except Exception: # best-effort：原始表任何失败（含孤代理 encode 崩溃）不影响主采集响应
|
||||
pass
|
||||
return Response({"code": 200,
|
||||
"data": {"accepted": r["accepted"], "duplicated": r["duplicated"]},
|
||||
"message": "ok"})
|
||||
@@ -95,6 +103,26 @@ class CollectBatchView(APIView):
|
||||
"message": response.data["detail"]}
|
||||
return response
|
||||
|
||||
def _save_raw_batch(self, site, payload, snapshot, r, ip):
|
||||
"""原始批次落库（每次接收一行，含被拒批次）。失败由调用方静默吞掉。
|
||||
|
||||
rejected（accepted=0 且有拒绝）整批截断 4KB——超批 51 事件的原文必然超限；
|
||||
partial（部分拒绝）不截：正常批次本就受 256KB 请求体上限约束，TextField 可容。
|
||||
visitor 允许 None：被拒批次（如超批）不会创建访客记录。
|
||||
"""
|
||||
body = json.dumps(snapshot, ensure_ascii=False)
|
||||
status = ("rejected" if not r["accepted"] and r["rejected"]
|
||||
else "partial" if r["rejected"] else "accepted")
|
||||
if status == "rejected" and len(body) > 4096:
|
||||
body = body[:4096] + "...[TRUNCATED]"
|
||||
vis = Visitor.objects.filter(site=site, visitor_id=payload.get("visitor_id")).first()
|
||||
RawBatch.objects.create(
|
||||
site=site, visitor=vis,
|
||||
batch_id=str(payload.get("batch_id", ""))[:64],
|
||||
body=body, event_count=r["accepted"],
|
||||
rejected_count=len(r["rejected"]) if isinstance(r["rejected"], list) else r["rejected"],
|
||||
status=status, ip=ip)
|
||||
|
||||
def _match_site(self, request):
|
||||
sid = (request.data or {}).get("site_id") if isinstance(request.data, dict) else None
|
||||
return Site.objects.filter(site_id=sid).first() if sid else None
|
||||
|
||||
@@ -13,9 +13,10 @@ from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from tracker.api.envelope import EnvelopeMixin, ok
|
||||
from tracker.models import Event, Site, VisitSession, Visitor
|
||||
from tracker.models import Event, RawBatch, Site, VisitSession, Visitor
|
||||
|
||||
PAGE_SIZE = 20
|
||||
RAW_PAGE_SIZE = 10 # 原始批次 body 较大，页小一些
|
||||
TIMELINE_DEFAULT_DAYS = 30
|
||||
TIMELINE_MAX_DAYS = 90
|
||||
TIMELINE_MAX_EVENTS = 1000
|
||||
@@ -246,3 +247,30 @@ class VisitorSessionView(EnvelopeMixin, APIView):
|
||||
return _not_found("session_not_found")
|
||||
events = Event.objects.filter(session=session).order_by("occurred_at", "pk") # (session,occurred_at) 索引
|
||||
return ok({**_session_summary(session), "events": _merge_page_stay(events)})
|
||||
|
||||
|
||||
class VisitorRawBatchesView(EnvelopeMixin, APIView):
|
||||
"""访客原始上报批次（raw_batch）：排查"上报了为何没进报表"。
|
||||
|
||||
body 是脱敏后的整包 JSON 文本（敏感值已 [FILTERED]、被拒批次已截断），
|
||||
前端以 <pre> 文本渲染（禁 v-html）。按 -received_at 倒序分页，走 (visitor,-received_at) 索引。
|
||||
"""
|
||||
|
||||
def get(self, request, visitor_id):
|
||||
visitor = _get_visitor(visitor_id)
|
||||
if not visitor:
|
||||
return _not_found("visitor_not_found")
|
||||
qs = RawBatch.objects.filter(visitor=visitor)
|
||||
total = qs.count()
|
||||
try:
|
||||
page = int(request.query_params.get("page", 1))
|
||||
except (TypeError, ValueError):
|
||||
page = 1
|
||||
page = max(1, page)
|
||||
batches = qs.order_by("-received_at", "-pk")[
|
||||
(page - 1) * RAW_PAGE_SIZE: page * RAW_PAGE_SIZE]
|
||||
return ok({"list": [{"id": b.id, "batch_id": b.batch_id, "status": b.status,
|
||||
"event_count": b.event_count, "rejected_count": b.rejected_count,
|
||||
"received_at": _iso(b.received_at), "body": b.body}
|
||||
for b in batches],
|
||||
"total": total, "page": page})
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
from tracker.models import RawBatch
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
# 生产调度：1Panel 计划任务每日执行（与 refresh_stats 同款）
|
||||
# docker exec <web容器> python manage.py cleanup_raw --days 30
|
||||
help = "清理 N 天前的原始上报批次（raw_batch 默认保留 30 天）"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("--days", type=int, default=30)
|
||||
|
||||
def handle(self, *args, **opts):
|
||||
cutoff = timezone.now() - timedelta(days=opts["days"])
|
||||
deleted, _ = RawBatch.objects.filter(received_at__lt=cutoff).delete()
|
||||
self.stdout.write(f"deleted {deleted} raw batches")
|
||||
@@ -0,0 +1,33 @@
|
||||
# Generated by Django 5.2.17 on 2026-09-16 07:33
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('tracker', '0002_alter_dailystat_site_alter_event_session_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='RawBatch',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('batch_id', models.CharField(blank=True, default='', max_length=64)),
|
||||
('body', models.TextField()),
|
||||
('event_count', models.PositiveIntegerField(default=0)),
|
||||
('rejected_count', models.PositiveIntegerField(default=0)),
|
||||
('status', models.CharField(default='accepted', max_length=16)),
|
||||
('received_at', models.DateTimeField(auto_now_add=True)),
|
||||
('ip', models.GenericIPAddressField(blank=True, null=True)),
|
||||
('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='raw_batches', to='tracker.site')),
|
||||
('visitor', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='raw_batches', to='tracker.visitor')),
|
||||
],
|
||||
options={
|
||||
'db_table': 'raw_batch',
|
||||
'indexes': [models.Index(fields=['visitor', '-received_at'], name='raw_batch_visitor_9ea281_idx'), models.Index(fields=['site', '-received_at'], name='raw_batch_site_id_ad8e9f_idx')],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -107,6 +107,30 @@ class Event(models.Model):
|
||||
]
|
||||
|
||||
|
||||
class RawBatch(models.Model):
|
||||
"""原始上报批次存档：每次 collect/batch 接收都落一行（含被拒批次）。
|
||||
|
||||
用途：排查"事件为何没进报表"——原始 JSON 与入库结果（event_count/rejected_count/status）
|
||||
并排可对照白名单/超批/幂等拒绝原因。body 是脱敏后快照（敏感键值 → [FILTERED]，结构保留）；
|
||||
被拒批次截断 4KB。保留期 30 天由 cleanup_raw 命令管理（生产 cron 每日执行）。
|
||||
写入是 best-effort：失败不影响主采集流程（见 collect.py）。
|
||||
"""
|
||||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="raw_batches")
|
||||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="raw_batches", null=True)
|
||||
batch_id = models.CharField(max_length=64, blank=True, default="")
|
||||
body = models.TextField() # 脱敏后的整包 JSON 文本
|
||||
event_count = models.PositiveIntegerField(default=0)
|
||||
rejected_count = models.PositiveIntegerField(default=0)
|
||||
status = models.CharField(max_length=16, default="accepted") # accepted/partial/rejected
|
||||
received_at = models.DateTimeField(auto_now_add=True)
|
||||
ip = models.GenericIPAddressField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
db_table = "raw_batch"
|
||||
indexes = [models.Index(fields=["visitor", "-received_at"]),
|
||||
models.Index(fields=["site", "-received_at"])]
|
||||
|
||||
|
||||
class DailyStat(models.Model):
|
||||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="daily_stats", db_index=False) # uq_site_date 前缀覆盖
|
||||
date = models.DateField()
|
||||
|
||||
@@ -51,6 +51,19 @@ def _walk_props(v, depth=0):
|
||||
return True, False, True
|
||||
return True, False, False
|
||||
|
||||
def sanitize_payload(obj):
|
||||
"""原始批次快照脱敏：递归遍历 dict/list，键名命中 SENSITIVE_KEY_PATTERNS（大小写不敏感）
|
||||
→ 值替换为字符串 "[FILTERED]"（结构保留，便于排查时仍看清字段形态）。
|
||||
返回新对象（容器全部重建，叶值原样共享）——调用方在 ingest 之前对 payload 深拷贝后传入，
|
||||
与 ingest/validate_event 的就地改写互不影响。"""
|
||||
if isinstance(obj, dict):
|
||||
return {k: ("[FILTERED]" if _SENSITIVE.search(str(k)) else sanitize_payload(v))
|
||||
for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [sanitize_payload(x) for x in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def validate_event(ev) -> str | None:
|
||||
"""返回 None 表示通过；否则返回拒绝原因码。类型/白名单/体积/深度全在此守门。
|
||||
通过时会就地截断/清洗并写回 ev（path 先 sanitize_url 再截 MAX_PATH_LEN，title 截
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""原始上报批次（raw_batch）：脱敏快照、拒批截断、重放留痕、API 分页、30 天清理。
|
||||
|
||||
设计契约（与用户确认）：
|
||||
- 敏感字段值替换为 [FILTERED]（结构保留）——sanitize_payload 复用 SENSITIVE_KEY_PATTERNS；
|
||||
- 被拒批次也存（body 截断 4KB）；invalid_json（解析失败）不存；
|
||||
- 原始表写入失败不得影响主采集流程（best-effort）；
|
||||
- 每次接收都记一行（"原始记录"语义，重放不合并）。
|
||||
"""
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from io import StringIO
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.cache import cache
|
||||
from django.core.management import call_command
|
||||
from django.utils import timezone
|
||||
from rest_framework.test import APIClient
|
||||
from tracker.models import RawBatch, Site, Visitor
|
||||
|
||||
User = get_user_model()
|
||||
NOW = timezone.now()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cache():
|
||||
cache.clear() # locmem 限流历史跨测试残留，会让压测 429（与 test_collect 同款）
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(): # 覆盖 pytest-django 的 Django Client——认证用例需要 credentials()
|
||||
return APIClient()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def member_user():
|
||||
return User.objects.create_user(username="member", password="memberpass123")
|
||||
|
||||
|
||||
def _auth(client, user):
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
token = RefreshToken.for_user(user)
|
||||
client.credentials(HTTP_AUTHORIZATION=f"Bearer {token.access_token}")
|
||||
|
||||
|
||||
def _post(client, body, origin="https://www.a.com"):
|
||||
return client.post("/api/collect/batch", data=json.dumps(body),
|
||||
content_type="text/plain", HTTP_ORIGIN=origin)
|
||||
|
||||
|
||||
def _body(n=1, sid="s1", vid="v1", events=None):
|
||||
return {"site_id": "site_x", "visitor_id": vid, "session_id": sid,
|
||||
"context": {"referrer_domain": "", "path": "/", "device_type": "PC"},
|
||||
"events": events if events is not None else
|
||||
[{"event_id": f"e{i}", "event_name": "page_view",
|
||||
"occurred_at": NOW.isoformat(), "path": "/", "title": "t",
|
||||
"properties": {}} for i in range(n)]}
|
||||
|
||||
|
||||
def _site():
|
||||
return Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x")
|
||||
|
||||
|
||||
# ---- a) 正常上报 ----
|
||||
|
||||
def test_raw_batch_accepted(db, client):
|
||||
_site()
|
||||
r = _post(client, _body(2, vid="v9"))
|
||||
assert r.status_code == 200
|
||||
rb = RawBatch.objects.get()
|
||||
assert rb.status == "accepted" and rb.event_count == 2 and rb.rejected_count == 0
|
||||
vis = Visitor.objects.get(visitor_id="v9")
|
||||
assert rb.visitor_id == vis.pk and rb.site_id == vis.site_id # 访客关联
|
||||
body = json.loads(rb.body)
|
||||
assert body["visitor_id"] == "v9" # 完整结构在场
|
||||
assert body["context"]["device_type"] == "PC"
|
||||
assert [e["path"] for e in body["events"]] == ["/", "/"] # 快照先于 ingest 就地清洗
|
||||
|
||||
|
||||
# ---- b) 敏感键脱敏 + partial ----
|
||||
|
||||
def test_raw_batch_partial_and_filtered(db, client):
|
||||
_site()
|
||||
events = [
|
||||
{"event_id": "ok1", "event_name": "page_view", "occurred_at": NOW.isoformat(),
|
||||
"path": "/", "title": "t", "properties": {}},
|
||||
{"event_id": "bad1", "event_name": "page_view", "occurred_at": NOW.isoformat(),
|
||||
"path": "/login", "title": "t", "properties": {"user_password": "hunter2"}},
|
||||
]
|
||||
r = _post(client, _body(events=events, vid="v1"))
|
||||
assert r.json()["data"]["accepted"] == 1 # 事件层：敏感事件被拒
|
||||
rb = RawBatch.objects.get()
|
||||
assert rb.status == "partial" and rb.event_count == 1 and rb.rejected_count == 1
|
||||
body = json.loads(rb.body)
|
||||
assert body["events"][1]["properties"]["user_password"] == "[FILTERED]" # 值替换结构保留
|
||||
assert body["events"][1]["properties"] != {"user_password": "hunter2"}
|
||||
|
||||
|
||||
# ---- c) 超批 rejected + 截断 ----
|
||||
|
||||
def test_raw_batch_rejected_truncated(db, client):
|
||||
_site()
|
||||
r = _post(client, _body(51, vid="v1")) # >50 事件整批拒
|
||||
assert r.json()["data"]["accepted"] == 0
|
||||
rb = RawBatch.objects.get()
|
||||
assert rb.status == "rejected" and rb.event_count == 0 and rb.rejected_count == 1
|
||||
# 51 事件原始 JSON 远超 4KB → 截断到 4096 + 标记后缀（≤ ~4.1KB）
|
||||
assert len(rb.body) == 4096 + len("...[TRUNCATED]")
|
||||
assert rb.body.endswith("...[TRUNCATED]")
|
||||
assert rb.visitor is None or rb.visitor.visitor_id == "v1" # 超批不建访客，允许 None
|
||||
|
||||
|
||||
# ---- d) 重放每次都记 ----
|
||||
|
||||
def test_raw_batch_replay_two_rows(db, client):
|
||||
_site()
|
||||
_post(client, _body(1, vid="v1"))
|
||||
_post(client, _body(1, vid="v1"))
|
||||
assert RawBatch.objects.count() == 2 # 原始记录语义：每次接收一行
|
||||
|
||||
|
||||
# ---- e) 管理台 API：分页与字段 ----
|
||||
|
||||
def test_raw_batches_api_pagination_and_fields(db, client, member_user):
|
||||
_auth(client, member_user)
|
||||
site = _site()
|
||||
v = Visitor.objects.create(site=site, visitor_id="v1")
|
||||
for i in range(15):
|
||||
rb = RawBatch.objects.create(site=site, visitor=v, batch_id=f"b{i:02d}",
|
||||
body='{"visitor_id":"v1"}', event_count=i % 3,
|
||||
rejected_count=0, status="accepted", ip="1.2.3.4")
|
||||
# received_at 是 auto_now_add，update 绕开制造时间差（与 test_visitors 同款手法）
|
||||
RawBatch.objects.filter(pk=rb.pk).update(received_at=NOW - timedelta(minutes=i))
|
||||
|
||||
r = client.get("/api/admin/visitors/v1/raw-batches")
|
||||
assert r.status_code == 200
|
||||
d = r.json()["data"]
|
||||
assert d["total"] == 15 and d["page"] == 1 and len(d["list"]) == 10 # page_size 10
|
||||
row = d["list"][0]
|
||||
assert set(row) == {"id", "batch_id", "status", "event_count",
|
||||
"rejected_count", "received_at", "body"}
|
||||
assert row["batch_id"] == "b00" # -received_at：最新在前
|
||||
d2 = client.get("/api/admin/visitors/v1/raw-batches", {"page": "2"}).json()["data"]
|
||||
assert d2["page"] == 2 and len(d2["list"]) == 5
|
||||
assert [x["batch_id"] for x in d2["list"]][-1] == "b14"
|
||||
|
||||
r = client.get("/api/admin/visitors/nope/raw-batches")
|
||||
assert r.status_code == 404
|
||||
body = r.json()
|
||||
assert body["code"] == 404 and body["data"] is None and body["message"] == "visitor_not_found"
|
||||
|
||||
|
||||
# ---- f) 清理命令 ----
|
||||
|
||||
def test_cleanup_raw_command(db):
|
||||
site = _site()
|
||||
v = Visitor.objects.create(site=site, visitor_id="v1")
|
||||
old = RawBatch.objects.create(site=site, visitor=v, body="{}")
|
||||
RawBatch.objects.filter(pk=old.pk).update(received_at=NOW - timedelta(days=31))
|
||||
new = RawBatch.objects.create(site=site, visitor=v, body="{}") # 刚收到
|
||||
|
||||
out = StringIO()
|
||||
call_command("cleanup_raw", "--days", 30, stdout=out)
|
||||
assert not RawBatch.objects.filter(pk=old.pk).exists() # 30 天前删除
|
||||
assert RawBatch.objects.filter(pk=new.pk).exists() # 新的保留
|
||||
assert "1" in out.getvalue() # 输出删除数
|
||||
Reference in New Issue
Block a user