浅念
189 lines
11 KiB
Python
189 lines
11 KiB
Python
from django.db import models
|
||
import secrets
|
||
|
||
from tracker import constants as C
|
||
|
||
|
||
def short_public_id() -> str:
|
||
return "site_" + secrets.token_urlsafe(12)[:16]
|
||
|
||
|
||
class Site(models.Model):
|
||
name = models.CharField(max_length=100)
|
||
site_id = models.CharField(max_length=32, unique=True, default=short_public_id)
|
||
domains = models.JSONField(default=list) # 域名白名单 ["www.example.com"]
|
||
allow_no_origin = models.BooleanField(default=False) # 本地/demo 调试用
|
||
is_active = models.BooleanField(default=True)
|
||
timezone = models.CharField(max_length=64, default="Asia/Shanghai")
|
||
created_at = models.DateTimeField(auto_now_add=True)
|
||
updated_at = models.DateTimeField(auto_now=True)
|
||
|
||
class Meta:
|
||
db_table = "site"
|
||
|
||
|
||
class Visitor(models.Model):
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="visitors")
|
||
visitor_id = models.CharField(max_length=64)
|
||
first_seen_at = models.DateTimeField(auto_now_add=True)
|
||
last_seen_at = models.DateTimeField(auto_now_add=True)
|
||
first_referrer_domain = models.CharField(max_length=200, blank=True, default="")
|
||
first_utm = models.JSONField(default=dict, blank=True)
|
||
first_landing_path = models.CharField(max_length=500, blank=True, default="")
|
||
device_type = models.CharField(max_length=16, blank=True, default="")
|
||
os = models.CharField(max_length=64, blank=True, default="")
|
||
browser = models.CharField(max_length=64, blank=True, default="")
|
||
language = models.CharField(max_length=32, blank=True, default="")
|
||
timezone_name = models.CharField(max_length=64, blank=True, default="")
|
||
screen = models.CharField(max_length=32, blank=True, default="")
|
||
device_model = models.CharField(max_length=64, blank=True, default="") # UA-CH model（如 Pixel 7；晚到 hints 允许补填一次）
|
||
env_extra = models.JSONField(default=dict, blank=True) # 扩展环境快照（constants.EXTENDED_ENV_KEYS 白名单，首触快照）
|
||
last_ip = models.GenericIPAddressField(null=True, blank=True)
|
||
country = models.CharField(max_length=64, blank=True, default="")
|
||
province = models.CharField(max_length=64, blank=True, default="")
|
||
city = models.CharField(max_length=64, blank=True, default="")
|
||
visit_count = models.PositiveIntegerField(default=0) # 会话数（服务端按新会话递增）
|
||
active_days = models.PositiveIntegerField(default=0) # 活跃天数
|
||
last_active_date = models.DateField(null=True, blank=True)
|
||
is_bot = models.BooleanField(default=False) # UA 爬虫标记，报表默认排除
|
||
|
||
class Meta:
|
||
db_table = "visitor"
|
||
constraints = [models.UniqueConstraint(fields=["site", "visitor_id"], name="uq_site_visitor")]
|
||
indexes = [
|
||
models.Index(fields=["site", "-last_seen_at"]),
|
||
models.Index(fields=["site", "country"]),
|
||
]
|
||
|
||
|
||
class VisitSession(models.Model):
|
||
# db_index=False：单列 FK 索引冗余——site 由 (site, started_at)、visitor 由 (visitor, -started_at) 复合索引前缀覆盖
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="sessions", db_index=False)
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="sessions", db_index=False)
|
||
session_id = models.CharField(max_length=64)
|
||
started_at = models.DateTimeField()
|
||
ended_at = models.DateTimeField()
|
||
referrer_domain = models.CharField(max_length=200, blank=True, default="")
|
||
utm = models.JSONField(default=dict, blank=True)
|
||
landing_path = models.CharField(max_length=500, blank=True, default="")
|
||
event_count = models.PositiveIntegerField(default=0)
|
||
duration_ms = models.PositiveBigIntegerField(default=0) # 会话内事件时间跨度
|
||
stay_ms = models.PositiveBigIntegerField(default=0) # 有效停留累计（page_stay active_ms 之和）
|
||
resume_of = models.ForeignKey("self", null=True, blank=True, on_delete=models.SET_NULL,
|
||
related_name="resumes") # schema v2：session_resume 续接的前一会话（被指向会话删除后置 NULL 保留链）
|
||
|
||
class Meta:
|
||
db_table = "visit_session"
|
||
constraints = [models.UniqueConstraint(fields=["site", "session_id"], name="uq_site_session")]
|
||
indexes = [
|
||
models.Index(fields=["visitor", "-started_at"]),
|
||
models.Index(fields=["site", "started_at"]), # 按日聚合/时间线查询
|
||
]
|
||
|
||
|
||
class Event(models.Model):
|
||
# site/session 的单列 FK 索引冗余：site 由 (site, visitor/-occurred_at)、(site, event_name, -occurred_at)、
|
||
# (site, occurred_at) 前缀覆盖，session 由 (session, occurred_at) 前缀覆盖，故 db_index=False。
|
||
# visitor 保留默认索引：无其他以 visitor 打头的索引，级联删除（visitor 反向删 events）需要它。
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="events", db_index=False)
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="events")
|
||
session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="events", null=True, db_index=False)
|
||
event_id = models.CharField(max_length=64)
|
||
event_name = models.CharField(max_length=50)
|
||
occurred_at = models.DateTimeField()
|
||
received_at = models.DateTimeField(auto_now_add=True)
|
||
path = models.CharField(max_length=500, blank=True, default="")
|
||
title = models.CharField(max_length=200, blank=True, default="")
|
||
properties = models.JSONField(default=dict, blank=True)
|
||
ip = models.GenericIPAddressField(null=True, blank=True)
|
||
country = models.CharField(max_length=64, blank=True, default="")
|
||
province = models.CharField(max_length=64, blank=True, default="")
|
||
city = models.CharField(max_length=64, blank=True, default="")
|
||
late = models.BooleanField(default=False)
|
||
is_bot = models.BooleanField(default=False)
|
||
# ---- schema v2（阶段A）：旧行空 = legacy ----
|
||
page_id = models.CharField(max_length=64, blank=True, default="", db_index=True) # 埋点实例归属（spa 路由切换每页新 id）
|
||
active_from_ms = models.BigIntegerField(null=True, blank=True) # 活跃区间起（页面时间戳 ms）
|
||
active_to_ms = models.BigIntegerField(null=True, blank=True) # 活跃区间止
|
||
flush_reason = models.CharField(max_length=16, blank=True, default="") # periodic/hidden/unload/interval等
|
||
time_quality = models.CharField(max_length=16, blank=True, default="") # ok/adjusted/monotonic…（late 的补充细化）
|
||
sdk_version = models.CharField(max_length=16, blank=True, default="")
|
||
schema_version = models.PositiveSmallIntegerField(null=True, blank=True) # 事件级信（缺省为批级/1）
|
||
|
||
class Meta:
|
||
db_table = "event"
|
||
constraints = [models.UniqueConstraint(fields=["site", "event_id"], name="uq_site_event")]
|
||
indexes = [
|
||
models.Index(fields=["site", "visitor", "-occurred_at"]),
|
||
models.Index(fields=["session", "occurred_at"]),
|
||
models.Index(fields=["site", "event_name", "-occurred_at"]),
|
||
models.Index(fields=["site", "occurred_at"]), # refresh_daily 按日聚合扫描
|
||
]
|
||
|
||
|
||
class RawBatch(models.Model):
|
||
"""原始上报批次存档：每次 collect/batch 接收都落一行（含被拒批次）。
|
||
|
||
用途：排查"事件为何没进报表"——原始 JSON 与入库结果（event_count/rejected_count/status）
|
||
并排可对照白名单/超批/幂等拒绝原因。body 是脱敏后快照（敏感键值 → [FILTERED]，结构保留）；
|
||
被拒批次截断 4KB。保留期 30 天由 cleanup_raw 命令管理（生产 cron 每日执行）。
|
||
写入是 best-effort：失败不影响主采集流程（见 collect.py）。
|
||
"""
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="raw_batches")
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="raw_batches", null=True)
|
||
batch_id = models.CharField(max_length=64, blank=True, default="")
|
||
body = models.TextField() # 脱敏后的整包 JSON 文本
|
||
event_count = models.PositiveIntegerField(default=0)
|
||
rejected_count = models.PositiveIntegerField(default=0)
|
||
status = models.CharField(max_length=16, default="accepted") # accepted/partial/rejected
|
||
received_at = models.DateTimeField(auto_now_add=True)
|
||
ip = models.GenericIPAddressField(null=True, blank=True)
|
||
request_id = models.CharField(max_length=64, blank=True, default="") # schema v2：批级请求标识（对账/去重排查）
|
||
|
||
class Meta:
|
||
db_table = "raw_batch"
|
||
indexes = [models.Index(fields=["visitor", "-received_at"]),
|
||
models.Index(fields=["site", "-received_at"])]
|
||
|
||
|
||
class SourceClassification(models.Model):
|
||
"""来源分类结果（attribution 规则引擎的派生存储）。
|
||
|
||
每访客两行 scope：first_touch（首访来源，session=None）/ session_entry（每次新会话入口）。
|
||
写入两路：ingest 增量（新访客/新会话时 best-effort）+ recompute_attribution 全量重算
|
||
（幂等 upsert）。resolution 表达证据可信度：resolved/combined/conflict/label_only/unknown——
|
||
观察到的 referrer 与 UTM 自述矛盾时并列存 detail，不武断取一边。
|
||
rule_version 关联 constants.ATTRIBUTION_RULE_VERSION：规则升级后可识别存量行需重算。
|
||
"""
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="source_classifications")
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="source_classifications", null=True)
|
||
session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="source_classifications", null=True)
|
||
scope = models.CharField(max_length=16) # first_touch / session_entry
|
||
channel = models.CharField(max_length=32, blank=True, default="")
|
||
platform = models.CharField(max_length=64, blank=True, default="")
|
||
variant = models.CharField(max_length=64, blank=True, default="")
|
||
search_term = models.CharField(max_length=200, blank=True, default="")
|
||
evidence_kind = models.CharField(max_length=32, blank=True, default="")
|
||
resolution = models.CharField(max_length=16, blank=True, default="")
|
||
detail = models.JSONField(default=dict, blank=True)
|
||
rule_version = models.CharField(max_length=16, default=C.ATTRIBUTION_RULE_VERSION)
|
||
computed_at = models.DateTimeField(auto_now=True)
|
||
|
||
class Meta:
|
||
db_table = "source_classification"
|
||
indexes = [models.Index(fields=["site", "visitor", "scope"]),
|
||
models.Index(fields=["site", "channel"])]
|
||
|
||
|
||
class DailyStat(models.Model):
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="daily_stats", db_index=False) # uq_site_date 前缀覆盖
|
||
date = models.DateField()
|
||
pv = models.PositiveIntegerField(default=0)
|
||
uv = models.PositiveIntegerField(default=0)
|
||
sessions = models.PositiveIntegerField(default=0)
|
||
new_visitors = models.PositiveIntegerField(default=0)
|
||
|
||
class Meta:
|
||
db_table = "daily_stat"
|
||
constraints = [models.UniqueConstraint(fields=["site", "date"], name="uq_site_date")]
|