浅念
59de51c062
- constants 规则库 v1：AI/引擎/社交域名表、搜索词参数白名单、付费点击参数、 应用内 UA 标记、CHANNELS 枚举；page_view 白名单开 nav_type/prerendering - services/attribution.classify 纯函数：付费>AI>引擎>社交>referral 分层， UTM 证据叠加（combined/conflict/label_only），应用内 UA 叠加/兜底， 引擎 TLD 国家 variant，搜索词白名单提取（Google 恒空不虚构） - SourceClassification 派生表（迁移 0004）：first_touch/session_entry 双 scope， rule_version 可追溯 - ingest 增量 best-effort：首访客/新会话时分类落库（savepoint+静默， 失败不影响采集）；validation 对 page_view referrer 完整 URL 参数清洗+截500 - recompute_attribution 命令：全量/按站幂等重算 - API：访客列表归因五字段+channel 筛选、详情 attribution 块（含最近5会话入口）、 dashboard channels 聚合（sources 原口径保留）；TS VisitorListParams 加 channel - 测试 97 → 124（引擎20 + 集成/接口7） Co-Authored-By: Claude Code <noreply@anthropic.com>
176 lines
9.5 KiB
Python
176 lines
9.5 KiB
Python
from django.db import models
|
||
import secrets
|
||
|
||
from tracker import constants as C
|
||
|
||
|
||
def short_public_id() -> str:
|
||
return "site_" + secrets.token_urlsafe(12)[:16]
|
||
|
||
|
||
class Site(models.Model):
|
||
name = models.CharField(max_length=100)
|
||
site_id = models.CharField(max_length=32, unique=True, default=short_public_id)
|
||
domains = models.JSONField(default=list) # 域名白名单 ["www.example.com"]
|
||
allow_no_origin = models.BooleanField(default=False) # 本地/demo 调试用
|
||
is_active = models.BooleanField(default=True)
|
||
timezone = models.CharField(max_length=64, default="Asia/Shanghai")
|
||
created_at = models.DateTimeField(auto_now_add=True)
|
||
updated_at = models.DateTimeField(auto_now=True)
|
||
|
||
class Meta:
|
||
db_table = "site"
|
||
|
||
|
||
class Visitor(models.Model):
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="visitors")
|
||
visitor_id = models.CharField(max_length=64)
|
||
first_seen_at = models.DateTimeField(auto_now_add=True)
|
||
last_seen_at = models.DateTimeField(auto_now_add=True)
|
||
first_referrer_domain = models.CharField(max_length=200, blank=True, default="")
|
||
first_utm = models.JSONField(default=dict, blank=True)
|
||
first_landing_path = models.CharField(max_length=500, blank=True, default="")
|
||
device_type = models.CharField(max_length=16, blank=True, default="")
|
||
os = models.CharField(max_length=64, blank=True, default="")
|
||
browser = models.CharField(max_length=64, blank=True, default="")
|
||
language = models.CharField(max_length=32, blank=True, default="")
|
||
timezone_name = models.CharField(max_length=64, blank=True, default="")
|
||
screen = models.CharField(max_length=32, blank=True, default="")
|
||
last_ip = models.GenericIPAddressField(null=True, blank=True)
|
||
country = models.CharField(max_length=64, blank=True, default="")
|
||
province = models.CharField(max_length=64, blank=True, default="")
|
||
city = models.CharField(max_length=64, blank=True, default="")
|
||
visit_count = models.PositiveIntegerField(default=0) # 会话数（服务端按新会话递增）
|
||
active_days = models.PositiveIntegerField(default=0) # 活跃天数
|
||
last_active_date = models.DateField(null=True, blank=True)
|
||
is_bot = models.BooleanField(default=False) # UA 爬虫标记，报表默认排除
|
||
|
||
class Meta:
|
||
db_table = "visitor"
|
||
constraints = [models.UniqueConstraint(fields=["site", "visitor_id"], name="uq_site_visitor")]
|
||
indexes = [
|
||
models.Index(fields=["site", "-last_seen_at"]),
|
||
models.Index(fields=["site", "country"]),
|
||
]
|
||
|
||
|
||
class VisitSession(models.Model):
|
||
# db_index=False：单列 FK 索引冗余——site 由 (site, started_at)、visitor 由 (visitor, -started_at) 复合索引前缀覆盖
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="sessions", db_index=False)
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="sessions", db_index=False)
|
||
session_id = models.CharField(max_length=64)
|
||
started_at = models.DateTimeField()
|
||
ended_at = models.DateTimeField()
|
||
referrer_domain = models.CharField(max_length=200, blank=True, default="")
|
||
utm = models.JSONField(default=dict, blank=True)
|
||
landing_path = models.CharField(max_length=500, blank=True, default="")
|
||
event_count = models.PositiveIntegerField(default=0)
|
||
duration_ms = models.PositiveBigIntegerField(default=0) # 会话内事件时间跨度
|
||
stay_ms = models.PositiveBigIntegerField(default=0) # 有效停留累计（page_stay active_ms 之和）
|
||
|
||
class Meta:
|
||
db_table = "visit_session"
|
||
constraints = [models.UniqueConstraint(fields=["site", "session_id"], name="uq_site_session")]
|
||
indexes = [
|
||
models.Index(fields=["visitor", "-started_at"]),
|
||
models.Index(fields=["site", "started_at"]), # 按日聚合/时间线查询
|
||
]
|
||
|
||
|
||
class Event(models.Model):
|
||
# site/session 的单列 FK 索引冗余：site 由 (site, visitor/-occurred_at)、(site, event_name, -occurred_at)、
|
||
# (site, occurred_at) 前缀覆盖，session 由 (session, occurred_at) 前缀覆盖，故 db_index=False。
|
||
# visitor 保留默认索引：无其他以 visitor 打头的索引，级联删除（visitor 反向删 events）需要它。
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="events", db_index=False)
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="events")
|
||
session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="events", null=True, db_index=False)
|
||
event_id = models.CharField(max_length=64)
|
||
event_name = models.CharField(max_length=50)
|
||
occurred_at = models.DateTimeField()
|
||
received_at = models.DateTimeField(auto_now_add=True)
|
||
path = models.CharField(max_length=500, blank=True, default="")
|
||
title = models.CharField(max_length=200, blank=True, default="")
|
||
properties = models.JSONField(default=dict, blank=True)
|
||
ip = models.GenericIPAddressField(null=True, blank=True)
|
||
country = models.CharField(max_length=64, blank=True, default="")
|
||
province = models.CharField(max_length=64, blank=True, default="")
|
||
city = models.CharField(max_length=64, blank=True, default="")
|
||
late = models.BooleanField(default=False)
|
||
is_bot = models.BooleanField(default=False)
|
||
|
||
class Meta:
|
||
db_table = "event"
|
||
constraints = [models.UniqueConstraint(fields=["site", "event_id"], name="uq_site_event")]
|
||
indexes = [
|
||
models.Index(fields=["site", "visitor", "-occurred_at"]),
|
||
models.Index(fields=["session", "occurred_at"]),
|
||
models.Index(fields=["site", "event_name", "-occurred_at"]),
|
||
models.Index(fields=["site", "occurred_at"]), # refresh_daily 按日聚合扫描
|
||
]
|
||
|
||
|
||
class RawBatch(models.Model):
|
||
"""原始上报批次存档：每次 collect/batch 接收都落一行（含被拒批次）。
|
||
|
||
用途：排查"事件为何没进报表"——原始 JSON 与入库结果（event_count/rejected_count/status）
|
||
并排可对照白名单/超批/幂等拒绝原因。body 是脱敏后快照（敏感键值 → [FILTERED]，结构保留）；
|
||
被拒批次截断 4KB。保留期 30 天由 cleanup_raw 命令管理（生产 cron 每日执行）。
|
||
写入是 best-effort：失败不影响主采集流程（见 collect.py）。
|
||
"""
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="raw_batches")
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="raw_batches", null=True)
|
||
batch_id = models.CharField(max_length=64, blank=True, default="")
|
||
body = models.TextField() # 脱敏后的整包 JSON 文本
|
||
event_count = models.PositiveIntegerField(default=0)
|
||
rejected_count = models.PositiveIntegerField(default=0)
|
||
status = models.CharField(max_length=16, default="accepted") # accepted/partial/rejected
|
||
received_at = models.DateTimeField(auto_now_add=True)
|
||
ip = models.GenericIPAddressField(null=True, blank=True)
|
||
|
||
class Meta:
|
||
db_table = "raw_batch"
|
||
indexes = [models.Index(fields=["visitor", "-received_at"]),
|
||
models.Index(fields=["site", "-received_at"])]
|
||
|
||
|
||
class SourceClassification(models.Model):
|
||
"""来源分类结果（attribution 规则引擎的派生存储）。
|
||
|
||
每访客两行 scope：first_touch（首访来源，session=None）/ session_entry（每次新会话入口）。
|
||
写入两路：ingest 增量（新访客/新会话时 best-effort）+ recompute_attribution 全量重算
|
||
（幂等 upsert）。resolution 表达证据可信度：resolved/combined/conflict/label_only/unknown——
|
||
观察到的 referrer 与 UTM 自述矛盾时并列存 detail，不武断取一边。
|
||
rule_version 关联 constants.ATTRIBUTION_RULE_VERSION：规则升级后可识别存量行需重算。
|
||
"""
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="source_classifications")
|
||
visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="source_classifications", null=True)
|
||
session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="source_classifications", null=True)
|
||
scope = models.CharField(max_length=16) # first_touch / session_entry
|
||
channel = models.CharField(max_length=32, blank=True, default="")
|
||
platform = models.CharField(max_length=64, blank=True, default="")
|
||
variant = models.CharField(max_length=64, blank=True, default="")
|
||
search_term = models.CharField(max_length=200, blank=True, default="")
|
||
evidence_kind = models.CharField(max_length=32, blank=True, default="")
|
||
resolution = models.CharField(max_length=16, blank=True, default="")
|
||
detail = models.JSONField(default=dict, blank=True)
|
||
rule_version = models.CharField(max_length=16, default=C.ATTRIBUTION_RULE_VERSION)
|
||
computed_at = models.DateTimeField(auto_now=True)
|
||
|
||
class Meta:
|
||
db_table = "source_classification"
|
||
indexes = [models.Index(fields=["site", "visitor", "scope"]),
|
||
models.Index(fields=["site", "channel"])]
|
||
|
||
|
||
class DailyStat(models.Model):
|
||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="daily_stats", db_index=False) # uq_site_date 前缀覆盖
|
||
date = models.DateField()
|
||
pv = models.PositiveIntegerField(default=0)
|
||
uv = models.PositiveIntegerField(default=0)
|
||
sessions = models.PositiveIntegerField(default=0)
|
||
new_visitors = models.PositiveIntegerField(default=0)
|
||
|
||
class Meta:
|
||
db_table = "daily_stat"
|
||
constraints = [models.UniqueConstraint(fields=["site", "date"], name="uq_site_date")]
|