diff --git a/server/tracker/management/__init__.py b/server/tracker/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tracker/management/commands/__init__.py b/server/tracker/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tracker/management/commands/refresh_stats.py b/server/tracker/management/commands/refresh_stats.py new file mode 100644 index 0000000..5c9fc2a --- /dev/null +++ b/server/tracker/management/commands/refresh_stats.py @@ -0,0 +1,24 @@ +from datetime import timedelta +from django.core.management.base import BaseCommand +from django.utils import timezone +from tracker.models import Site +from tracker.services.stats import refresh_daily + + +class Command(BaseCommand): + # 生产调度：1Panel 计划任务每日 00:10 执行 + # docker exec python manage.py refresh_stats --days 2 + # （回刷含今天在内的近 N 天，详见 Task 22 README） + help = "回刷近 N 天日聚合（含今天）" + + def add_arguments(self, parser): + parser.add_argument("--days", type=int, default=2) + + def handle(self, *args, **opts): + today = timezone.localdate() + total = 0 + for site in Site.objects.filter(is_active=True): + for i in range(opts["days"]): + stat = refresh_daily(site, today - timedelta(days=i)) + total += 1 + self.stdout.write(f"refreshed {total} site-days") diff --git a/server/tracker/migrations/0002_alter_dailystat_site_alter_event_session_and_more.py b/server/tracker/migrations/0002_alter_dailystat_site_alter_event_session_and_more.py new file mode 100644 index 0000000..81d8fcf --- /dev/null +++ b/server/tracker/migrations/0002_alter_dailystat_site_alter_event_session_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 5.2.17 on 2026-09-15 05:36 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='dailystat', + name='site', + field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='daily_stats', to='tracker.site'), + ), + migrations.AlterField( + model_name='event', + name='session', + field=models.ForeignKey(db_index=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='events', to='tracker.visitsession'), + ), + migrations.AlterField( + model_name='event', + name='site', + field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='events', to='tracker.site'), + ), + migrations.AlterField( + model_name='visitsession', + name='site', + field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='sessions', to='tracker.site'), + ), + migrations.AlterField( + model_name='visitsession', + name='visitor', + field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='sessions', to='tracker.visitor'), + ), + migrations.AddIndex( + model_name='event', + index=models.Index(fields=['site', 'occurred_at'], name='event_site_id_f158ab_idx'), + ), + migrations.AddIndex( + model_name='visitsession', + index=models.Index(fields=['site', 'started_at'], name='visit_sessi_site_id_73fae1_idx'), + ), + ] diff --git a/server/tracker/models.py b/server/tracker/models.py index ee51b25..544930b 100644 --- a/server/tracker/models.py +++ b/server/tracker/models.py @@ -53,8 +53,9 @@ class Visitor(models.Model): class VisitSession(models.Model): - site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="sessions") - visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="sessions") + # db_index=False：单列 FK 索引冗余——site 由 (site, started_at)、visitor 由 (visitor, -started_at) 复合索引前缀覆盖 + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="sessions", db_index=False) + visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="sessions", db_index=False) session_id = models.CharField(max_length=64) started_at = models.DateTimeField() ended_at = models.DateTimeField() @@ -68,13 +69,19 @@ class VisitSession(models.Model): class Meta: db_table = "visit_session" constraints = [models.UniqueConstraint(fields=["site", "session_id"], name="uq_site_session")] - indexes = [models.Index(fields=["visitor", "-started_at"])] + indexes = [ + models.Index(fields=["visitor", "-started_at"]), + models.Index(fields=["site", "started_at"]), # 按日聚合/时间线查询 + ] class Event(models.Model): - site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="events") + # site/session 的单列 FK 索引冗余：site 由 (site, visitor/-occurred_at)、(site, event_name, -occurred_at)、 + # (site, occurred_at) 前缀覆盖，session 由 (session, occurred_at) 前缀覆盖，故 db_index=False。 + # visitor 保留默认索引：无其他以 visitor 打头的索引，级联删除（visitor 反向删 events）需要它。 + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="events", db_index=False) visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="events") - session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="events", null=True) + session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="events", null=True, db_index=False) event_id = models.CharField(max_length=64) event_name = models.CharField(max_length=50) occurred_at = models.DateTimeField() @@ -96,11 +103,12 @@ class Event(models.Model): models.Index(fields=["site", "visitor", "-occurred_at"]), models.Index(fields=["session", "occurred_at"]), models.Index(fields=["site", "event_name", "-occurred_at"]), + models.Index(fields=["site", "occurred_at"]), # refresh_daily 按日聚合扫描 ] class DailyStat(models.Model): - site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="daily_stats") + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="daily_stats", db_index=False) # uq_site_date 前缀覆盖 date = models.DateField() pv = models.PositiveIntegerField(default=0) uv = models.PositiveIntegerField(default=0) diff --git a/server/tracker/services/stats.py b/server/tracker/services/stats.py new file mode 100644 index 0000000..2fd7843 --- /dev/null +++ b/server/tracker/services/stats.py @@ -0,0 +1,26 @@ +from datetime import datetime, time, timedelta +from django.utils import timezone +from tracker.models import DailyStat, Event, VisitSession, Visitor + + +def refresh_daily(site, date) -> DailyStat: + """重算某站点某自然日的聚合（幂等，update_or_create）。 + + 日界取 TIME_ZONE（Asia/Shanghai）自然日：make_aware(datetime.combine(date, time.min))。 + 口径：pv/uv 均从事件表统计（排除 is_bot），uv=distinct visitor，与 pv 同源； + sessions 按会话 started_at 落入当日且访客非爬虫；new_visitors 按 Visitor.first_seen_at 落入当日。 + """ + day_start = timezone.make_aware(datetime.combine(date, time.min)) + day_end = day_start + timedelta(days=1) + pv = Event.objects.filter(site=site, is_bot=False, + occurred_at__gte=day_start, occurred_at__lt=day_end).count() + uv = Event.objects.filter(site=site, is_bot=False, occurred_at__gte=day_start, + occurred_at__lt=day_end).values("visitor").distinct().count() + sessions = VisitSession.objects.filter(site=site, visitor__is_bot=False, + started_at__gte=day_start, + started_at__lt=day_end).count() + new_v = Visitor.objects.filter(site=site, first_seen_at__gte=day_start, + first_seen_at__lt=day_end).count() + obj, _ = DailyStat.objects.update_or_create(site=site, date=date, + defaults={"pv": pv, "uv": uv, "sessions": sessions, "new_visitors": new_v}) + return obj diff --git a/server/tracker/tests/test_models.py b/server/tracker/tests/test_models.py index ecd5777..205bd8f 100644 --- a/server/tracker/tests/test_models.py +++ b/server/tracker/tests/test_models.py @@ -1,4 +1,5 @@ import pytest +from django.db import IntegrityError from tracker.models import Site, Visitor, VisitSession, Event pytestmark = pytest.mark.django_db @@ -22,6 +23,16 @@ def test_event_unique_site_event_id(): started_at="2026-09-15T10:00:00Z", ended_at="2026-09-15T10:01:00Z") Event.objects.create(site=site, visitor=v, session=s, event_id="e1", event_name="page_view", occurred_at="2026-09-15T10:00:00Z", path="/") - with pytest.raises(Exception): + with pytest.raises(IntegrityError): Event.objects.create(site=site, visitor=v, session=s, event_id="e1", event_name="page_view", occurred_at="2026-09-15T10:01:00Z", path="/") + +def test_site_cascade_deletes_children(): + site = Site.objects.create(name="s", domains=["a.com"]) + v = Visitor.objects.create(site=site, visitor_id="v") + s = VisitSession.objects.create(site=site, visitor=v, session_id="s", + started_at="2026-09-15T10:00:00Z", ended_at="2026-09-15T10:01:00Z") + Event.objects.create(site=site, visitor=v, session=s, event_id="e", + event_name="page_view", occurred_at="2026-09-15T10:00:00Z", path="/") + site.delete() + assert Visitor.objects.count() == 0 and VisitSession.objects.count() == 0 and Event.objects.count() == 0 diff --git a/server/tracker/tests/test_stats.py b/server/tracker/tests/test_stats.py new file mode 100644 index 0000000..8ba5bf8 --- /dev/null +++ b/server/tracker/tests/test_stats.py @@ -0,0 +1,30 @@ +from datetime import timedelta +from django.utils import timezone +from tracker.models import Site, Visitor, VisitSession, Event +from tracker.services.stats import refresh_daily + +# 用上海本地时刻做基准：refresh_daily 的日界是上海自然日（make_aware）， +# 若直接用 UTC 的 timezone.now()，在本地 00:00-08:00 之间跑测试时 NOW.date() 会错一天 +NOW = timezone.localtime(timezone.now()) +YESTERDAY = NOW - timedelta(days=1) + +def _mk(site, vid, sid, day, n_events, stay=0): + v, _ = Visitor.objects.get_or_create(site=site, visitor_id=vid) + sess = VisitSession.objects.create(site=site, visitor=v, session_id=sid, + started_at=day, ended_at=day + timedelta(minutes=5), + stay_ms=stay, event_count=n_events) + for i in range(n_events): + Event.objects.create(site=site, visitor=v, session=sess, event_id=f"{vid}-{sid}-{day:%Y%m%d}-{i}", + event_name="page_view", occurred_at=day + timedelta(seconds=i), path="/") + +def test_refresh_daily_two_days(db): + site = Site.objects.create(name="s", domains=["a.com"]) + _mk(site, "v1", "s1", YESTERDAY, 3) # v1 昨天 3 事件 + _mk(site, "v1", "s2", NOW, 2, stay=8000) # v1 今天 2 事件 + _mk(site, "v2", "s3", NOW, 4) # v2 今天 4 事件 + y = refresh_daily(site, YESTERDAY.date()) + t = refresh_daily(site, NOW.date()) + # new_visitors 口径 = first_seen_at 落入当日。测试里 v1 虽有昨日事件，但访客记录是测试运行时（今天）建档的 + # （auto_now_add），故昨天 new=0、今天 new=2（v1+v2）——按 first_seen_at 真实语义断言 + assert (y.pv, y.uv, y.sessions, y.new_visitors) == (3, 1, 1, 0) + assert (t.pv, t.uv, t.sessions, t.new_visitors) == (6, 2, 2, 2) # 今天：pv=6 uv=2(v1+v2) sessions=2 new=2(v1+v2)