From 35f7b559469a8acdb88e4f6c9f2fe121f9302d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Thu, 17 Sep 2026 14:17:08 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E8=B6=8B=E5=8A=BF=E6=94=B9=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E8=81=9A=E5=90=88+DailyStat=E8=87=AA=E6=84=88?= =?UTF-8?q?=E5=9B=9E=E5=86=99=EF=BC=8C=E6=91=86=E8=84=B1cron=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Code --- server/tracker/api/dashboard.py | 77 +++++++++++++++++++------- server/tracker/tests/test_dashboard.py | 50 +++++++++++++---- 2 files changed, 96 insertions(+), 31 deletions(-) diff --git a/server/tracker/api/dashboard.py b/server/tracker/api/dashboard.py index 0684129..5b0a49f 100644 --- a/server/tracker/api/dashboard.py +++ b/server/tracker/api/dashboard.py @@ -3,8 +3,9 @@ ?range=today|7d|30d（默认 7d，非法值回落默认）+ ?site=（可选，缺省聚合全部站点）。 - 周期卡四指标全口径排除爬虫，与上一个等长周期比较出 delta_pct（today 比昨天）； 上期值为 0 时 delta 为 null（前端显示 —）。 -- 趋势为日粒度：历史位读 DailyStat 聚合（缺日补零），今天一位用 daily_counts 实时值 - （聚合作业未跑也不缺当日）——与 refresh_daily 共享口径（Task8 审查结论）。 +- 趋势为日粒度：窗口内 Event/VisitSession 按上海自然日实时聚合（缺日补零，今天一位 + 天然实时），不依赖定时聚合作业；口径与 refresh_daily/daily_counts 一致，且对有数据的 + 日顺手自愈回写 DailyStat（best-effort）——DailyStat 保留给历史归档消费者。 - channels 按窗口内会话入口分类聚合，pct = 占已分类会话比（列表含 direct/unknown， 总和 ≈ 1；未重算出分类行的历史会话不计入）。 - 最近访问不在本接口做：前端调既有 /api/admin/visitors?page=1 取前 5 行。 @@ -12,13 +13,13 @@ from datetime import datetime, time, timedelta from django.db.models import Count, Sum +from django.db.models.functions import Trunc from django.utils import timezone from rest_framework.response import Response from rest_framework.views import APIView from tracker.api.envelope import EnvelopeMixin, ok from tracker.models import DailyStat, Event, Site, SourceClassification, VisitSession, Visitor -from tracker.services.stats import daily_counts RANGE_DAYS = {"today": 1, "7d": 7, "30d": 30} CARD_KEYS = ("visitors", "visits", "avg_stay_ms", "returning") @@ -97,25 +98,63 @@ class OverviewView(EnvelopeMixin, APIView): return q.count() def _trend(self, site, today, days): - """窗口每日 [{date, uv, sessions}]：历史位读 DailyStat（缺日补零）， - 今天一位用 daily_counts 实时值覆盖——聚合作业未跑也不缺当日。""" - start = today - timedelta(days=days - 1) - qs = DailyStat.objects.filter(date__gte=start, date__lte=today) + """窗口每日 [{date, uv, sessions}]：对窗口内 Event/VisitSession 按上海自然日 + 实时聚合（缺日补零，今天一位天然实时），不依赖定时聚合作业——DailyStat 为空 + 时历史位也是真实值。四指标口径与 daily_counts 一致；聚合出的按站按日数据顺手 + 自愈回写 DailyStat（best-effort，失败不影响响应；多站点按站拆行互不干扰）， + DailyStat 保留给历史归档消费者。 + 性能：窗口 ≤30 天，走 (site,occurred_at)/(site,started_at) 索引的按日聚合， + 当前量级毫秒级。""" + tz = timezone.get_current_timezone() + start = timezone.make_aware( + datetime.combine(today - timedelta(days=days - 1), time.min)) + end = timezone.make_aware( + datetime.combine(today + timedelta(days=1), time.min)) + ev = Event.objects.filter(is_bot=False, + occurred_at__gte=start, occurred_at__lt=end) + vs = VisitSession.objects.filter(visitor__is_bot=False, + started_at__gte=start, started_at__lt=end) + vis = Visitor.objects.filter(is_bot=False, + first_seen_at__gte=start, first_seen_at__lt=end) if site is not None: - qs = qs.filter(site=site) - rows = {r["date"]: r for r in qs.values("date").annotate( - uv=Sum("uv"), sessions=Sum("sessions"))} + ev = ev.filter(site=site) + vs = vs.filter(site=site) + vis = vis.filter(site=site) + # 按站按日 Trunc 聚合（site=None 也拆站行：回写 DailyStat 需要站点维度； + # visitor/session 天然属单站，跨站求和 = 全局聚合值） + rows = {} # (site_id, date) -> {pv, uv, sessions, new_visitors}，仅有数据的日 + + def _row(sid, day): + return rows.setdefault( + (sid, day), {"pv": 0, "uv": 0, "sessions": 0, "new_visitors": 0}) + + for r in (ev.annotate(day=Trunc("occurred_at", "day", tzinfo=tz)) + .values("site", "day") + .annotate(pv=Count("id"), uv=Count("visitor", distinct=True))): + row = _row(r["site"], r["day"].date()) + row["pv"], row["uv"] = r["pv"], r["uv"] + for r in (vs.annotate(day=Trunc("started_at", "day", tzinfo=tz)) + .values("site", "day") + .annotate(n=Count("id"))): + _row(r["site"], r["day"].date())["sessions"] = r["n"] + for r in (vis.annotate(day=Trunc("first_seen_at", "day", tzinfo=tz)) + .values("site", "day") + .annotate(n=Count("id"))): + _row(r["site"], r["day"].date())["new_visitors"] = r["n"] + # 自愈回写：仅窗口内有数据的日（空窗口不产生行），失败不影响趋势响应 + for (sid, date), v in rows.items(): + try: + DailyStat.objects.update_or_create(site_id=sid, date=date, defaults=v) + except Exception: # best-effort：回写失败不影响响应 + pass out = [] + first = today - timedelta(days=days - 1) for i in range(days): - date = start + timedelta(days=i) - if date == today: - counts = daily_counts(site, today) - out.append({"date": date.isoformat(), - "uv": counts["uv"], "sessions": counts["sessions"]}) - else: - r = rows.get(date, {}) - out.append({"date": date.isoformat(), - "uv": r.get("uv") or 0, "sessions": r.get("sessions") or 0}) + date = first + timedelta(days=i) + day_rows = [v for (_, d), v in rows.items() if d == date] + out.append({"date": date.isoformat(), + "uv": sum(r["uv"] for r in day_rows), + "sessions": sum(r["sessions"] for r in day_rows)}) return out def _channels(self, site, start, end): diff --git a/server/tracker/tests/test_dashboard.py b/server/tracker/tests/test_dashboard.py index 9b8163c..07ca27c 100644 --- a/server/tracker/tests/test_dashboard.py +++ b/server/tracker/tests/test_dashboard.py @@ -3,8 +3,7 @@ from datetime import timedelta from django.contrib.auth import get_user_model from django.utils import timezone from rest_framework.test import APIClient -from tracker.models import Event, Site, SourceClassification, VisitSession, Visitor -from tracker.services.stats import refresh_daily +from tracker.models import DailyStat, Event, Site, SourceClassification, VisitSession, Visitor pytestmark = pytest.mark.django_db User = get_user_model() @@ -103,11 +102,13 @@ def test_overview_7d_cards_and_deltas(client, member_user): assert d["cards"]["returning"] == {"value": 1, "delta_pct": None} # 上期 0 → null # 在线：v1/v2/v3 活跃（bot 排除、v4 十天前不在线） assert d["header"] == {"online": 3, "active_now": 3} # active_now 兼容保留 - # 趋势：7 点日粒度，历史无 DailyStat 补零，今天一位用实时值（uv=2、sessions=2，bot 排除） + # 趋势：7 点日粒度实时聚合，无数据的日补零——3 天前 v3 回访有真实值 + # （uv=1、sessions=1，bot 排除），不依赖 DailyStat/聚合作业 trend = d["trend"] assert len(trend) == 7 assert trend[0]["date"] == (TODAY - timedelta(days=6)).isoformat() assert trend[-1] == {"date": TODAY.isoformat(), "uv": 2, "sessions": 2} + assert trend[-4] == {"date": (TODAY - timedelta(days=3)).isoformat(), "uv": 1, "sessions": 1} assert trend[-2] == {"date": YESTERDAY.isoformat(), "uv": 0, "sessions": 0} assert d["channels"] == [] # 无分类行 → 空列表 @@ -183,26 +184,52 @@ def test_overview_range_today(client, member_user): assert len(d["trend"]) == 30 -def test_overview_trend_uses_dailystat_history(client, member_user): - """7d 趋势历史位读 DailyStat 聚合（跨站求和），今天位用实时值。""" +def test_overview_trend_realtime_and_selfheal(client, member_user): + """趋势不依赖 DailyStat/聚合作业：昨天+今天的数据（不跑 refresh_daily） + 两天都有真实值（昨天跨站求和）；调用后 DailyStat 自愈出现对应行（update_or_create 幂等）。""" _auth(client, member_user) site_a = Site.objects.create(name="a", domains=["a.com"]) site_b = Site.objects.create(name="b", domains=["b.com"]) yn = NOW - timedelta(days=1) # 昨天同一本地时刻，必落上海昨日窗口 - va = Visitor.objects.create(site=site_a, visitor_id="va") + va = _visitor(site_a, "va", last_seen=yn, first_seen=yn) sa = _session(site_a, va, "sa", at=yn) _events(site_a, va, sa, 3, "/", at=yn, prefix="ya") - refresh_daily(site_a, YESTERDAY) - vb = Visitor.objects.create(site=site_b, visitor_id="vb") + vb = _visitor(site_b, "vb", last_seen=yn, first_seen=yn) sb = _session(site_b, vb, "sb", at=yn) _events(site_b, vb, sb, 4, "/", at=yn, prefix="yb") - refresh_daily(site_b, YESTERDAY) + # 今天：site_b 一个新访客一条会话两条事件 + vt = _visitor(site_b, "vt", last_seen=timezone.now()) + st = _session(site_b, vt, "st") + _events(site_b, vt, st, 2, "/t", prefix="t") + assert DailyStat.objects.count() == 0 # 前置：聚合作业从未跑过 d = client.get("/api/admin/dashboard/overview").json()["data"] trend = d["trend"] assert len(trend) == 7 assert trend[-2] == {"date": YESTERDAY.isoformat(), "uv": 2, "sessions": 2} # 跨站求和 - assert trend[-1] == {"date": TODAY.isoformat(), "uv": 0, "sessions": 0} # 今天无事件 + assert trend[-1] == {"date": TODAY.isoformat(), "uv": 1, "sessions": 1} + # 自愈回写：按站按日出现三行，四指标口径与 daily_counts 一致 + ra = DailyStat.objects.get(site=site_a, date=YESTERDAY) + assert (ra.pv, ra.uv, ra.sessions, ra.new_visitors) == (3, 1, 1, 1) + rb = DailyStat.objects.get(site=site_b, date=YESTERDAY) + assert (rb.pv, rb.uv, rb.sessions, rb.new_visitors) == (4, 1, 1, 1) + rt = DailyStat.objects.get(site=site_b, date=TODAY) + assert (rt.pv, rt.uv, rt.sessions, rt.new_visitors) == (2, 1, 1, 1) + assert DailyStat.objects.count() == 3 + client.get("/api/admin/dashboard/overview") # 再调一次：update 不重复建行 + assert DailyStat.objects.count() == 3 + + +def test_overview_trend_empty_window_no_write(client, member_user): + """空窗口：趋势为全零序列，且不写任何 DailyStat（自愈只写有数据的日）。""" + _auth(client, member_user) + Site.objects.create(name="s", domains=["a.com"]) + d = client.get("/api/admin/dashboard/overview").json()["data"] + trend = d["trend"] + assert len(trend) == 7 + assert trend[0]["date"] == (TODAY - timedelta(days=6)).isoformat() + assert all(p["uv"] == 0 and p["sessions"] == 0 for p in trend) + assert DailyStat.objects.count() == 0 def test_overview_site_filter_and_404(client, member_user): @@ -217,10 +244,9 @@ def test_overview_site_filter_and_404(client, member_user): _events(site_b, vb, sb, 5, "/b", prefix="b") SourceClassification.objects.create(site=site_b, visitor=vb, session=sb, scope="session_entry", channel="direct", resolution="unknown") - # a 站昨天有 DailyStat 行——过滤 site_b 时不得混入 + # a 站昨天有数据（实时聚合）——过滤 site_b 时不得混入 yn = NOW - timedelta(days=1) _events(site_a, va, sa, 3, "/", at=yn, prefix="ay") - refresh_daily(site_a, YESTERDAY) d = client.get("/api/admin/dashboard/overview").json()["data"] assert d["cards"]["visitors"]["value"] == 2 # 缺省聚合全部站点