fix: 趋势改实时聚合+DailyStat自愈回写，摆脱cron依赖
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -3,8 +3,9 @@
|
||||
?range=today|7d|30d（默认 7d，非法值回落默认）+ ?site=（可选，缺省聚合全部站点）。
|
||||
- 周期卡四指标全口径排除爬虫，与上一个等长周期比较出 delta_pct（today 比昨天）；
|
||||
上期值为 0 时 delta 为 null（前端显示 —）。
|
||||
- 趋势为日粒度：历史位读 DailyStat 聚合（缺日补零），今天一位用 daily_counts 实时值
|
||||
（聚合作业未跑也不缺当日）——与 refresh_daily 共享口径（Task8 审查结论）。
|
||||
- 趋势为日粒度：窗口内 Event/VisitSession 按上海自然日实时聚合（缺日补零，今天一位
|
||||
天然实时），不依赖定时聚合作业；口径与 refresh_daily/daily_counts 一致，且对有数据的
|
||||
日顺手自愈回写 DailyStat（best-effort）——DailyStat 保留给历史归档消费者。
|
||||
- channels 按窗口内会话入口分类聚合，pct = 占已分类会话比（列表含 direct/unknown，
|
||||
总和 ≈ 1；未重算出分类行的历史会话不计入）。
|
||||
- 最近访问不在本接口做：前端调既有 /api/admin/visitors?page=1 取前 5 行。
|
||||
@@ -12,13 +13,13 @@
|
||||
from datetime import datetime, time, timedelta
|
||||
|
||||
from django.db.models import Count, Sum
|
||||
from django.db.models.functions import Trunc
|
||||
from django.utils import timezone
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from tracker.api.envelope import EnvelopeMixin, ok
|
||||
from tracker.models import DailyStat, Event, Site, SourceClassification, VisitSession, Visitor
|
||||
from tracker.services.stats import daily_counts
|
||||
|
||||
RANGE_DAYS = {"today": 1, "7d": 7, "30d": 30}
|
||||
CARD_KEYS = ("visitors", "visits", "avg_stay_ms", "returning")
|
||||
@@ -97,25 +98,63 @@ class OverviewView(EnvelopeMixin, APIView):
|
||||
return q.count()
|
||||
|
||||
def _trend(self, site, today, days):
|
||||
"""窗口每日 [{date, uv, sessions}]：历史位读 DailyStat（缺日补零），
|
||||
今天一位用 daily_counts 实时值覆盖——聚合作业未跑也不缺当日。"""
|
||||
start = today - timedelta(days=days - 1)
|
||||
qs = DailyStat.objects.filter(date__gte=start, date__lte=today)
|
||||
"""窗口每日 [{date, uv, sessions}]：对窗口内 Event/VisitSession 按上海自然日
|
||||
实时聚合（缺日补零，今天一位天然实时），不依赖定时聚合作业——DailyStat 为空
|
||||
时历史位也是真实值。四指标口径与 daily_counts 一致；聚合出的按站按日数据顺手
|
||||
自愈回写 DailyStat（best-effort，失败不影响响应；多站点按站拆行互不干扰），
|
||||
DailyStat 保留给历史归档消费者。
|
||||
性能：窗口 ≤30 天，走 (site,occurred_at)/(site,started_at) 索引的按日聚合，
|
||||
当前量级毫秒级。"""
|
||||
tz = timezone.get_current_timezone()
|
||||
start = timezone.make_aware(
|
||||
datetime.combine(today - timedelta(days=days - 1), time.min))
|
||||
end = timezone.make_aware(
|
||||
datetime.combine(today + timedelta(days=1), time.min))
|
||||
ev = Event.objects.filter(is_bot=False,
|
||||
occurred_at__gte=start, occurred_at__lt=end)
|
||||
vs = VisitSession.objects.filter(visitor__is_bot=False,
|
||||
started_at__gte=start, started_at__lt=end)
|
||||
vis = Visitor.objects.filter(is_bot=False,
|
||||
first_seen_at__gte=start, first_seen_at__lt=end)
|
||||
if site is not None:
|
||||
qs = qs.filter(site=site)
|
||||
rows = {r["date"]: r for r in qs.values("date").annotate(
|
||||
uv=Sum("uv"), sessions=Sum("sessions"))}
|
||||
ev = ev.filter(site=site)
|
||||
vs = vs.filter(site=site)
|
||||
vis = vis.filter(site=site)
|
||||
# 按站按日 Trunc 聚合（site=None 也拆站行：回写 DailyStat 需要站点维度；
|
||||
# visitor/session 天然属单站，跨站求和 = 全局聚合值）
|
||||
rows = {} # (site_id, date) -> {pv, uv, sessions, new_visitors}，仅有数据的日
|
||||
|
||||
def _row(sid, day):
|
||||
return rows.setdefault(
|
||||
(sid, day), {"pv": 0, "uv": 0, "sessions": 0, "new_visitors": 0})
|
||||
|
||||
for r in (ev.annotate(day=Trunc("occurred_at", "day", tzinfo=tz))
|
||||
.values("site", "day")
|
||||
.annotate(pv=Count("id"), uv=Count("visitor", distinct=True))):
|
||||
row = _row(r["site"], r["day"].date())
|
||||
row["pv"], row["uv"] = r["pv"], r["uv"]
|
||||
for r in (vs.annotate(day=Trunc("started_at", "day", tzinfo=tz))
|
||||
.values("site", "day")
|
||||
.annotate(n=Count("id"))):
|
||||
_row(r["site"], r["day"].date())["sessions"] = r["n"]
|
||||
for r in (vis.annotate(day=Trunc("first_seen_at", "day", tzinfo=tz))
|
||||
.values("site", "day")
|
||||
.annotate(n=Count("id"))):
|
||||
_row(r["site"], r["day"].date())["new_visitors"] = r["n"]
|
||||
# 自愈回写：仅窗口内有数据的日（空窗口不产生行），失败不影响趋势响应
|
||||
for (sid, date), v in rows.items():
|
||||
try:
|
||||
DailyStat.objects.update_or_create(site_id=sid, date=date, defaults=v)
|
||||
except Exception: # best-effort：回写失败不影响响应
|
||||
pass
|
||||
out = []
|
||||
first = today - timedelta(days=days - 1)
|
||||
for i in range(days):
|
||||
date = start + timedelta(days=i)
|
||||
if date == today:
|
||||
counts = daily_counts(site, today)
|
||||
out.append({"date": date.isoformat(),
|
||||
"uv": counts["uv"], "sessions": counts["sessions"]})
|
||||
else:
|
||||
r = rows.get(date, {})
|
||||
out.append({"date": date.isoformat(),
|
||||
"uv": r.get("uv") or 0, "sessions": r.get("sessions") or 0})
|
||||
date = first + timedelta(days=i)
|
||||
day_rows = [v for (_, d), v in rows.items() if d == date]
|
||||
out.append({"date": date.isoformat(),
|
||||
"uv": sum(r["uv"] for r in day_rows),
|
||||
"sessions": sum(r["sessions"] for r in day_rows)})
|
||||
return out
|
||||
|
||||
def _channels(self, site, start, end):
|
||||
|
||||
@@ -3,8 +3,7 @@ from datetime import timedelta
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.utils import timezone
|
||||
from rest_framework.test import APIClient
|
||||
from tracker.models import Event, Site, SourceClassification, VisitSession, Visitor
|
||||
from tracker.services.stats import refresh_daily
|
||||
from tracker.models import DailyStat, Event, Site, SourceClassification, VisitSession, Visitor
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
User = get_user_model()
|
||||
@@ -103,11 +102,13 @@ def test_overview_7d_cards_and_deltas(client, member_user):
|
||||
assert d["cards"]["returning"] == {"value": 1, "delta_pct": None} # 上期 0 → null
|
||||
# 在线：v1/v2/v3 活跃（bot 排除、v4 十天前不在线）
|
||||
assert d["header"] == {"online": 3, "active_now": 3} # active_now 兼容保留
|
||||
# 趋势：7 点日粒度，历史无 DailyStat 补零，今天一位用实时值（uv=2、sessions=2，bot 排除）
|
||||
# 趋势：7 点日粒度实时聚合，无数据的日补零——3 天前 v3 回访有真实值
|
||||
# （uv=1、sessions=1，bot 排除），不依赖 DailyStat/聚合作业
|
||||
trend = d["trend"]
|
||||
assert len(trend) == 7
|
||||
assert trend[0]["date"] == (TODAY - timedelta(days=6)).isoformat()
|
||||
assert trend[-1] == {"date": TODAY.isoformat(), "uv": 2, "sessions": 2}
|
||||
assert trend[-4] == {"date": (TODAY - timedelta(days=3)).isoformat(), "uv": 1, "sessions": 1}
|
||||
assert trend[-2] == {"date": YESTERDAY.isoformat(), "uv": 0, "sessions": 0}
|
||||
assert d["channels"] == [] # 无分类行 → 空列表
|
||||
|
||||
@@ -183,26 +184,52 @@ def test_overview_range_today(client, member_user):
|
||||
assert len(d["trend"]) == 30
|
||||
|
||||
|
||||
def test_overview_trend_uses_dailystat_history(client, member_user):
|
||||
"""7d 趋势历史位读 DailyStat 聚合（跨站求和），今天位用实时值。"""
|
||||
def test_overview_trend_realtime_and_selfheal(client, member_user):
|
||||
"""趋势不依赖 DailyStat/聚合作业：昨天+今天的数据（不跑 refresh_daily）
|
||||
两天都有真实值（昨天跨站求和）；调用后 DailyStat 自愈出现对应行（update_or_create 幂等）。"""
|
||||
_auth(client, member_user)
|
||||
site_a = Site.objects.create(name="a", domains=["a.com"])
|
||||
site_b = Site.objects.create(name="b", domains=["b.com"])
|
||||
yn = NOW - timedelta(days=1) # 昨天同一本地时刻，必落上海昨日窗口
|
||||
va = Visitor.objects.create(site=site_a, visitor_id="va")
|
||||
va = _visitor(site_a, "va", last_seen=yn, first_seen=yn)
|
||||
sa = _session(site_a, va, "sa", at=yn)
|
||||
_events(site_a, va, sa, 3, "/", at=yn, prefix="ya")
|
||||
refresh_daily(site_a, YESTERDAY)
|
||||
vb = Visitor.objects.create(site=site_b, visitor_id="vb")
|
||||
vb = _visitor(site_b, "vb", last_seen=yn, first_seen=yn)
|
||||
sb = _session(site_b, vb, "sb", at=yn)
|
||||
_events(site_b, vb, sb, 4, "/", at=yn, prefix="yb")
|
||||
refresh_daily(site_b, YESTERDAY)
|
||||
# 今天：site_b 一个新访客一条会话两条事件
|
||||
vt = _visitor(site_b, "vt", last_seen=timezone.now())
|
||||
st = _session(site_b, vt, "st")
|
||||
_events(site_b, vt, st, 2, "/t", prefix="t")
|
||||
assert DailyStat.objects.count() == 0 # 前置：聚合作业从未跑过
|
||||
|
||||
d = client.get("/api/admin/dashboard/overview").json()["data"]
|
||||
trend = d["trend"]
|
||||
assert len(trend) == 7
|
||||
assert trend[-2] == {"date": YESTERDAY.isoformat(), "uv": 2, "sessions": 2} # 跨站求和
|
||||
assert trend[-1] == {"date": TODAY.isoformat(), "uv": 0, "sessions": 0} # 今天无事件
|
||||
assert trend[-1] == {"date": TODAY.isoformat(), "uv": 1, "sessions": 1}
|
||||
# 自愈回写：按站按日出现三行，四指标口径与 daily_counts 一致
|
||||
ra = DailyStat.objects.get(site=site_a, date=YESTERDAY)
|
||||
assert (ra.pv, ra.uv, ra.sessions, ra.new_visitors) == (3, 1, 1, 1)
|
||||
rb = DailyStat.objects.get(site=site_b, date=YESTERDAY)
|
||||
assert (rb.pv, rb.uv, rb.sessions, rb.new_visitors) == (4, 1, 1, 1)
|
||||
rt = DailyStat.objects.get(site=site_b, date=TODAY)
|
||||
assert (rt.pv, rt.uv, rt.sessions, rt.new_visitors) == (2, 1, 1, 1)
|
||||
assert DailyStat.objects.count() == 3
|
||||
client.get("/api/admin/dashboard/overview") # 再调一次：update 不重复建行
|
||||
assert DailyStat.objects.count() == 3
|
||||
|
||||
|
||||
def test_overview_trend_empty_window_no_write(client, member_user):
|
||||
"""空窗口：趋势为全零序列，且不写任何 DailyStat（自愈只写有数据的日）。"""
|
||||
_auth(client, member_user)
|
||||
Site.objects.create(name="s", domains=["a.com"])
|
||||
d = client.get("/api/admin/dashboard/overview").json()["data"]
|
||||
trend = d["trend"]
|
||||
assert len(trend) == 7
|
||||
assert trend[0]["date"] == (TODAY - timedelta(days=6)).isoformat()
|
||||
assert all(p["uv"] == 0 and p["sessions"] == 0 for p in trend)
|
||||
assert DailyStat.objects.count() == 0
|
||||
|
||||
|
||||
def test_overview_site_filter_and_404(client, member_user):
|
||||
@@ -217,10 +244,9 @@ def test_overview_site_filter_and_404(client, member_user):
|
||||
_events(site_b, vb, sb, 5, "/b", prefix="b")
|
||||
SourceClassification.objects.create(site=site_b, visitor=vb, session=sb,
|
||||
scope="session_entry", channel="direct", resolution="unknown")
|
||||
# a 站昨天有 DailyStat 行——过滤 site_b 时不得混入
|
||||
# a 站昨天有数据（实时聚合）——过滤 site_b 时不得混入
|
||||
yn = NOW - timedelta(days=1)
|
||||
_events(site_a, va, sa, 3, "/", at=yn, prefix="ay")
|
||||
refresh_daily(site_a, YESTERDAY)
|
||||
|
||||
d = client.get("/api/admin/dashboard/overview").json()["data"]
|
||||
assert d["cards"]["visitors"]["value"] == 2 # 缺省聚合全部站点
|
||||
|
||||
Reference in New Issue
Block a user