fix: 趋势改实时聚合+DailyStat自愈回写,摆脱cron依赖

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-09-17 14:17:08 +08:00
parent 8f20d8d9a2
commit 35f7b55946
2 changed files with 96 additions and 31 deletions
+58 -19
View File
@@ -3,8 +3,9 @@
?range=today|7d|30d(默认 7d,非法值回落默认)+ ?site=(可选,缺省聚合全部站点)。
- 周期卡四指标全口径排除爬虫,与上一个等长周期比较出 delta_pct(today 比昨天);
上期值为 0 时 delta 为 null(前端显示 —)。
- 趋势为日粒度:历史位读 DailyStat 聚合(缺日补零),今天一位用 daily_counts 实时值
(聚合作业未跑也不缺当日)——与 refresh_daily 共享口径(Task8 审查结论)。
- 趋势为日粒度:窗口内 Event/VisitSession 按上海自然日实时聚合(缺日补零,今天一位
天然实时),不依赖定时聚合作业;口径与 refresh_daily/daily_counts 一致,且对有数据的
日顺手自愈回写 DailyStat(best-effort)——DailyStat 保留给历史归档消费者。
- channels 按窗口内会话入口分类聚合,pct = 占已分类会话比(列表含 direct/unknown,
总和 ≈ 1;未重算出分类行的历史会话不计入)。
- 最近访问不在本接口做:前端调既有 /api/admin/visitors?page=1 取前 5 行。
@@ -12,13 +13,13 @@
from datetime import datetime, time, timedelta
from django.db.models import Count, Sum
from django.db.models.functions import Trunc
from django.utils import timezone
from rest_framework.response import Response
from rest_framework.views import APIView
from tracker.api.envelope import EnvelopeMixin, ok
from tracker.models import DailyStat, Event, Site, SourceClassification, VisitSession, Visitor
from tracker.services.stats import daily_counts
RANGE_DAYS = {"today": 1, "7d": 7, "30d": 30}
CARD_KEYS = ("visitors", "visits", "avg_stay_ms", "returning")
@@ -97,25 +98,63 @@ class OverviewView(EnvelopeMixin, APIView):
return q.count()
def _trend(self, site, today, days):
"""窗口每日 [{date, uv, sessions}]:历史位读 DailyStat(缺日补零),
今天一位用 daily_counts 实时值覆盖——聚合作业未跑也不缺当日。"""
start = today - timedelta(days=days - 1)
qs = DailyStat.objects.filter(date__gte=start, date__lte=today)
"""窗口每日 [{date, uv, sessions}]:对窗口内 Event/VisitSession 按上海自然日
实时聚合(缺日补零,今天一位天然实时),不依赖定时聚合作业——DailyStat 为空
时历史位也是真实值。四指标口径与 daily_counts 一致;聚合出的按站按日数据顺手
自愈回写 DailyStat(best-effort,失败不影响响应;多站点按站拆行互不干扰),
DailyStat 保留给历史归档消费者。
性能:窗口 ≤30 天,走 (site,occurred_at)/(site,started_at) 索引的按日聚合,
当前量级毫秒级。"""
tz = timezone.get_current_timezone()
start = timezone.make_aware(
datetime.combine(today - timedelta(days=days - 1), time.min))
end = timezone.make_aware(
datetime.combine(today + timedelta(days=1), time.min))
ev = Event.objects.filter(is_bot=False,
occurred_at__gte=start, occurred_at__lt=end)
vs = VisitSession.objects.filter(visitor__is_bot=False,
started_at__gte=start, started_at__lt=end)
vis = Visitor.objects.filter(is_bot=False,
first_seen_at__gte=start, first_seen_at__lt=end)
if site is not None:
qs = qs.filter(site=site)
rows = {r["date"]: r for r in qs.values("date").annotate(
uv=Sum("uv"), sessions=Sum("sessions"))}
ev = ev.filter(site=site)
vs = vs.filter(site=site)
vis = vis.filter(site=site)
# 按站按日 Trunc 聚合(site=None 也拆站行:回写 DailyStat 需要站点维度;
# visitor/session 天然属单站,跨站求和 = 全局聚合值)
rows = {} # (site_id, date) -> {pv, uv, sessions, new_visitors},仅有数据的日
def _row(sid, day):
return rows.setdefault(
(sid, day), {"pv": 0, "uv": 0, "sessions": 0, "new_visitors": 0})
for r in (ev.annotate(day=Trunc("occurred_at", "day", tzinfo=tz))
.values("site", "day")
.annotate(pv=Count("id"), uv=Count("visitor", distinct=True))):
row = _row(r["site"], r["day"].date())
row["pv"], row["uv"] = r["pv"], r["uv"]
for r in (vs.annotate(day=Trunc("started_at", "day", tzinfo=tz))
.values("site", "day")
.annotate(n=Count("id"))):
_row(r["site"], r["day"].date())["sessions"] = r["n"]
for r in (vis.annotate(day=Trunc("first_seen_at", "day", tzinfo=tz))
.values("site", "day")
.annotate(n=Count("id"))):
_row(r["site"], r["day"].date())["new_visitors"] = r["n"]
# 自愈回写:仅窗口内有数据的日(空窗口不产生行),失败不影响趋势响应
for (sid, date), v in rows.items():
try:
DailyStat.objects.update_or_create(site_id=sid, date=date, defaults=v)
except Exception: # best-effort:回写失败不影响响应
pass
out = []
first = today - timedelta(days=days - 1)
for i in range(days):
date = start + timedelta(days=i)
if date == today:
counts = daily_counts(site, today)
out.append({"date": date.isoformat(),
"uv": counts["uv"], "sessions": counts["sessions"]})
else:
r = rows.get(date, {})
out.append({"date": date.isoformat(),
"uv": r.get("uv") or 0, "sessions": r.get("sessions") or 0})
date = first + timedelta(days=i)
day_rows = [v for (_, d), v in rows.items() if d == date]
out.append({"date": date.isoformat(),
"uv": sum(r["uv"] for r in day_rows),
"sessions": sum(r["sessions"] for r in day_rows)})
return out
def _channels(self, site, start, end):