fix: new_visitors 排爬虫+日界/爬虫口径回归测试

- refresh_daily new_visitors 加 is_bot=False(爬虫不是受众,四指标口径统一)
- test_refresh_daily_shanghai_boundary:固定 UTC 时间戳钉死上海自然日界
- test_refresh_daily_excludes_bots:事件/会话/访客 is_bot 均不计入
- refresh_stats 删未使用的 stat = 赋值

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
2026-09-15 13:55:13 +08:00
parent 79a26c77f3
commit 77a7f568b2
3 changed files with 34 additions and 4 deletions
@@ -19,6 +19,6 @@ class Command(BaseCommand):
total = 0 total = 0
for site in Site.objects.filter(is_active=True): for site in Site.objects.filter(is_active=True):
for i in range(opts["days"]): for i in range(opts["days"]):
stat = refresh_daily(site, today - timedelta(days=i)) refresh_daily(site, today - timedelta(days=i))
total += 1 total += 1
self.stdout.write(f"refreshed {total} site-days") self.stdout.write(f"refreshed {total} site-days")
+2 -2
View File
@@ -8,7 +8,7 @@ def refresh_daily(site, date) -> DailyStat:
日界取 TIME_ZONE(Asia/Shanghai)自然日:make_aware(datetime.combine(date, time.min))。 日界取 TIME_ZONE(Asia/Shanghai)自然日:make_aware(datetime.combine(date, time.min))。
口径:pv/uv 均从事件表统计(排除 is_bot),uv=distinct visitor,与 pv 同源; 口径:pv/uv 均从事件表统计(排除 is_bot),uv=distinct visitor,与 pv 同源;
sessions 按会话 started_at 落入当日且访客非爬虫;new_visitors 按 Visitor.first_seen_at 落入当日。 sessions 按会话 started_at 落入当日且访客非爬虫;new_visitors 按 Visitor.first_seen_at 落入当日(同样排除爬虫访客,四指标口径统一)。
""" """
day_start = timezone.make_aware(datetime.combine(date, time.min)) day_start = timezone.make_aware(datetime.combine(date, time.min))
day_end = day_start + timedelta(days=1) day_end = day_start + timedelta(days=1)
@@ -19,7 +19,7 @@ def refresh_daily(site, date) -> DailyStat:
sessions = VisitSession.objects.filter(site=site, visitor__is_bot=False, sessions = VisitSession.objects.filter(site=site, visitor__is_bot=False,
started_at__gte=day_start, started_at__gte=day_start,
started_at__lt=day_end).count() started_at__lt=day_end).count()
new_v = Visitor.objects.filter(site=site, first_seen_at__gte=day_start, new_v = Visitor.objects.filter(site=site, is_bot=False, first_seen_at__gte=day_start,
first_seen_at__lt=day_end).count() first_seen_at__lt=day_end).count()
obj, _ = DailyStat.objects.update_or_create(site=site, date=date, obj, _ = DailyStat.objects.update_or_create(site=site, date=date,
defaults={"pv": pv, "uv": uv, "sessions": sessions, "new_visitors": new_v}) defaults={"pv": pv, "uv": uv, "sessions": sessions, "new_visitors": new_v})
+31 -1
View File
@@ -1,4 +1,4 @@
from datetime import timedelta from datetime import date, timedelta
from django.utils import timezone from django.utils import timezone
from tracker.models import Site, Visitor, VisitSession, Event from tracker.models import Site, Visitor, VisitSession, Event
from tracker.services.stats import refresh_daily from tracker.services.stats import refresh_daily
@@ -28,3 +28,33 @@ def test_refresh_daily_two_days(db):
# (auto_now_add),故昨天 new=0、今天 new=2(v1+v2)——按 first_seen_at 真实语义断言 # (auto_now_add),故昨天 new=0、今天 new=2(v1+v2)——按 first_seen_at 真实语义断言
assert (y.pv, y.uv, y.sessions, y.new_visitors) == (3, 1, 1, 0) assert (y.pv, y.uv, y.sessions, y.new_visitors) == (3, 1, 1, 0)
assert (t.pv, t.uv, t.sessions, t.new_visitors) == (6, 2, 2, 2) # 今天:pv=6 uv=2(v1+v2) sessions=2 new=2(v1+v2) assert (t.pv, t.uv, t.sessions, t.new_visitors) == (6, 2, 2, 2) # 今天:pv=6 uv=2(v1+v2) sessions=2 new=2(v1+v2)
def test_refresh_daily_shanghai_boundary(db):
# 固定时间戳钉死日界(不依赖运行时刻):UTC 2026-09-15T16:30 = 上海 09-16 00:30 → 落 9-16 不落 9-15
site = Site.objects.create(name="s", domains=["a.com"])
v = Visitor.objects.create(site=site, visitor_id="v")
s = VisitSession.objects.create(site=site, visitor=v, session_id="s",
started_at="2026-09-15T16:30:00Z", ended_at="2026-09-15T16:30:00Z")
Event.objects.create(site=site, visitor=v, session=s, event_id="e",
event_name="page_view", occurred_at="2026-09-15T16:30:00Z", path="/")
d15 = refresh_daily(site, date(2026, 9, 15))
d16 = refresh_daily(site, date(2026, 9, 16))
assert (d15.pv, d16.pv) == (0, 1)
def test_refresh_daily_excludes_bots(db):
# 爬虫口径:事件/会话/访客三处 is_bot 均不计入四指标(与 pv/uv/sessions 统一)
site = Site.objects.create(name="s", domains=["a.com"])
bot = Visitor.objects.create(site=site, visitor_id="bot", is_bot=True)
human = Visitor.objects.create(site=site, visitor_id="human")
bs = VisitSession.objects.create(site=site, visitor=bot, session_id="bs",
started_at="2026-09-15T10:00:00Z", ended_at="2026-09-15T10:01:00Z")
hs = VisitSession.objects.create(site=site, visitor=human, session_id="hs",
started_at="2026-09-15T10:00:00Z", ended_at="2026-09-15T10:01:00Z")
Event.objects.create(site=site, visitor=bot, session=bs, event_id="b1",
event_name="page_view", occurred_at="2026-09-15T10:00:00Z", path="/", is_bot=True)
Event.objects.create(site=site, visitor=human, session=hs, event_id="h1",
event_name="page_view", occurred_at="2026-09-15T10:00:30Z", path="/")
d = refresh_daily(site, date(2026, 9, 15))
assert (d.pv, d.uv, d.sessions) == (1, 1, 1) # 只计 human 的事件/会话
t = refresh_daily(site, timezone.localdate()) # 两位访客 first_seen_at 均为运行时,刷今天窗口才能非空验证 new_visitors
assert t.new_visitors == 1 # 爬虫访客不计入新增