fix: ingest 同批去重/会话族切分修复/xdb 无关测试/边界加固

Co-Authored-By: Claude Code <noreply@anthropic.com>

server/tracker/services/ingest.py

@@ -1,17 +1,22 @@
 """批量入库服务：访客/会话 get_or_create、事件幂等写入、聚合更新。
 
 并发不变量（四轮审查收口）：
-  ① event 绝不重复入库：uq_site_event + bulk_create(ignore_conflicts=True) + 入库前 existed 预查；
+  ① event 绝不重复入库：uq_site_event + bulk_create(ignore_conflicts=True) + 入库前 existed 预查
+     + 同批 event_id 去重（seen_eids）；
   ② 重放命中的事件不参与会话聚合（new_events 只含未入库事件）；
   ③ visit_count/active_days/event_count/duration_ms/stay_ms 全走 F 表达式，
      filter().update() 避免 read-modify-write 全量 save() 用旧值覆盖并发赢家；
   ④ 会话 ended_at 用 Greatest 只前进不回退，乱序批次不改早结束时间；
   ⑤ 同批切分出的会话段按"原始 sid"缓存复用，后续事件不再切分。
+
+已接受的残余风险（M1 不修）：existed 预查与 bulk_create 之间存在毫秒级 TOCTOU 窗口——
+并发请求在预查后、写入前提交同一 event_id 时，本请求的 ignore_conflicts 静默吞掉冲突，
+但该事件已被计入 new_events 参与聚合，聚合可能双计。窗口极窄且仅影响并发重放场景。
 """
 from datetime import timedelta
 
 from django.db import transaction, IntegrityError
-from django.db.models import F
+from django.db.models import F, Q
 from django.db.models.functions import Greatest
 from django.utils import timezone
 
@@ -56,19 +61,24 @@ def _touch_visitor(site, vid, ctx, ip):
                        province=_geo_field(g, "province")[:64],
                        city=_geo_field(g, "city")[:64])
     Visitor.objects.filter(pk=v.pk).update(**upd)
-    v.refresh_from_db()
-    return v
+    return v   # 下游只用 pk（FK/子查询），无需 refresh_from_db 多耗一条 SELECT
 
 
 def _get_session(site, visitor, sid, ctx, occurred_at):
-    """会话权威在 SDK；服务端兜底：同 session_id 距上事件>30min 切分后缀段。"""
+    """会话权威在 SDK；服务端兜底：同 session_id 族（s1、s1#2…）距最后事件>30min 切分后缀段。
+
+    空闲比较必须取整个会话族的最大 ended_at：若只查根段（session_id=sid 精确匹配），
+    切分一次后根段 ended_at 陈旧，后续小间隔批次会反复误判超时、每批新开段（visit_count 灌倍）。
+    """
     idle = timedelta(milliseconds=C.SESSION_IDLE_MS)
-    sess = VisitSession.objects.filter(site=site, session_id=sid).order_by("-started_at").first()
-    if sess and occurred_at - sess.ended_at > idle:
-        suffix = 2 + VisitSession.objects.filter(
-            site=site, session_id__startswith=sid + "#").count()
-        sid = f"{sid}#{suffix}"
-        sess = None
+    fam = list(VisitSession.objects.filter(
+        Q(session_id=sid) | Q(session_id__startswith=sid + "#"), site=site))
+    last = max(fam, key=lambda s: s.ended_at) if fam else None
+    if last and occurred_at - last.ended_at > idle:
+        suffix = 2 + sum(1 for s in fam if s.session_id.startswith(sid + "#"))
+        sid, sess = f"{sid}#{suffix}", None
+    else:
+        sess = last
     if not sess:
         try:
             with transaction.atomic():   # savepoint：IntegrityError 后外层事务还能继续
@@ -92,6 +102,9 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
     ctx = payload.get("context") or {}
     events = payload.get("events") or []
     result = {"accepted": 0, "duplicated": 0, "rejected": []}
+    if not payload.get("visitor_id") or not payload.get("session_id"):
+        return {"accepted": 0, "duplicated": 0,
+                "rejected": [{"reason": "invalid_payload"}]}
     if not events or len(events) > C.MAX_EVENTS_PER_BATCH:
         return {"accepted": 0, "duplicated": 0,
                 "rejected": [{"reason": "batch_size_invalid"}]}
@@ -100,13 +113,17 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
     is_bot = any(k in (ua or "").lower() for k in C.BOT_UA_KEYWORDS)
     if is_bot:
         Visitor.objects.filter(pk=visitor.pk).update(is_bot=True)
-    ready, session_cache = [], {}
+    ready, session_cache, seen_eids = [], {}, set()
     for i, ev in enumerate(events):
         reason = validate_event(ev)
         if reason:
             result["rejected"].append({"index": i, "event_id": ev.get("event_id", ""),
                                        "reason": reason})
             continue
+        if ev["event_id"] in seen_eids:   # 同批重复 event_id：只入一份，防聚合/计数灌倍
+            result["duplicated"] += 1
+            continue
+        seen_eids.add(ev["event_id"])
         occ = ev["occurred_at"]
         sid = payload["session_id"]
         sess = session_cache.get(sid)
@@ -121,8 +138,8 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
             title=(ev.get("title") or "")[:200],
             properties=ev.get("properties") or {},
             ip=ip, late=abs((now - occ).total_seconds()) > C.CLOCK_SKEW_LATE_SEC,
-            country=_geo_field(geo, "country"), province=_geo_field(geo, "province"),
-            city=_geo_field(geo, "city"), is_bot=is_bot))
+            country=_geo_field(geo, "country")[:64], province=_geo_field(geo, "province")[:64],
+            city=_geo_field(geo, "city")[:64], is_bot=is_bot))
     if ready:
         existed = set(Event.objects.filter(
             site=site, event_id__in=[e.event_id for e in ready]
@@ -130,7 +147,7 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
         new_events = [e for e in ready if e.event_id not in existed]   # 重放命中的不参与聚合！
         Event.objects.bulk_create(new_events, ignore_conflicts=True)
         result["accepted"] = len(new_events)
-        result["duplicated"] = len(ready) - len(new_events)
+        result["duplicated"] += len(ready) - len(new_events)   # += ：累加同批去重已计入的份数
         for sess in {e.session for e in new_events}:
             se = [e for e in new_events if e.session.pk == sess.pk]
             new_end = max(x.occurred_at for x in se)