fangke/server/tracker/services/ingest.py

"""批量入库服务：访客/会话 get_or_create、事件幂等写入、聚合更新。

并发不变量（四轮审查收口）：
  ① event 绝不重复入库：uq_site_event + bulk_create(ignore_conflicts=True) + 入库前 existed 预查
     + 同批 event_id 去重（seen_eids）；
  ② 重放命中的事件不参与会话聚合（new_events 只含未入库事件）；
  ③ visit_count/active_days/event_count/duration_ms/stay_ms 全走 F 表达式，
     filter().update() 避免 read-modify-write 全量 save() 用旧值覆盖并发赢家；
  ④ 会话 ended_at 用 Greatest 只前进不回退，乱序批次不改早结束时间；
  ⑤ 同批切分出的会话段按"原始 sid"缓存复用，后续事件不再切分。

已接受的残余风险（M1 不修）：existed 预查与 bulk_create 之间存在毫秒级 TOCTOU 窗口——
并发请求在预查后、写入前提交同一 event_id 时，本请求的 ignore_conflicts 静默吞掉冲突，
但该事件已被计入 new_events 参与聚合，聚合可能双计。窗口极窄且仅影响并发重放场景。
"""
from datetime import timedelta

from django.db import transaction, IntegrityError
from django.db.models import F, Q
from django.db.models.functions import Greatest
from django.utils import timezone

from tracker import constants as C
from tracker.models import SourceClassification, Visitor, VisitSession, Event
from tracker.services.validation import validate_event
from tracker.services.attribution import classify
from tracker.services.geo import resolver as _geo   # 模块级单例（勿实例化，11MB/份）

_FIELD_LEN = {"device_type": 16, "os": 64, "browser": 64, "language": 32, "screen": 32}
_CLASS_FIELDS = ("channel", "platform", "variant", "search_term",
                 "evidence_kind", "resolution", "detail")


def _clean_utm(d):
    """utm 键白名单（utm_* 前缀）+ 截断——任意字符串不得经 utm 入库渲染。"""
    return {k[:50]: str(v)[:100] for k, v in (d or {}).items()
            if isinstance(k, str) and k.startswith("utm_")}


def _geo_field(g, key):
    """geo 值归一：缺失段的 "0"/"" 占位统一为空串。geo dict 被 lru_cache 共享，只读勿改。"""
    v = (g or {}).get(key, "")
    return v if v not in ("", "0") else ""


def _first_pageview_referrer(events) -> str:
    """本批第一条带 referrer 的 page_view 的完整来源 URL（旧版 SDK 无此字段 → ""）。"""
    for ev in events:
        if isinstance(ev, dict) and ev.get("event_name") == "page_view":
            r = (ev.get("properties") or {}).get("referrer")
            if isinstance(r, str) and r:
                return r
    return ""


def _referrer_url(domain, full: str) -> str:
    """分类入参统一为完整 URL：优先 page_view 的完整 referrer（含搜索词），
    缺失时用 ctx 的 referrer_domain 合成（丢查询串，搜索词无法找回——诚实降级）。"""
    if full:
        return full
    d = str(domain or "").strip().lower().strip("/")
    return f"https://{d}" if d else ""


def _classify_and_store(*, site, visitor, session, scope, referrer_url,
                        utm, landing_path, ua):
    """来源分类落库（best-effort）：任何失败静默吞掉，绝不影响采集主流程。
    savepoint 隔离——DB 异常不至于标记外层采集事务需回滚。"""
    try:
        with transaction.atomic():
            r = classify(referrer_url=referrer_url, utm=utm or {},
                         landing_path=landing_path or "", ua=ua or "",
                         internal_domains=site.domains or [])
            vals = {"channel": (r["channel"] or "")[:32],
                    "platform": (r["platform"] or "")[:64],
                    "variant": (r["variant"] or "")[:64],
                    "search_term": (r["search_term"] or "")[:200],
                    "evidence_kind": (r["evidence_kind"] or "")[:32],
                    "resolution": (r["resolution"] or "")[:16],
                    "detail": r["detail"] or {}, "session": session}
            if session is None:   # first_touch：键 site+visitor+scope（每访客一行）
                SourceClassification.objects.update_or_create(
                    site=site, visitor=visitor, scope=scope, defaults=vals)
            else:                 # session_entry：键 site+session+scope（每会话一行）
                SourceClassification.objects.update_or_create(
                    site=site, session=session, scope=scope,
                    defaults={**vals, "visitor": visitor})
    except Exception:
        pass


def _touch_visitor(site, vid, ctx, ip):
    env = {k: str(ctx.get(k, "") or "")[:n] for k, n in _FIELD_LEN.items()}
    # Task16 联调发现：键名映射缺口——SDK context 发 "timezone"，模型字段是 timezone_name，
    # 同名推导的 _FIELD_LEN 取不到值，导致 timezone_name 恒空。
    env["timezone_name"] = str(ctx.get("timezone", "") or "")[:64]
    v, created = Visitor.objects.get_or_create(
        site=site, visitor_id=vid, defaults={**env, "last_ip": ip})
    now = timezone.now()
    upd = {"last_seen_at": now}   # 聚合字段一律 filter().update()，避免并发下全量 save() 用旧值覆盖赢家
    if created:
        upd.update(first_referrer_domain=(ctx.get("referrer_domain") or "")[:200],
                   first_utm=_clean_utm(ctx.get("utm")),
                   first_landing_path=(ctx.get("path") or "")[:500],
                   last_active_date=now.date(), active_days=1)
    elif not Visitor.objects.filter(pk=v.pk, last_active_date=now.date()).exists():
        upd.update(last_active_date=now.date(), active_days=F("active_days") + 1)
    if ip:
        upd["last_ip"] = ip
        g = _geo.resolve(ip)
        if g:
            upd.update(country=_geo_field(g, "country")[:64],
                       province=_geo_field(g, "province")[:64],
                       city=_geo_field(g, "city")[:64])
    Visitor.objects.filter(pk=v.pk).update(**upd)
    return v, created   # created 供首触来源分类（ingest_batch 消费）


def _get_session(site, visitor, sid, ctx, occurred_at, ua="", referrer_url=""):
    """会话权威在 SDK；服务端兜底：同 session_id 族（s1、s1#2…）距最后事件>30min 切分后缀段。

    空闲比较必须取整个会话族的最大 ended_at：若只查根段（session_id=sid 精确匹配），
    切分一次后根段 ended_at 陈旧，后续小间隔批次会反复误判超时、每批新开段（visit_count 灌倍）。
    """
    idle = timedelta(milliseconds=C.SESSION_IDLE_MS)
    fam = list(VisitSession.objects.filter(
        Q(session_id=sid) | Q(session_id__startswith=sid + "#"), site=site))
    last = max(fam, key=lambda s: s.ended_at) if fam else None
    if last and occurred_at - last.ended_at > idle:
        suffix = 2 + sum(1 for s in fam if s.session_id.startswith(sid + "#"))
        sid, sess = f"{sid}#{suffix}", None
    else:
        sess = last
    if not sess:
        try:
            with transaction.atomic():   # savepoint：IntegrityError 后外层事务还能继续
                sess = VisitSession.objects.create(
                    site=site, visitor=visitor, session_id=sid,
                    started_at=occurred_at, ended_at=occurred_at,
                    referrer_domain=(ctx.get("referrer_domain") or "")[:200],
                    utm=_clean_utm(ctx.get("utm")),
                    landing_path=(ctx.get("path") or "")[:500])
        except IntegrityError:   # 并发同 sid 同时切分出相同后缀撞 uq_site_session——重查复用
            sess = VisitSession.objects.filter(site=site, session_id=sid).first()
            if not sess:
                raise
        Visitor.objects.filter(pk=visitor.pk).update(visit_count=F("visit_count") + 1)
        # 新会话入口来源分类（best-effort）：创建成功与并发撞键重查两条路径都会走到，
        # 同键 upsert 幂等——赢家请求已写时这只是覆写同值。
        _classify_and_store(site=site, visitor=visitor, session=sess,
                            scope="session_entry",
                            referrer_url=_referrer_url(ctx.get("referrer_domain"), referrer_url),
                            utm=_clean_utm(ctx.get("utm")),
                            landing_path=str(ctx.get("path") or ""), ua=ua)
    return sess


@transaction.atomic
def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> dict:
    now = timezone.now()
    ctx = payload.get("context") or {}
    events = payload.get("events") or []
    result = {"accepted": 0, "duplicated": 0, "rejected": []}
    if not payload.get("visitor_id") or not payload.get("session_id"):
        return {"accepted": 0, "duplicated": 0,
                "rejected": [{"reason": "invalid_payload"}]}
    if not events or len(events) > C.MAX_EVENTS_PER_BATCH:
        return {"accepted": 0, "duplicated": 0,
                "rejected": [{"reason": "batch_size_invalid"}]}
    visitor, created = _touch_visitor(site, payload["visitor_id"], ctx, ip)
    geo = _geo.resolve(ip) if ip else None
    is_bot = any(k in (ua or "").lower() for k in C.BOT_UA_KEYWORDS)
    if is_bot:
        Visitor.objects.filter(pk=visitor.pk).update(is_bot=True)
    # 首触来源分类（best-effort）：首批 page_view 的完整 referrer 优先（含搜索词），
    # 旧 SDK 只有 ctx.referrer_domain 时合成降级。仅在访客首次建档时写一次。
    pv_referrer = _first_pageview_referrer(events)
    if created:
        _classify_and_store(site=site, visitor=visitor, session=None, scope="first_touch",
                            referrer_url=_referrer_url(ctx.get("referrer_domain"), pv_referrer),
                            utm=_clean_utm(ctx.get("utm")),
                            landing_path=str(ctx.get("path") or ""), ua=ua or "")
    ready, session_cache, seen_eids = [], {}, set()
    for i, ev in enumerate(events):
        reason = validate_event(ev)
        if reason:
            result["rejected"].append({"index": i, "event_id": ev.get("event_id", ""),
                                       "reason": reason})
            continue
        if ev["event_id"] in seen_eids:   # 同批重复 event_id：只入一份，防聚合/计数灌倍
            result["duplicated"] += 1
            continue
        seen_eids.add(ev["event_id"])
        occ = ev["occurred_at"]
        sid = payload["session_id"]
        sess = session_cache.get(sid)
        if not sess:
            sess = _get_session(site, visitor, sid, ctx, occ, ua=ua or "",
                                referrer_url=pv_referrer)
            session_cache[sid] = sess      # 按原始 sid 存：切分后 sess.session_id 是 "s1#2"，
                                           # 若按它存，同批后续事件按 "s1" 查会 miss → 每条再切新会话
        ready.append(Event(
            site=site, visitor=visitor, session=sess,
            event_id=ev["event_id"], event_name=ev["event_name"],
            occurred_at=occ, path=(ev.get("path") or "")[:500],
            title=(ev.get("title") or "")[:200],
            properties=ev.get("properties") or {},
            ip=ip, late=abs((now - occ).total_seconds()) > C.CLOCK_SKEW_LATE_SEC,
            country=_geo_field(geo, "country")[:64], province=_geo_field(geo, "province")[:64],
            city=_geo_field(geo, "city")[:64], is_bot=is_bot))
    if ready:
        existed = set(Event.objects.filter(
            site=site, event_id__in=[e.event_id for e in ready]
        ).values_list("event_id", flat=True))
        new_events = [e for e in ready if e.event_id not in existed]   # 重放命中的不参与聚合！
        Event.objects.bulk_create(new_events, ignore_conflicts=True)
        result["accepted"] = len(new_events)
        result["duplicated"] += len(ready) - len(new_events)   # += ：累加同批去重已计入的份数
        for sess in {e.session for e in new_events}:
            se = [e for e in new_events if e.session.pk == sess.pk]
            new_end = max(x.occurred_at for x in se)
            span_ms = int((new_end - min(x.occurred_at for x in se)).total_seconds() * 1000)
            stay_add = sum(int(e.properties.get("active_ms", 0) or 0)
                           for e in se if e.event_name == "page_stay")
            VisitSession.objects.filter(pk=sess.pk).update(
                ended_at=Greatest(F("ended_at"), new_end),   # 只前进不回退，防乱序批次改早结束时间
                event_count=F("event_count") + len(se),
                duration_ms=F("duration_ms") + span_ms,
                stay_ms=F("stay_ms") + stay_add)             # 有效停留单独累计，供访客"总停留"取数
    return result