feat: 来源归因规则引擎（渠道/AI/搜索词/诚实状态/历史重算）
- constants 规则库 v1：AI/引擎/社交域名表、搜索词参数白名单、付费点击参数、 应用内 UA 标记、CHANNELS 枚举；page_view 白名单开 nav_type/prerendering - services/attribution.classify 纯函数：付费>AI>引擎>社交>referral 分层， UTM 证据叠加（combined/conflict/label_only），应用内 UA 叠加/兜底， 引擎 TLD 国家 variant，搜索词白名单提取（Google 恒空不虚构） - SourceClassification 派生表（迁移 0004）：first_touch/session_entry 双 scope， rule_version 可追溯 - ingest 增量 best-effort：首访客/新会话时分类落库（savepoint+静默， 失败不影响采集）；validation 对 page_view referrer 完整 URL 参数清洗+截500 - recompute_attribution 命令：全量/按站幂等重算 - API：访客列表归因五字段+channel 筛选、详情 attribution 块（含最近5会话入口）、 dashboard channels 聚合（sources 原口径保留）；TS VisitorListParams 加 channel - 测试 97 → 124（引擎20 + 集成/接口7） Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -21,11 +21,14 @@ from django.db.models.functions import Greatest
|
||||
from django.utils import timezone
|
||||
|
||||
from tracker import constants as C
|
||||
from tracker.models import Visitor, VisitSession, Event
|
||||
from tracker.models import SourceClassification, Visitor, VisitSession, Event
|
||||
from tracker.services.validation import validate_event
|
||||
from tracker.services.attribution import classify
|
||||
from tracker.services.geo import resolver as _geo # 模块级单例（勿实例化，11MB/份）
|
||||
|
||||
_FIELD_LEN = {"device_type": 16, "os": 64, "browser": 64, "language": 32, "screen": 32}
|
||||
_CLASS_FIELDS = ("channel", "platform", "variant", "search_term",
|
||||
"evidence_kind", "resolution", "detail")
|
||||
|
||||
|
||||
def _clean_utm(d):
|
||||
@@ -40,6 +43,52 @@ def _geo_field(g, key):
|
||||
return v if v not in ("", "0") else ""
|
||||
|
||||
|
||||
def _first_pageview_referrer(events) -> str:
|
||||
"""本批第一条带 referrer 的 page_view 的完整来源 URL（旧版 SDK 无此字段 → ""）。"""
|
||||
for ev in events:
|
||||
if isinstance(ev, dict) and ev.get("event_name") == "page_view":
|
||||
r = (ev.get("properties") or {}).get("referrer")
|
||||
if isinstance(r, str) and r:
|
||||
return r
|
||||
return ""
|
||||
|
||||
|
||||
def _referrer_url(domain, full: str) -> str:
|
||||
"""分类入参统一为完整 URL：优先 page_view 的完整 referrer（含搜索词），
|
||||
缺失时用 ctx 的 referrer_domain 合成（丢查询串，搜索词无法找回——诚实降级）。"""
|
||||
if full:
|
||||
return full
|
||||
d = str(domain or "").strip().lower().strip("/")
|
||||
return f"https://{d}" if d else ""
|
||||
|
||||
|
||||
def _classify_and_store(*, site, visitor, session, scope, referrer_url,
|
||||
utm, landing_path, ua):
|
||||
"""来源分类落库（best-effort）：任何失败静默吞掉，绝不影响采集主流程。
|
||||
savepoint 隔离——DB 异常不至于标记外层采集事务需回滚。"""
|
||||
try:
|
||||
with transaction.atomic():
|
||||
r = classify(referrer_url=referrer_url, utm=utm or {},
|
||||
landing_path=landing_path or "", ua=ua or "",
|
||||
internal_domains=site.domains or [])
|
||||
vals = {"channel": (r["channel"] or "")[:32],
|
||||
"platform": (r["platform"] or "")[:64],
|
||||
"variant": (r["variant"] or "")[:64],
|
||||
"search_term": (r["search_term"] or "")[:200],
|
||||
"evidence_kind": (r["evidence_kind"] or "")[:32],
|
||||
"resolution": (r["resolution"] or "")[:16],
|
||||
"detail": r["detail"] or {}, "session": session}
|
||||
if session is None: # first_touch：键 site+visitor+scope（每访客一行）
|
||||
SourceClassification.objects.update_or_create(
|
||||
site=site, visitor=visitor, scope=scope, defaults=vals)
|
||||
else: # session_entry：键 site+session+scope（每会话一行）
|
||||
SourceClassification.objects.update_or_create(
|
||||
site=site, session=session, scope=scope,
|
||||
defaults={**vals, "visitor": visitor})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _touch_visitor(site, vid, ctx, ip):
|
||||
env = {k: str(ctx.get(k, "") or "")[:n] for k, n in _FIELD_LEN.items()}
|
||||
# Task16 联调发现：键名映射缺口——SDK context 发 "timezone"，模型字段是 timezone_name，
|
||||
@@ -64,10 +113,10 @@ def _touch_visitor(site, vid, ctx, ip):
|
||||
province=_geo_field(g, "province")[:64],
|
||||
city=_geo_field(g, "city")[:64])
|
||||
Visitor.objects.filter(pk=v.pk).update(**upd)
|
||||
return v # 下游只用 pk（FK/子查询），无需 refresh_from_db 多耗一条 SELECT
|
||||
return v, created # created 供首触来源分类（ingest_batch 消费）
|
||||
|
||||
|
||||
def _get_session(site, visitor, sid, ctx, occurred_at):
|
||||
def _get_session(site, visitor, sid, ctx, occurred_at, ua="", referrer_url=""):
|
||||
"""会话权威在 SDK；服务端兜底：同 session_id 族（s1、s1#2…）距最后事件>30min 切分后缀段。
|
||||
|
||||
空闲比较必须取整个会话族的最大 ended_at：若只查根段（session_id=sid 精确匹配），
|
||||
@@ -96,6 +145,13 @@ def _get_session(site, visitor, sid, ctx, occurred_at):
|
||||
if not sess:
|
||||
raise
|
||||
Visitor.objects.filter(pk=visitor.pk).update(visit_count=F("visit_count") + 1)
|
||||
# 新会话入口来源分类（best-effort）：创建成功与并发撞键重查两条路径都会走到，
|
||||
# 同键 upsert 幂等——赢家请求已写时这只是覆写同值。
|
||||
_classify_and_store(site=site, visitor=visitor, session=sess,
|
||||
scope="session_entry",
|
||||
referrer_url=_referrer_url(ctx.get("referrer_domain"), referrer_url),
|
||||
utm=_clean_utm(ctx.get("utm")),
|
||||
landing_path=str(ctx.get("path") or ""), ua=ua)
|
||||
return sess
|
||||
|
||||
|
||||
@@ -111,11 +167,19 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
|
||||
if not events or len(events) > C.MAX_EVENTS_PER_BATCH:
|
||||
return {"accepted": 0, "duplicated": 0,
|
||||
"rejected": [{"reason": "batch_size_invalid"}]}
|
||||
visitor = _touch_visitor(site, payload["visitor_id"], ctx, ip)
|
||||
visitor, created = _touch_visitor(site, payload["visitor_id"], ctx, ip)
|
||||
geo = _geo.resolve(ip) if ip else None
|
||||
is_bot = any(k in (ua or "").lower() for k in C.BOT_UA_KEYWORDS)
|
||||
if is_bot:
|
||||
Visitor.objects.filter(pk=visitor.pk).update(is_bot=True)
|
||||
# 首触来源分类（best-effort）：首批 page_view 的完整 referrer 优先（含搜索词），
|
||||
# 旧 SDK 只有 ctx.referrer_domain 时合成降级。仅在访客首次建档时写一次。
|
||||
pv_referrer = _first_pageview_referrer(events)
|
||||
if created:
|
||||
_classify_and_store(site=site, visitor=visitor, session=None, scope="first_touch",
|
||||
referrer_url=_referrer_url(ctx.get("referrer_domain"), pv_referrer),
|
||||
utm=_clean_utm(ctx.get("utm")),
|
||||
landing_path=str(ctx.get("path") or ""), ua=ua or "")
|
||||
ready, session_cache, seen_eids = [], {}, set()
|
||||
for i, ev in enumerate(events):
|
||||
reason = validate_event(ev)
|
||||
@@ -131,7 +195,8 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
|
||||
sid = payload["session_id"]
|
||||
sess = session_cache.get(sid)
|
||||
if not sess:
|
||||
sess = _get_session(site, visitor, sid, ctx, occ)
|
||||
sess = _get_session(site, visitor, sid, ctx, occ, ua=ua or "",
|
||||
referrer_url=pv_referrer)
|
||||
session_cache[sid] = sess # 按原始 sid 存：切分后 sess.session_id 是 "s1#2"，
|
||||
# 若按它存，同批后续事件按 "s1" 查会 miss → 每条再切新会话
|
||||
ready.append(Event(
|
||||
|
||||
Reference in New Issue
Block a user