feat: 来源归因规则引擎（渠道/AI/搜索词/诚实状态/历史重算）
- constants 规则库 v1：AI/引擎/社交域名表、搜索词参数白名单、付费点击参数、 应用内 UA 标记、CHANNELS 枚举；page_view 白名单开 nav_type/prerendering - services/attribution.classify 纯函数：付费>AI>引擎>社交>referral 分层， UTM 证据叠加（combined/conflict/label_only），应用内 UA 叠加/兜底， 引擎 TLD 国家 variant，搜索词白名单提取（Google 恒空不虚构） - SourceClassification 派生表（迁移 0004）：first_touch/session_entry 双 scope， rule_version 可追溯 - ingest 增量 best-effort：首访客/新会话时分类落库（savepoint+静默， 失败不影响采集）；validation 对 page_view referrer 完整 URL 参数清洗+截500 - recompute_attribution 命令：全量/按站幂等重算 - API：访客列表归因五字段+channel 筛选、详情 attribution 块（含最近5会话入口）、 dashboard channels 聚合（sources 原口径保留）；TS VisitorListParams 加 channel - 测试 97 → 124（引擎20 + 集成/接口7） Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,180 @@
|
||||
"""来源归因规则引擎：裸域名 → 渠道/平台/搜索词/证据可信度分层识别。
|
||||
|
||||
纯函数、无 IO、无 DB——classify 是唯一入口，ingest 增量计算与
|
||||
recompute_attribution 历史重算共用同一套规则（改规则必须 bump
|
||||
constants.ATTRIBUTION_RULE_VERSION 并重跑重算）。
|
||||
|
||||
诚实状态语义（resolution）：
|
||||
resolved 观察到 referrer/点击参数/UA 等硬证据
|
||||
combined referrer 观察与 UTM 标签互相印证
|
||||
conflict referrer 结论与 UTM 标签矛盾——观察证据优先，标签并列存 detail
|
||||
label_only 仅 UTM 自述，无观察证据
|
||||
unknown 全空（直接访问）——不猜测手输/书签
|
||||
"""
|
||||
import re
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
from tracker import constants as C
|
||||
|
||||
_CTRL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
|
||||
_GENERIC_TLDS = ("com",) # 引擎 host 落在这些 TLD 时不作为国家 variant 显示
|
||||
|
||||
|
||||
def _host(url: str) -> str:
|
||||
"""提取 referrer host；异常/空 → ""。hostname 自动去端口并小写。"""
|
||||
try:
|
||||
return (urlsplit(url).hostname or "").lower()
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
def _canonical(host: str) -> str:
|
||||
"""去掉单个 www. 前缀后做域名匹配（www.bing.com → bing.com）。"""
|
||||
return host[4:] if host.startswith("www.") else host
|
||||
|
||||
|
||||
def _domain_match(host: str, key: str) -> bool:
|
||||
"""host 是否命中规则键。key 以 "." 结尾（google./yandex.）→ 前缀匹配任意国家站
|
||||
TLD；否则精确或子域（m.facebook.com 命中 facebook.com）。"""
|
||||
h = _canonical(host)
|
||||
k = key.lower()
|
||||
if k.endswith("."):
|
||||
return h == k[:-1] or h.startswith(k)
|
||||
return h == k or h.endswith("." + k)
|
||||
|
||||
|
||||
def _tld_variant(host: str) -> str:
|
||||
"""引擎 host 的 TLD 国家（www.google.co.ru → "ru"）；通用 TLD 不显示。"""
|
||||
tld = _canonical(host).rsplit(".", 1)[-1]
|
||||
return "" if tld in _GENERIC_TLDS else tld
|
||||
|
||||
|
||||
def _clean_term(raw: str) -> str:
|
||||
"""搜索词清洗：strip → 剔除控制字符 → 截 100。"""
|
||||
return _CTRL_CHARS.sub("", (raw or "").strip())[:100]
|
||||
|
||||
|
||||
def _search_term(engine: str, referrer_url: str) -> str:
|
||||
"""按引擎参数名白名单从 referrer URL 提取搜索词。Google 不在表内 → 恒空（绝不虚构）。"""
|
||||
for param in C.SEARCH_TERM_PARAMS.get(engine, []):
|
||||
for k, v in parse_qsl(urlsplit(referrer_url).query, keep_blank_values=True):
|
||||
if k == param and v.strip():
|
||||
return _clean_term(v)
|
||||
return ""
|
||||
|
||||
|
||||
def _paid_param(landing_path: str, referrer_url: str) -> tuple[str, str] | None:
|
||||
"""付费优先于自然：落地页（或 referrer）query 命中点击参数 → (参数名, 平台名)。"""
|
||||
for source in (landing_path, referrer_url):
|
||||
if not source:
|
||||
continue
|
||||
try:
|
||||
params = [k for k, _ in parse_qsl(urlsplit(source).query,
|
||||
keep_blank_values=True)]
|
||||
except ValueError:
|
||||
continue
|
||||
for k in params:
|
||||
if k in C.PAID_CLICK_PARAMS:
|
||||
return k, C.PAID_CLICK_PARAMS[k]
|
||||
return None
|
||||
|
||||
|
||||
def _in_app_name(ua: str) -> str:
|
||||
"""UA 命中应用内浏览器标记（大小写不敏感）→ 应用显示名；未命中 → ""。"""
|
||||
low = (ua or "").lower()
|
||||
for marker, name in C.INAPP_UA_MARKERS.items():
|
||||
if marker.lower() in low:
|
||||
return name
|
||||
return ""
|
||||
|
||||
|
||||
def _utm_channel(label: str) -> str:
|
||||
"""UTM 标签尽力归类：已知 AI/引擎/社交标签（裸标签 "baidu"/显示名 "Google" 皆可）
|
||||
映射到对应渠道，未知 → referral（有名来源，可信度由 label_only 单独表达）。"""
|
||||
h = (label or "").lower().strip()
|
||||
if not h:
|
||||
return "referral"
|
||||
for table, channel in ((C.AI_REFERRAL_DOMAINS, "ai_referral"),
|
||||
(C.SEARCH_ENGINE_DOMAINS, "natural_search"),
|
||||
(C.SOCIAL_DOMAINS, "social")):
|
||||
for key, name in table.items():
|
||||
if _domain_match(h, key) or key.startswith(h + ".") or h == name.lower():
|
||||
return channel
|
||||
return "referral"
|
||||
|
||||
|
||||
def classify(*, referrer_url: str = "", utm: dict | None = None,
|
||||
landing_path: str = "", ua: str = "", internal_domains: list | None = None) -> dict:
|
||||
"""把一次到访的原始信号分类为 {channel, platform, variant, search_term,
|
||||
evidence_kind, resolution, detail}。各字段默认空串/空 dict，纯函数无副作用。
|
||||
|
||||
规则序：本站域名去信号 → 付费点击参数 → AI → 搜索引擎 → 社交 → 泛 referral
|
||||
→ UTM 证据叠加（combined/conflict/label_only）→ 应用内 UA 叠加/兜底 → direct。
|
||||
"""
|
||||
out = {"channel": "", "platform": "", "variant": "", "search_term": "",
|
||||
"evidence_kind": "", "resolution": "", "detail": {}}
|
||||
utm = utm or {}
|
||||
utm_source = str(utm.get("utm_source") or "").strip()
|
||||
|
||||
host = _host(referrer_url)
|
||||
if host and any(_domain_match(host, d) for d in (internal_domains or []) if d):
|
||||
host = "" # 本站跳转不是外部来源信号
|
||||
|
||||
paid = _paid_param(landing_path, referrer_url)
|
||||
if paid:
|
||||
out.update(channel="paid_search", platform=paid[1],
|
||||
evidence_kind="observed_referrer", resolution="resolved",
|
||||
detail={"click_param": paid[0]})
|
||||
elif host:
|
||||
for key, name in C.AI_REFERRAL_DOMAINS.items():
|
||||
if _domain_match(host, key):
|
||||
out.update(channel="ai_referral", platform=name)
|
||||
break
|
||||
else:
|
||||
for key, name in C.SEARCH_ENGINE_DOMAINS.items():
|
||||
if _domain_match(host, key):
|
||||
out.update(channel="natural_search", platform=name,
|
||||
variant=_tld_variant(host),
|
||||
search_term=_search_term(name, referrer_url))
|
||||
break
|
||||
else:
|
||||
for key, name in C.SOCIAL_DOMAINS.items():
|
||||
if _domain_match(host, key):
|
||||
out.update(channel="social", platform=name)
|
||||
break
|
||||
else:
|
||||
out.update(channel="referral", platform=host)
|
||||
if not out["resolution"]:
|
||||
out.update(evidence_kind="observed_referrer", resolution="resolved")
|
||||
|
||||
# UTM 证据叠加：观察结论之上只做印证/并列，不覆盖
|
||||
if utm_source:
|
||||
if out["resolution"] == "resolved":
|
||||
if utm_source.lower() == out["platform"].lower():
|
||||
out["evidence_kind"] = "combined"
|
||||
else: # 矛盾：观察证据保留在顶层，两路证据各自完整并列进 detail
|
||||
out["resolution"] = "conflict"
|
||||
out["detail"] = {**out["detail"],
|
||||
"referrer_evidence": {
|
||||
"channel": out["channel"], "platform": out["platform"],
|
||||
"variant": out["variant"], "search_term": out["search_term"],
|
||||
"evidence_kind": "observed_referrer", "referrer_host": host},
|
||||
"utm_evidence": {
|
||||
"channel": _utm_channel(utm_source), "platform": utm_source,
|
||||
"evidence_kind": "utm_label"}}
|
||||
else: # 无 referrer 观察 → 仅标签：按 utm_source 归类显示，状态诚实标注
|
||||
out.update(channel=_utm_channel(utm_source), platform=utm_source,
|
||||
evidence_kind="utm_label", resolution="label_only")
|
||||
|
||||
# 应用内浏览器 UA：有来源 → 作为 surface 叠加；无任何来源 → 本身就是来源
|
||||
app = _in_app_name(ua)
|
||||
if app:
|
||||
if out["resolution"]:
|
||||
out["detail"] = {**out["detail"], "in_app": app}
|
||||
else:
|
||||
out.update(channel="in_app", platform=app,
|
||||
evidence_kind="observed_ua", resolution="resolved")
|
||||
|
||||
if not out["channel"]:
|
||||
out.update(channel="direct", resolution="unknown") # 不猜测手输/书签
|
||||
return out
|
||||
@@ -21,11 +21,14 @@ from django.db.models.functions import Greatest
|
||||
from django.utils import timezone
|
||||
|
||||
from tracker import constants as C
|
||||
from tracker.models import Visitor, VisitSession, Event
|
||||
from tracker.models import SourceClassification, Visitor, VisitSession, Event
|
||||
from tracker.services.validation import validate_event
|
||||
from tracker.services.attribution import classify
|
||||
from tracker.services.geo import resolver as _geo # 模块级单例（勿实例化，11MB/份）
|
||||
|
||||
_FIELD_LEN = {"device_type": 16, "os": 64, "browser": 64, "language": 32, "screen": 32}
|
||||
_CLASS_FIELDS = ("channel", "platform", "variant", "search_term",
|
||||
"evidence_kind", "resolution", "detail")
|
||||
|
||||
|
||||
def _clean_utm(d):
|
||||
@@ -40,6 +43,52 @@ def _geo_field(g, key):
|
||||
return v if v not in ("", "0") else ""
|
||||
|
||||
|
||||
def _first_pageview_referrer(events) -> str:
|
||||
"""本批第一条带 referrer 的 page_view 的完整来源 URL（旧版 SDK 无此字段 → ""）。"""
|
||||
for ev in events:
|
||||
if isinstance(ev, dict) and ev.get("event_name") == "page_view":
|
||||
r = (ev.get("properties") or {}).get("referrer")
|
||||
if isinstance(r, str) and r:
|
||||
return r
|
||||
return ""
|
||||
|
||||
|
||||
def _referrer_url(domain, full: str) -> str:
|
||||
"""分类入参统一为完整 URL：优先 page_view 的完整 referrer（含搜索词），
|
||||
缺失时用 ctx 的 referrer_domain 合成（丢查询串，搜索词无法找回——诚实降级）。"""
|
||||
if full:
|
||||
return full
|
||||
d = str(domain or "").strip().lower().strip("/")
|
||||
return f"https://{d}" if d else ""
|
||||
|
||||
|
||||
def _classify_and_store(*, site, visitor, session, scope, referrer_url,
|
||||
utm, landing_path, ua):
|
||||
"""来源分类落库（best-effort）：任何失败静默吞掉，绝不影响采集主流程。
|
||||
savepoint 隔离——DB 异常不至于标记外层采集事务需回滚。"""
|
||||
try:
|
||||
with transaction.atomic():
|
||||
r = classify(referrer_url=referrer_url, utm=utm or {},
|
||||
landing_path=landing_path or "", ua=ua or "",
|
||||
internal_domains=site.domains or [])
|
||||
vals = {"channel": (r["channel"] or "")[:32],
|
||||
"platform": (r["platform"] or "")[:64],
|
||||
"variant": (r["variant"] or "")[:64],
|
||||
"search_term": (r["search_term"] or "")[:200],
|
||||
"evidence_kind": (r["evidence_kind"] or "")[:32],
|
||||
"resolution": (r["resolution"] or "")[:16],
|
||||
"detail": r["detail"] or {}, "session": session}
|
||||
if session is None: # first_touch：键 site+visitor+scope（每访客一行）
|
||||
SourceClassification.objects.update_or_create(
|
||||
site=site, visitor=visitor, scope=scope, defaults=vals)
|
||||
else: # session_entry：键 site+session+scope（每会话一行）
|
||||
SourceClassification.objects.update_or_create(
|
||||
site=site, session=session, scope=scope,
|
||||
defaults={**vals, "visitor": visitor})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _touch_visitor(site, vid, ctx, ip):
|
||||
env = {k: str(ctx.get(k, "") or "")[:n] for k, n in _FIELD_LEN.items()}
|
||||
# Task16 联调发现：键名映射缺口——SDK context 发 "timezone"，模型字段是 timezone_name，
|
||||
@@ -64,10 +113,10 @@ def _touch_visitor(site, vid, ctx, ip):
|
||||
province=_geo_field(g, "province")[:64],
|
||||
city=_geo_field(g, "city")[:64])
|
||||
Visitor.objects.filter(pk=v.pk).update(**upd)
|
||||
return v # 下游只用 pk（FK/子查询），无需 refresh_from_db 多耗一条 SELECT
|
||||
return v, created # created 供首触来源分类（ingest_batch 消费）
|
||||
|
||||
|
||||
def _get_session(site, visitor, sid, ctx, occurred_at):
|
||||
def _get_session(site, visitor, sid, ctx, occurred_at, ua="", referrer_url=""):
|
||||
"""会话权威在 SDK；服务端兜底：同 session_id 族（s1、s1#2…）距最后事件>30min 切分后缀段。
|
||||
|
||||
空闲比较必须取整个会话族的最大 ended_at：若只查根段（session_id=sid 精确匹配），
|
||||
@@ -96,6 +145,13 @@ def _get_session(site, visitor, sid, ctx, occurred_at):
|
||||
if not sess:
|
||||
raise
|
||||
Visitor.objects.filter(pk=visitor.pk).update(visit_count=F("visit_count") + 1)
|
||||
# 新会话入口来源分类（best-effort）：创建成功与并发撞键重查两条路径都会走到，
|
||||
# 同键 upsert 幂等——赢家请求已写时这只是覆写同值。
|
||||
_classify_and_store(site=site, visitor=visitor, session=sess,
|
||||
scope="session_entry",
|
||||
referrer_url=_referrer_url(ctx.get("referrer_domain"), referrer_url),
|
||||
utm=_clean_utm(ctx.get("utm")),
|
||||
landing_path=str(ctx.get("path") or ""), ua=ua)
|
||||
return sess
|
||||
|
||||
|
||||
@@ -111,11 +167,19 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
|
||||
if not events or len(events) > C.MAX_EVENTS_PER_BATCH:
|
||||
return {"accepted": 0, "duplicated": 0,
|
||||
"rejected": [{"reason": "batch_size_invalid"}]}
|
||||
visitor = _touch_visitor(site, payload["visitor_id"], ctx, ip)
|
||||
visitor, created = _touch_visitor(site, payload["visitor_id"], ctx, ip)
|
||||
geo = _geo.resolve(ip) if ip else None
|
||||
is_bot = any(k in (ua or "").lower() for k in C.BOT_UA_KEYWORDS)
|
||||
if is_bot:
|
||||
Visitor.objects.filter(pk=visitor.pk).update(is_bot=True)
|
||||
# 首触来源分类（best-effort）：首批 page_view 的完整 referrer 优先（含搜索词），
|
||||
# 旧 SDK 只有 ctx.referrer_domain 时合成降级。仅在访客首次建档时写一次。
|
||||
pv_referrer = _first_pageview_referrer(events)
|
||||
if created:
|
||||
_classify_and_store(site=site, visitor=visitor, session=None, scope="first_touch",
|
||||
referrer_url=_referrer_url(ctx.get("referrer_domain"), pv_referrer),
|
||||
utm=_clean_utm(ctx.get("utm")),
|
||||
landing_path=str(ctx.get("path") or ""), ua=ua or "")
|
||||
ready, session_cache, seen_eids = [], {}, set()
|
||||
for i, ev in enumerate(events):
|
||||
reason = validate_event(ev)
|
||||
@@ -131,7 +195,8 @@ def ingest_batch(site, payload: dict, ip: str | None = None, ua: str = "") -> di
|
||||
sid = payload["session_id"]
|
||||
sess = session_cache.get(sid)
|
||||
if not sess:
|
||||
sess = _get_session(site, visitor, sid, ctx, occ)
|
||||
sess = _get_session(site, visitor, sid, ctx, occ, ua=ua or "",
|
||||
referrer_url=pv_referrer)
|
||||
session_cache[sid] = sess # 按原始 sid 存：切分后 sess.session_id 是 "s1#2"，
|
||||
# 若按它存，同批后续事件按 "s1" 查会 miss → 每条再切新会话
|
||||
ready.append(Event(
|
||||
|
||||
@@ -113,6 +113,10 @@ def validate_event(ev) -> str | None:
|
||||
ev["path"] = sanitize_url(ev["path"])
|
||||
if len(ev.get("path") or "") > C.MAX_PATH_LEN:
|
||||
ev["path"] = ev["path"][:C.MAX_PATH_LEN]
|
||||
# referrer 是完整 URL：同样过参数清洗（email/token 等整键剔除）再截 MAX_STR_LEN。
|
||||
# 放在 str_too_long 检查之后——超长 referrer 仍按 properties_too_long 拒，不静默截半。
|
||||
if ev["event_name"] == "page_view" and isinstance(props.get("referrer"), str) and props["referrer"]:
|
||||
props["referrer"] = sanitize_url(props["referrer"])[:C.MAX_STR_LEN]
|
||||
if len(ev.get("title") or "") > C.MAX_TITLE_LEN:
|
||||
ev["title"] = ev["title"][:C.MAX_TITLE_LEN]
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user