浅念
22d89a3f42
SDK：buildEnvContext 同步扩展 12 键（dpr/viewport/orientation/maxTouchPoints/ deviceMemory/languages/colorScheme/standalone/network downlink·rtt·saveData/ 微信版本），逐键 try/catch 静默缺则不挂；新增 collectHighEntropyHints（UA-CH model/platformVersion/architecture/bitness），init 后 Promise.race 2s 超时 异步并入 __ctk_ctx——批次 context flush 时读取，不阻塞首屏。 后端：Visitor 加 device_model/env_extra（迁移 0005）；ingest 首批快照 EXTENDED_ENV_KEYS 白名单（值截 100），非首批不覆盖、唯 device_model 空且 本批有值时补填一次（晚到 hints）；列表/详情 API 随行输出。 前端：详情访问环境卡设备行升级显示型号（手机 · Pixel 7），追加像素比/ 视口/屏幕方向/触点/内存/网络质量/深浅色/微信版本行（env_extra 逐键判空， 旧数据整块不显示）；列表设备列主行拼型号（手机·Pixel 7）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
220 lines
11 KiB
Python
220 lines
11 KiB
Python
from datetime import timedelta
|
||
from django.utils import timezone
|
||
from tracker.models import Site, Visitor, VisitSession, Event
|
||
from tracker.services.geo import resolver as _geo
|
||
from tracker.services.ingest import ingest_batch
|
||
|
||
NOW = timezone.now()
|
||
|
||
def _batch(events, vid="v1", sid="s1"):
|
||
return {"visitor_id": vid, "session_id": sid, "context": {
|
||
"referrer_domain": "google.com", "utm": {"utm_source": "google"},
|
||
"path": "/landing", "device_type": "PC", "os": "Windows",
|
||
"browser": "Chrome", "language": "en-US", "screen": "1920x1080"},
|
||
"events": events}
|
||
|
||
def _ev(i, name="page_view", **kw):
|
||
d = dict(event_id=f"e{i}", event_name=name, occurred_at=NOW,
|
||
path="/p", title="T", properties={})
|
||
d.update(kw)
|
||
return d
|
||
|
||
# 1. 首批建档
|
||
def test_first_batch_creates_visitor_session_events(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
r = ingest_batch(site, _batch([_ev(1), _ev(2)]), ip="114.114.114.114")
|
||
assert (r["accepted"], r["duplicated"], r["rejected"]) == (2, 0, [])
|
||
v = Visitor.objects.get(site=site, visitor_id="v1")
|
||
assert v.first_referrer_domain == "google.com" and v.visit_count == 1
|
||
assert v.active_days == 1 and v.last_ip == "114.114.114.114"
|
||
if _geo._searcher: # 地理断言仅在有真实 xdb 时执行，CI/fresh clone 不因缺库变红
|
||
assert v.country == "中国" and v.city == "南京市" # 114 DNS（江苏南京）
|
||
assert Event.objects.filter(site=site).count() == 2
|
||
|
||
# 2. 重放幂等 + 不抬高会话聚合
|
||
def test_duplicate_events_never_inflate_aggregates(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1), _ev(2, "page_stay", properties={"active_ms": 3000, "path": "/p"})]))
|
||
r = ingest_batch(site, _batch([_ev(1)])) # 重放
|
||
assert r["duplicated"] == 1 and Event.objects.count() == 2
|
||
sess = VisitSession.objects.get()
|
||
assert sess.event_count == 2 and sess.stay_ms == 3000 # 重放没有重复加分
|
||
|
||
# 3. 新会话递增 visit_count（F 表达式）
|
||
def test_new_session_increments_visit_count(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1)], sid="s1"))
|
||
ingest_batch(site, _batch([_ev(2)], sid="s2"))
|
||
v = Visitor.objects.get(visitor_id="v1")
|
||
assert v.visit_count == 2 and VisitSession.objects.count() == 2
|
||
|
||
# 4. 服务端 30 分钟兜底切分（同 sid 间隔>30min 切出 s1#2，且同批后续事件复用同段）
|
||
def test_server_backstop_splits_idle_gap(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1, occurred_at=NOW - timedelta(minutes=90))]))
|
||
r = ingest_batch(site, _batch([_ev(2), _ev(3)])) # 两条同批
|
||
assert VisitSession.objects.count() == 2 # s1 与 s1#2，不会切成 3 段
|
||
assert r["accepted"] == 2
|
||
|
||
# 5. 拒绝事件带原因上报
|
||
def test_rejected_events_reported_with_reason(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
r = ingest_batch(site, _batch([_ev(1, event_name="bad"), _ev(2)]))
|
||
assert r["accepted"] == 1 and r["rejected"][0]["reason"] == "event_name_not_allowed"
|
||
|
||
# 6. page_stay 的 active_ms 累计进 stay_ms
|
||
def test_stay_ms_accumulates(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([
|
||
_ev(1), _ev(2, "page_stay", properties={"active_ms": 5000, "path": "/p"})]))
|
||
ingest_batch(site, _batch([
|
||
_ev(3, occurred_at=NOW + timedelta(minutes=1)),
|
||
_ev(4, "page_stay", properties={"active_ms": 7000, "path": "/p"},
|
||
occurred_at=NOW + timedelta(minutes=1))]))
|
||
assert VisitSession.objects.get().stay_ms == 12000
|
||
|
||
# 7. 爬虫 UA 标记（访客+事件）
|
||
def test_bot_ua_marked(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1)]), ip="66.249.66.1", ua="Mozilla/5.0 (compatible; Googlebot/2.1)")
|
||
assert Visitor.objects.get().is_bot is True
|
||
assert Event.objects.get().is_bot is True
|
||
|
||
# 8. 超批拒绝
|
||
def test_batch_size_invalid(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
r = ingest_batch(site, _batch([_ev(i) for i in range(60)]))
|
||
assert r["rejected"] == [{"reason": "batch_size_invalid"}]
|
||
|
||
# 9. 同批重复 event_id：去重后只入一份，聚合不灌倍
|
||
def test_intra_batch_duplicate_event_id_deduped(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
evs = [_ev(1, "page_stay", properties={"active_ms": 1000, "path": "/p"})
|
||
for _ in range(3)] # 3 份同 id
|
||
r = ingest_batch(site, _batch(evs))
|
||
assert (r["accepted"], r["duplicated"]) == (1, 2)
|
||
sess = VisitSession.objects.get()
|
||
assert sess.event_count == 1 and sess.stay_ms == 1000
|
||
|
||
# 10. 切分后同 sid 小间隔连发：空闲比较取会话族最大 ended_at，不再每批新开段
|
||
def test_split_then_small_gap_reuses_segment(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1, occurred_at=NOW - timedelta(minutes=90))]))
|
||
ingest_batch(site, _batch([_ev(2)])) # 90min 间隔 → 切出 s1#2
|
||
ingest_batch(site, _batch([_ev(3, occurred_at=NOW + timedelta(minutes=1))]))
|
||
ingest_batch(site, _batch([_ev(4, occurred_at=NOW + timedelta(minutes=2))]))
|
||
assert VisitSession.objects.count() == 2 # 保持 s1 与 s1#2，不切 s1#3/s1#4
|
||
assert Visitor.objects.get(visitor_id="v1").visit_count == 2
|
||
|
||
# 11. invariant 4：乱序批次（后入更早发生）ended_at 不回退
|
||
def test_ended_at_never_regresses_on_out_of_order_batch(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1, occurred_at=NOW + timedelta(minutes=1))]))
|
||
ingest_batch(site, _batch([_ev(2, occurred_at=NOW)]))
|
||
sess = VisitSession.objects.get()
|
||
assert abs((sess.ended_at - (NOW + timedelta(minutes=1))).total_seconds()) < 1
|
||
|
||
# 12. 首触持久：第二批换 referrer 不改首触字段
|
||
def test_first_touch_immutable_across_batches(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1)]))
|
||
b = _batch([_ev(2)])
|
||
b["context"]["referrer_domain"] = "bing.com"
|
||
ingest_batch(site, b)
|
||
v = Visitor.objects.get(visitor_id="v1")
|
||
assert v.first_referrer_domain == "google.com"
|
||
assert v.first_utm == {"utm_source": "google"}
|
||
|
||
# 13. payload 缺 visitor_id/session_id 提前拒绝
|
||
def test_missing_payload_fields_rejected(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
p = _batch([_ev(1)])
|
||
del p["visitor_id"]
|
||
assert ingest_batch(site, p)["rejected"] == [{"reason": "invalid_payload"}]
|
||
p = _batch([_ev(1)])
|
||
del p["session_id"]
|
||
assert ingest_batch(site, p)["rejected"] == [{"reason": "invalid_payload"}]
|
||
|
||
# 14. SDK context 的 timezone 落库 Visitor.timezone_name（Task16 联调发现：键名映射缺口）
|
||
def test_context_timezone_persisted_to_visitor(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
b = _batch([_ev(1)])
|
||
b["context"]["timezone"] = "Asia/Shanghai"
|
||
ingest_batch(site, b)
|
||
assert Visitor.objects.get(visitor_id="v1").timezone_name == "Asia/Shanghai"
|
||
|
||
# 15. 归因分类 best-effort：引擎抛异常不影响采集入库（first_touch/session_entry 双路）
|
||
def test_attribution_failure_never_breaks_ingest(db, monkeypatch):
|
||
from tracker.services import ingest as ingest_mod
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
monkeypatch.setattr(ingest_mod, "classify", lambda **kw: (_ for _ in ()).throw(RuntimeError("boom")))
|
||
r = ingest_batch(site, _batch([_ev(1)]))
|
||
assert r["accepted"] == 1 and Event.objects.count() == 1
|
||
assert Visitor.objects.count() == 1 and VisitSession.objects.count() == 1
|
||
|
||
# 16. 首触分类用本批第一条 page_view 的完整 referrer（搜索词可恢复），非 ctx 裸域名
|
||
def test_first_touch_classification_uses_full_pageview_referrer(db):
|
||
from tracker.models import SourceClassification
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
b = _batch([_ev(1, properties={"referrer": "https://m.baidu.com/s?wd=挖掘机"})])
|
||
b["context"]["utm"] = {} # 默认 fixture 带 utm_source=google，与百度矛盾 → conflict
|
||
ingest_batch(site, b)
|
||
ft = SourceClassification.objects.get(scope="first_touch")
|
||
assert ft.channel == "natural_search" and ft.platform == "百度"
|
||
assert ft.search_term == "挖掘机" and ft.resolution == "resolved"
|
||
|
||
# 17. 扩展环境键快照：device_model 独立列 + env_extra 白名单（白名单外不入库）
|
||
def test_extended_env_snapshot_persisted(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
b = _batch([_ev(1)])
|
||
b["context"].update({
|
||
"device_model": "Pixel 7", "device_pixel_ratio": 2.625, "viewport": "393x851",
|
||
"orientation": "portrait-primary", "max_touch_points": 5, "device_memory": 8,
|
||
"languages": "zh-CN,en", "color_scheme": "dark", "standalone": "0",
|
||
"network_downlink": 10, "network_rtt": 50, "save_data": "0",
|
||
"wechat_version": "8.0.49", "platform_version": "13.0.0",
|
||
"architecture": "arm", "bitness": "64",
|
||
"evil_key": "<script>", # 白名单外：不入 env_extra
|
||
})
|
||
ingest_batch(site, b)
|
||
v = Visitor.objects.get(visitor_id="v1")
|
||
assert v.device_model == "Pixel 7"
|
||
assert v.env_extra == {
|
||
"device_model": "Pixel 7", "device_pixel_ratio": "2.625", "viewport": "393x851",
|
||
"orientation": "portrait-primary", "max_touch_points": "5", "device_memory": "8",
|
||
"languages": "zh-CN,en", "color_scheme": "dark", "standalone": "0",
|
||
"network_downlink": "10", "network_rtt": "50", "save_data": "0",
|
||
"wechat_version": "8.0.49", "platform_version": "13.0.0",
|
||
"architecture": "arm", "bitness": "64"}
|
||
|
||
# 18. 扩展字段截断：device_model 64 / env_extra 值 100
|
||
def test_extended_env_fields_truncated(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
b = _batch([_ev(1)])
|
||
b["context"].update({"device_model": "M" * 80, "languages": "x" * 200})
|
||
ingest_batch(site, b)
|
||
v = Visitor.objects.get(visitor_id="v1")
|
||
assert v.device_model == "M" * 64 # 独立列按 CharField(64) 截断
|
||
assert v.env_extra["device_model"] == "M" * 80 # env_extra 白名单统一 [:100]（80<100 原样）
|
||
assert v.env_extra["languages"] == "x" * 100
|
||
|
||
# 19. 晚到 UA-CH hints：首批无型号 → 第二批只补填 device_model 列，env_extra 保持首触快照
|
||
def test_device_model_backfilled_on_later_batch(db):
|
||
site = Site.objects.create(name="s", domains=["a.com"])
|
||
ingest_batch(site, _batch([_ev(1)])) # 首批无任何扩展键
|
||
v = Visitor.objects.get(visitor_id="v1")
|
||
assert v.device_model == "" and v.env_extra == {}
|
||
|
||
b = _batch([_ev(2)])
|
||
b["context"].update({"device_model": "Pixel 7", "viewport": "393x851"})
|
||
ingest_batch(site, b)
|
||
v.refresh_from_db()
|
||
assert v.device_model == "Pixel 7" # 空列补填一次
|
||
assert v.env_extra == {} # env_extra 不补写（首触快照语义）
|
||
|
||
b3 = _batch([_ev(3)])
|
||
b3["context"]["device_model"] = "iPhone 15 Pro"
|
||
ingest_batch(site, b3)
|
||
v.refresh_from_db()
|
||
assert v.device_model == "Pixel 7" # 已有值不再覆盖
|