From 57ea40ed95fb811587574430e49652f8c132759f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Tue, 15 Sep 2026 10:17:52 +0800 Subject: [PATCH] =?UTF-8?q?docs:=20M1=20=E5=AE=9E=E6=96=BD=E8=AE=A1?= =?UTF-8?q?=E5=88=92=EF=BC=8823=20=E4=BB=BB=E5=8A=A1=EF=BC=8C=E8=AF=84?= =?UTF-8?q?=E5=AE=A1=E9=80=9A=E8=BF=87+=E4=BF=AE=E6=AD=A3=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Code --- ...026-09-15-m1-data-collection-foundation.md | 1348 +++++++++++++++++ 1 file changed, 1348 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md diff --git a/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md b/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md new file mode 100644 index 0000000..b8685b7 --- /dev/null +++ b/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md @@ -0,0 +1,1348 @@ +# M1 数据采集底座 · 实施计划（JS SDK + Django 采集后端 + Art Design Pro 后台） + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 打通端到端最小闭环——网站接入 JS SDK → 批量上报事件 → Django 采集入库（MySQL）→ Art Design Pro 后台展示访客跨天行为时间线。 + +**Architecture:** Monorepo 三组件：`server/`（Django+DRF 单应用 `tracker`，采集公开接口 + JWT 管理接口）、`sdk/`（原生 TS 写的浏览器 SDK，esbuild 打包 IIFE，≤15KB gzip）、`admin/`（Art Design Pro 模板精简后开发 5 个页面）。事件全量入库（M1 不分区，PK/索引设计为将来按月分区留路），会话权威在 SDK、服务端兜底校验。 + +**Tech Stack:** Python 3.11+ / Django 5.x / DRF / MySQL 8（测试用 SQLite）/ simplejwt / django-cors-headers / ip2region xdb；Node 20+ / TypeScript / esbuild / vitest + happy-dom；Vue3 + Element Plus（Art Design Pro 模板）。 + +**Spec:** `docs/superpowers/specs/2026-09-15-customer-behavior-platform-design.md`（M1 范围：§13 M1；红线：§4.6、§9） + +--- + +## 文件结构总览 + +``` +cms-cj/ +├── server/ # Django 后端 +│ ├── manage.py +│ ├── requirements.txt +│ ├── pytest.ini +│ ├── config/ # Django 工程（settings/urls/wsgi） +│ ├── tracker/ # 唯一业务应用 +│ │ ├── models.py # Site/Visitor/VisitSession/Event/DailyStat +│ │ ├── constants.py # 事件名白名单、限制、敏感键黑名单 +│ │ ├── services/ +│ │ │ ├── validation.py # 事件结构/敏感键/时间窗校验 +│ │ │ ├── ingest.py # 批量入库：幂等/访客/会话/首触 +│ │ │ ├── geo.py # ip2region 离线解析 +│ │ │ └── stats.py # 总览聚合查询 +│ │ ├── api/ +│ │ │ ├── collect.py # POST /api/collect/batch +│ │ │ ├── auth.py # JWT 登录/刷新/me +│ │ │ ├── sites.py # 站点 CRUD +│ │ │ ├── dashboard.py # 总览数据 +│ │ │ └── visitors.py # 访客列表/详情/时间线/会话 +│ │ ├── management/commands/refresh_stats.py +│ │ └── tests/ # pytest 测试 +│ ├── Dockerfile +│ └── data/ip2region.xdb # 离线 IP 库（下载，不进 git） +├── sdk/ # JS SDK +│ ├── package.json vitest.config.ts build.mjs +│ ├── src/{index,config,identity,context,transport,events,util}.ts +│ ├── test/*.test.ts +│ └── demo/index.html # 本地联调演示页 +├── admin/ # Art Design Pro 前端（Task 14 克隆） +│ └── src/{api,views,router/modules}/...（新增 tracker 相关文件） +├── deploy/ +│ ├── docker-compose.yml nginx.conf +│ └── README-deploy.md # 1Panel 部署步骤 +└── docs/superpowers/{specs,plans}/ +``` + +## 全局约定 + +- **响应包**：所有 `/api/admin/*` 返回 `{code: 200, data: ..., message: "ok"}`（适配 Art Design Pro 拦截器默认约定）；错误 `code` 为业务码（401 未登录/403 无权/4xx 参数/500 内部）。 +- **时间**：全库 UTC 存储（`USE_TZ=True`），展示层按 `settings.TIME_ZONE='Asia/Shanghai'`。 +- **采集跨域**：SDK 用 `Content-Type: text/plain` 发 JSON（简单请求免预检），服务端用 `PlainJSONParser` 解析；`django-cors-headers` + 动态白名单（DB 中 Site.domain_whitelist，60s 缓存）回 `Access-Control-Allow-Origin`。 +- **测试**：后端 `pytest`（SQLite 内存库，`pytest.ini` 指定 `--ds=config.settings_test`）；SDK `vitest` + happy-dom + fake timers。 +- **提交**：每任务一提交，`feat:/test:/chore:` 前缀。 +- **M1 明确不做**（防执行者自作主张加码）：SPA 路由、点击/滚动/表单/意图事件、客户(contact)体系、Redis、事件表分区迁移、npm 发布。 + +--- + +# Phase 0 · 骨架 + +### Task 1: Django 工程骨架 + +**Files:** +- Create: `server/requirements.txt`, `server/pytest.ini`, `server/manage.py`, `server/config/__init__.py`, `server/config/settings.py`, `server/config/urls.py`, `server/config/wsgi.py`, `server/tracker/__init__.py`, `server/tracker/apps.py`, `server/tracker/constants.py` + +- [ ] **Step 1: 建目录与依赖文件** + +`server/requirements.txt`: +``` +Django>=5.1,<5.3 +djangorestframework>=3.15 +djangorestframework-simplejwt>=5.3 +django-cors-headers>=4.4 +PyMySQL>=1.1 +ip2region>=2.0 +gunicorn>=22.0 +``` + +`server/config/__init__.py`（MySQL 驱动适配，Windows/容器免编译）： +```python +try: + import pymysql + pymysql.install_as_MySQLdb() +except ImportError: + pass +``` + +`server/pytest.ini`: +```ini +[pytest] +DJANGO_SETTINGS_MODULE = config.settings_test +python_files = test_*.py +addopts = -q +``` + +- [ ] **Step 2: 写 settings** + +`server/config/settings.py` 关键内容（完整写出）： +```python +from pathlib import Path +import os + +BASE_DIR = Path(__file__).resolve().parent.parent +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY", "dev-insecure") +DEBUG = os.environ.get("DJANGO_DEBUG", "1") == "1" +ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "*").split(",") + +INSTALLED_APPS = [ + "django.contrib.contenttypes", + "django.contrib.auth", + "corsheaders", + "rest_framework", + "tracker", +] +MIDDLEWARE = [ + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", +] +ROOT_URLCONF = "config.urls" +WSGI_APPLICATION = "config.wsgi.application" +TEMPLATES = [{ + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], "APP_DIRS": True, "OPTIONS": {"context_processors": []}, +}] + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": os.environ.get("DB_NAME", "tracker"), + "USER": os.environ.get("DB_USER", "root"), + "PASSWORD": os.environ.get("DB_PASSWORD", ""), + "HOST": os.environ.get("DB_HOST", "127.0.0.1"), + "PORT": os.environ.get("DB_PORT", "3306"), + "OPTIONS": {"charset": "utf8mb4"}, + } +} + +REST_FRAMEWORK = { + "DEFAULT_AUTHENTICATION_CLASSES": ("rest_framework_simplejwt.authentication.JWTAuthentication",), + "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), + "DEFAULT_THROTTLE_RATES": {"collect": "120/min", "admin": "600/min"}, + "NUM_PROXIES": 1, +} +SIMPLE_JWT = {"ACCESS_TOKEN_LIFETIME": timedelta(hours=12)} +CORS_ALLOW_ALL_ORIGINS = False +TIME_ZONE = "Asia/Shanghai" +USE_TZ = True +LANGUAGE_CODE = "zh-hans" +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" +IP2REGION_XDB = str(BASE_DIR / "tracker" / "data" / "ip2region.xdb") +``` +（`from datetime import timedelta` 放文件头；无 admin/session/contrib.staticfiles——管理台是独立 SPA。） + +`server/config/settings_test.py`: +```python +from .settings import * +DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}} +``` + +`server/config/urls.py`: +```python +urlpatterns = [] # Task 7 起逐任务填充 +``` +`server/tracker/constants.py`: +```python +EVENT_NAMES = {"page_view", "page_stay", "sdk_loaded"} +MAX_EVENTS_PER_BATCH = 50 +MAX_PROP_KEYS = 32 +MAX_STR_LEN = 500 +MAX_PATH_LEN = 500 +MAX_TITLE_LEN = 200 +CLOCK_SKEW_LATE_SEC = 120 # 超过即标 late +CLOCK_SKEW_REJECT_SEC = 86400 # 超过即拒绝 +SENSITIVE_KEY_PATTERNS = ("password", "passwd", "token", "secret", "card", "verif", "otp", "idcard") +URL_PARAM_DROP_PATTERNS = ("email", "phone", "mobile", "token", "key", "password", "verif") +SESSION_IDLE_MS = 30 * 60 * 1000 +``` + +- [ ] **Step 3: 建 Django 工程与 app 骨架** + +按上面文件清单创建 `manage.py`（标准模板）、`config/wsgi.py`（标准）、`tracker/apps.py`（name="tracker"）。 + +- [ ] **Step 4: 验证** + +```bash +cd server && python -m venv .venv && .venv/Scripts/pip install -r requirements.txt pytest pytest-django +.venv/Scripts/python manage.py check +``` +Expected: `System check identified no issues`。`pip freeze > requirements.lock.txt`（不含 pytest 两条则手工剔除 pytest 行，锁定运行依赖）。 + +- [ ] **Step 5: Commit** `chore: Django 工程骨架（config+tracker app+常量）` + +### Task 2: SDK 工程骨架 + +**Files:** +- Create: `sdk/package.json`, `sdk/tsconfig.json`, `sdk/vitest.config.ts`, `sdk/build.mjs`, `sdk/src/util.ts`, `sdk/test/util.test.ts` + +- [ ] **Step 1: 初始化** + +`sdk/package.json`: +```json +{ + "name": "ctk-sdk", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "build": "node build.mjs", + "test": "vitest run", + "checksize": "node build.mjs && gzip -c dist/v1.js | wc -c" + }, + "devDependencies": { + "esbuild": "^0.24.0", + "happy-dom": "^15.0.0", + "typescript": "^5.6.0", + "vitest": "^2.1.0" + } +} +``` +`sdk/vitest.config.ts`: +```ts +import { defineConfig } from "vitest/config"; +export default defineConfig({ + test: { environment: "happy-dom", setupFiles: [] }, +}); +``` +`sdk/build.mjs`: +```js +import { build } from "esbuild"; +await build({ + entryPoints: ["src/index.ts"], + bundle: true, minify: true, format: "iife", + outfile: "dist/v1.js", target: ["es2018"], +}); +``` +`sdk/src/util.ts`: +```ts +export function uuid(): string { + if (globalThis.crypto?.randomUUID) return crypto.randomUUID(); + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => { + const r = (Math.random() * 16) | 0; + return (c === "x" ? r : (r & 0x3) | 0x8).toString(16); + }); +} +``` + +- [ ] **Step 2: TDD uuid** + +`sdk/test/util.test.ts`: +```ts +import { describe, it, expect } from "vitest"; +import { uuid } from "../src/util"; +describe("uuid", () => { + it("生成 36 位且不重复", () => { + const a = uuid(), b = uuid(); + expect(a).toMatch(/^[0-9a-f-]{36}$/); + expect(a).not.toBe(b); + }); +}); +``` +```bash +cd sdk && pnpm install && pnpm test +``` +Expected: 1 passed。 + +- [ ] **Step 3: Commit** `chore: SDK 工程骨架（esbuild+vitest）` + +--- + +# Phase 1 · 数据模型与采集后端 + +### Task 3: 五个核心模型 + +**Files:** +- Create: `server/tracker/models.py`, `server/tracker/tests/test_models.py`, `server/tracker/migrations/__init__.py` + +- [ ] **Step 1: 写失败测试** + +`server/tracker/tests/test_models.py`: +```python +import pytest +from tracker.models import Site, Visitor, VisitSession, Event + +pytestmark = pytest.mark.django_db + +def test_create_site_generates_public_site_id(): + site = Site.objects.create(name="官网", domains=["www.example.com"]) + assert site.site_id.startswith("site_") and len(site.site_id) == 21 + assert site.is_active is True + +def test_visitor_first_touch_fields(): + site = Site.objects.create(name="s", domains=["a.com"]) + v = Visitor.objects.create(site=site, visitor_id="vid1", + first_referrer_domain="google.com", first_utm={"utm_source": "google"}, + first_landing_path="/p", country="美国", device_type="PC") + assert v.visit_count == 0 and v.active_days == 0 + +def test_event_unique_site_event_id(): + site = Site.objects.create(name="s", domains=["a.com"]) + v = Visitor.objects.create(site=site, visitor_id="vid1") + s = VisitSession.objects.create(site=site, visitor=v, session_id="sid1") + Event.objects.create(site=site, visitor=v, session=s, event_id="e1", + event_name="page_view", occurred_at="2026-09-15T10:00:00Z", path="/") + with pytest.raises(Exception): + Event.objects.create(site=site, visitor=v, session=s, event_id="e1", + event_name="page_view", occurred_at="2026-09-15T10:01:00Z", path="/") +``` + +- [ ] **Step 2: 跑测试确认失败** `cd server && .venv/Scripts/python -m pytest tracker/tests/test_models.py` Expected: ImportError（models 不存在）。 + +- [ ] **Step 3: 实现 models.py** + +```python +from django.db import models +import secrets + +def short_public_id() -> str: + return "site_" + secrets.token_urlsafe(12)[:16] + +class Site(models.Model): + name = models.CharField(max_length=100) + site_id = models.CharField(max_length=32, unique=True, default=short_public_id) + domains = models.JSONField(default=list) # 域名白名单 ["www.example.com"] + allow_no_origin = models.BooleanField(default=False) # 本地/demo 调试用 + is_active = models.BooleanField(default=True) + timezone = models.CharField(max_length=64, default="Asia/Shanghai") + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + class Meta: + db_table = "site" + +class Visitor(models.Model): + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="visitors") + visitor_id = models.CharField(max_length=64) + first_seen_at = models.DateTimeField(auto_now_add=True) + last_seen_at = models.DateTimeField(auto_now_add=True) + first_referrer_domain = models.CharField(max_length=200, blank=True, default="") + first_utm = models.JSONField(default=dict, blank=True) + first_landing_path = models.CharField(max_length=500, blank=True, default="") + device_type = models.CharField(max_length=16, blank=True, default="") + os = models.CharField(max_length=64, blank=True, default="") + browser = models.CharField(max_length=64, blank=True, default="") + language = models.CharField(max_length=32, blank=True, default="") + timezone_name = models.CharField(max_length=64, blank=True, default="") + screen = models.CharField(max_length=32, blank=True, default="") + last_ip = models.GenericIPAddressField(null=True, blank=True) + country = models.CharField(max_length=64, blank=True, default="") + province = models.CharField(max_length=64, blank=True, default="") + city = models.CharField(max_length=64, blank=True, default="") + visit_count = models.PositiveIntegerField(default=0) # 会话数（服务端按新会话递增） + active_days = models.PositiveIntegerField(default=0) # 活跃天数 + last_active_date = models.DateField(null=True, blank=True) + class Meta: + db_table = "visitor" + constraints = [models.UniqueConstraint(fields=["site", "visitor_id"], name="uq_site_visitor")] + indexes = [ + models.Index(fields=["site", "-last_seen_at"]), + models.Index(fields=["site", "country"]), + ] + +class VisitSession(models.Model): + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="sessions") + visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="sessions") + session_id = models.CharField(max_length=64) + started_at = models.DateTimeField() + ended_at = models.DateTimeField() + referrer_domain = models.CharField(max_length=200, blank=True, default="") + utm = models.JSONField(default=dict, blank=True) + landing_path = models.CharField(max_length=500, blank=True, default="") + event_count = models.PositiveIntegerField(default=0) + duration_ms = models.PositiveBigIntegerField(default=0) + class Meta: + db_table = "visit_session" + constraints = [models.UniqueConstraint(fields=["site", "session_id"], name="uq_site_session")] + indexes = [models.Index(fields=["visitor", "-started_at"])] + +class Event(models.Model): + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="events") + visitor = models.ForeignKey(Visitor, on_delete=models.CASCADE, related_name="events") + session = models.ForeignKey(VisitSession, on_delete=models.CASCADE, related_name="events", null=True) + event_id = models.CharField(max_length=64) + event_name = models.CharField(max_length=50) + occurred_at = models.DateTimeField() + received_at = models.DateTimeField(auto_now_add=True) + path = models.CharField(max_length=500, blank=True, default="") + title = models.CharField(max_length=200, blank=True, default="") + properties = models.JSONField(default=dict, blank=True) + ip = models.GenericIPAddressField(null=True, blank=True) + country = models.CharField(max_length=64, blank=True, default="") + province = models.CharField(max_length=64, blank=True, default="") + city = models.CharField(max_length=64, blank=True, default="") + late = models.BooleanField(default=False) + class Meta: + db_table = "event" + constraints = [models.UniqueConstraint(fields=["site", "event_id"], name="uq_site_event")] + indexes = [ + models.Index(fields=["site", "visitor", "-occurred_at"]), + models.Index(fields=["session", "occurred_at"]), + models.Index(fields=["site", "event_name", "-occurred_at"]), + ] + +class DailyStat(models.Model): + site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name="daily_stats") + date = models.DateField() + pv = models.PositiveIntegerField(default=0) + uv = models.PositiveIntegerField(default=0) + sessions = models.PositiveIntegerField(default=0) + new_visitors = models.PositiveIntegerField(default=0) + class Meta: + db_table = "daily_stat" + constraints = [models.UniqueConstraint(fields=["site", "date"], name="uq_site_date")] +``` + +- [ ] **Step 4: 迁移并跑测试** +```bash +.venv/Scripts/python manage.py makemigrations tracker +.venv/Scripts/python -m pytest tracker/tests/test_models.py +``` +Expected: 3 passed。 + +- [ ] **Step 5: Commit** `feat: 核心数据模型（Site/Visitor/VisitSession/Event/DailyStat）` + +### Task 4: 事件校验服务 validation.py + +**Files:** +- Create: `server/tracker/services/__init__.py`, `server/tracker/services/validation.py`, `server/tracker/tests/test_validation.py` + +- [ ] **Step 1: 失败测试**（覆盖：白名单事件名/敏感键拒绝/属性数与长度截断/URL 参数清洗/时间窗） + +`server/tracker/tests/test_validation.py` 核心用例： +```python +from datetime import datetime, timedelta, timezone as tz +from tracker.services.validation import validate_event, sanitize_url +from django.utils import timezone + +NOW = timezone.now() + +def _ev(**kw): + base = dict(event_id="e1", event_name="page_view", + occurred_at=NOW, path="/p", title="t", properties={}) + base.update(kw) + return base + +def test_ok_event(): + assert validate_event(_ev()) is None # None = 通过 + +def test_unknown_event_name(): + assert "event_name" in validate_event(_ev(event_name="xxx")) + +def test_sensitive_property_rejected(): + r = validate_event(_ev(properties={"user_password": "1"})) + assert "sensitive" in r + +def test_too_many_props_and_long_string(): + r = validate_event(_ev(properties={f"k{i}": "v" for i in range(40)})) + assert "properties" in r + r = validate_event(_ev(properties={"a": "x" * 600})) + assert "properties" in r + +def test_future_and_stale_time(): + r = validate_event(_ev(occurred_at=NOW + timedelta(days=2))) + assert "occurred_at" in r # 超窗拒绝 + assert validate_event(_ev(occurred_at=NOW - timedelta(hours=2))) is None # 窗内迟到可收 + +def test_sanitize_url_drops_sensitive_params(): + u = sanitize_url("/p?email=a@b.com&token=x&id=3&utm_source=google") + assert "email=" not in u and "token=" not in u and "utm_source=google" in u +``` + +- [ ] **Step 2: 跑测试确认失败** → ModuleNotFoundError。 + +- [ ] **Step 3: 实现** + +```python +import re +from datetime import timedelta +from django.utils import timezone +from tracker import constants as C + +_SENSITIVE = re.compile("|".join(C.SENSITIVE_KEY_PATTERNS), re.I) +_URL_DROP = re.compile("&?(%s)=[^&]*" % "|".join(C.URL_PARAM_DROP_PATTERNS), re.I) + +def sanitize_url(url: str) -> str: + return _URL_DROP.sub("", url).replace("?&", "?").rstrip("?") + +def validate_event(ev: dict) -> str | None: + """返回 None 表示通过；否则返回拒绝原因码。""" + if ev.get("event_name") not in C.EVENT_NAMES: + return "event_name_not_allowed" + if not ev.get("event_id") or len(ev["event_id"]) > 64: + return "invalid_event_id" + occ = ev.get("occurred_at") + if not isinstance(occ, datetime): + return "invalid_occurred_at" + skew = (timezone.now() - occ).total_seconds() + if abs(skew) > C.CLOCK_SKEW_REJECT_SEC: + return "occurred_at_out_of_window" + props = ev.get("properties") or {} + if not isinstance(props, dict) or len(props) > C.MAX_PROP_KEYS: + return "properties_invalid" + for k, v in props.items(): + if _SENSITIVE.search(str(k)): + return "sensitive_field" + if isinstance(v, str) and len(v) > C.MAX_STR_LEN: + return "properties_too_long" + if len(ev.get("path") or "") > C.MAX_PATH_LEN: + ev["path"] = ev["path"][:C.MAX_PATH_LEN] + if len(ev.get("title") or "") > C.MAX_TITLE_LEN: + ev["title"] = ev["title"][:C.MAX_TITLE_LEN] + if ev.get("path"): + ev["path"] = sanitize_url(ev["path"]) + return None +``` + +- [ ] **Step 4: 跑测试通过** → 6 passed。 + +- [ ] **Step 5: Commit** `feat: 事件校验服务（白名单/敏感键/时间窗/URL清洗）` + +### Task 5: geo.py 离线 IP 解析 + +**Files:** +- Create: `server/tracker/services/geo.py`, `server/tracker/tests/test_geo.py` +- Download: `server/tracker/data/ip2region.xdb`（[ip2region 官方仓库](https://github.com/lionsoul2014/ip2region) data/ip2region.xdb，~11MB，加入 .gitignore） + +- [ ] **Step 1: 失败测试** +```python +from tracker.services.geo import GeoResolver + +def test_private_ip_returns_local(): + r = GeoResolver() + assert r.resolve("192.168.1.5") is None # 内网不解析（后台显示"本地"） + assert r.resolve("127.0.0.1") is None + +def test_public_ip_parsed_or_none(monkeypatch): + r = GeoResolver() + # xdb 未下载/解析失败时优雅降级返回 None，不抛异常 + out = r.resolve("8.8.8.8") + assert out is None or set(out) == {"country", "province", "city"} +``` + +- [ ] **Step 2: 实现** +```python +import functools, ipaddress +from django.conf import settings + +class GeoResolver: + def __init__(self): + self._searcher = None + try: + from ip2region import XdbSearcher # pip ip2region；失败则降级 + self._searcher_cls = XdbSearcher + except Exception: + self._searcher_cls = None + + @functools.lru_cache(maxsize=10000) + def resolve(self, ip: str): + if not ip: + return None + try: + if ipaddress.ip_address(ip).is_private: + return None + except ValueError: + return None + if not self._searcher_cls: + return None + try: + with self._searcher_cls(contentBuff=open(settings.IP2REGION_XDB, "rb").read()) as s: + raw = s.search(ip) # "国家|区域|省|市|ISP" + parts = (raw or "").split("|") + return {"country": parts[0], "province": parts[2], "city": parts[3]} + except Exception: + return None +``` +注：若 `ip2region` pip 包 API 不同，以官方 README 为准适配 `XdbSearcher` 的调用方式；解析结果格式 `国家|区域|省|市|ISP`。中文区返回如 `中国|0|广东省|深圳市|电信`，海外如 `美国|0|加利福尼亚|旧金山|0`。 + +- [ ] **Step 3: 跑测试通过**（`server/.gitignore` 加 `tracker/data/*.xdb`） + +- [ ] **Step 4: Commit** `feat: ip2region 离线地理解析（优雅降级）` + +### Task 6: ingest.py 批量入库服务 + +**Files:** +- Create: `server/tracker/services/ingest.py`, `server/tracker/tests/test_ingest.py` + +**职责**：对已通过校验的批次执行——访客 get_or_create（首触来源只写一次）、会话 get_or_create（30 分钟兜底校验）、`bulk_create(ignore_conflicts=True)` 幂等写事件、更新访客聚合（visit_count/active_days/last_seen/last_ip/geo）。**测试用 SQLite，忽略 MySQL 差异。** + +- [ ] **Step 1: 失败测试**（核心场景全列）： +```python +from datetime import timedelta +from django.utils import timezone +from tracker.models import Site, Visitor, VisitSession, Event +from tracker.services.ingest import ingest_batch + +NOW = timezone.now() + +def _batch(events, vid="v1", sid="s1"): + return {"visitor_id": vid, "session_id": sid, "context": { + "referrer_domain": "google.com", "utm": {"utm_source": "google"}, + "path": "/landing", "device_type": "PC", "os": "Windows", + "browser": "Chrome", "language": "en-US", "screen": "1920x1080"}, + "events": events} + +def _ev(i, name="page_view", **kw): + d = dict(event_id=f"e{i}", event_name=name, occurred_at=NOW, + path="/p", title="T", properties={}) + d.update(kw) + return d + +def test_first_batch_creates_visitor_session_events(): + site = Site.objects.create(name="s", domains=["a.com"]) + r = ingest_batch(site, _batch([_ev(1), _ev(2)]), ip="8.8.8.8") + assert (r["accepted"], r["duplicated"], r["rejected"]) == (2, 0, []) + v = Visitor.objects.get(site=site, visitor_id="v1") + assert v.first_referrer_domain == "google.com" and v.visit_count == 1 + assert v.active_days == 1 and v.last_ip == "8.8.8.8" + assert Event.objects.filter(site=site).count() == 2 + +def test_duplicate_event_ids_counted_not_rewritten(): + site = Site.objects.create(name="s", domains=["a.com"]) + ingest_batch(site, _batch([_ev(1)])) + r = ingest_batch(site, _batch([_ev(1)])) # 重放 + assert r["duplicated"] == 1 and Event.objects.count() == 1 + +def test_new_session_increments_visit_count(): + site = Site.objects.create(name="s", domains=["a.com"]) + ingest_batch(site, _batch([_ev(1)], sid="s1")) + ingest_batch(site, _batch([_ev(2)], sid="s2")) + v = Visitor.objects.get(visitor_id="v1") + assert v.visit_count == 2 and VisitSession.objects.count() == 2 + +def test_server_session_backstop_splits_idle_gap(): + # SDK 权威；服务端仅当同一 session_id 事件间隔>30min 时切分出后续段（记新 session_id 后缀） + site = Site.objects.create(name="s", domains=["a.com"]) + ingest_batch(site, _batch([_ev(1, occurred_at=NOW - timedelta(minutes=90))], sid="s1")) + ingest_batch(site, _batch([_ev(2, occurred_at=NOW)], sid="s1")) + assert VisitSession.objects.count() == 2 # s1 与 s1#2 + +def test_rejected_events_reported_with_reason(): + site = Site.objects.create(name="s", domains=["a.com"]) + r = ingest_batch(site, _batch([_ev(1, event_name="bad"), _ev(2)])) + assert r["accepted"] == 1 and r["rejected"][0]["reason"] == "event_name_not_allowed" +``` + +- [ ] **Step 2: 跑测试确认失败** → ImportError。 + +- [ ] **Step 3: 实现** +```python +from datetime import timedelta +from django.db import transaction +from django.utils import timezone +from tracker import constants as C +from tracker.models import Visitor, VisitSession, Event +from tracker.services.validation import validate_event +from tracker.services.geo import GeoResolver + +_geo = GeoResolver() + +def _touch_visitor(site, vid, ctx, ip): + v, created = Visitor.objects.get_or_create( + site=site, visitor_id=vid, + defaults={"last_ip": ip, **{k: str(ctx.get(k, "") or "")[:64] + for k in ("device_type", "os", "browser", "language", "screen")}}, + ) + if created: + v.first_referrer_domain = (ctx.get("referrer_domain") or "")[:200] + v.first_utm = {k: str(val)[:100] for k, val in (ctx.get("utm") or {}).items()} + v.first_landing_path = (ctx.get("path") or "")[:500] + now = timezone.now() + v.last_seen_at = now + if ip: + v.last_ip = ip + g = _geo.resolve(ip) + if g: + v.country, v.province, v.city = g["country"], g["province"], g["city"] + if v.last_active_date != now.date(): + v.active_days += 1 + v.last_active_date = now.date() + v.save() + return v + +def _get_session(site, visitor, sid, ctx, occurred_at): + """会话权威在 SDK；服务端兜底：同 session_id 距上事件>30min 切分后缀段。""" + idle = timedelta(milliseconds=C.SESSION_IDLE_MS) + sess = VisitSession.objects.filter(site=site, session_id=sid).order_by("-started_at").first() + if sess and occurred_at - sess.ended_at > idle: + suffix = 2 + VisitSession.objects.filter( + site=site, session_id__startswith=sid + "#").count() + sid = f"{sid}#{suffix}" + sess = None + if not sess: + sess = VisitSession.objects.create( + site=site, visitor=visitor, session_id=sid, + started_at=occurred_at, ended_at=occurred_at, + referrer_domain=(ctx.get("referrer_domain") or "")[:200], + utm={k: str(val)[:100] for k, val in (ctx.get("utm") or {}).items()}, + landing_path=(ctx.get("path") or "")[:500]) + Visitor.objects.filter(pk=visitor.pk).update(visit_count=visitor.visit_count + 1) + visitor.visit_count += 1 + return sess + +@transaction.atomic +def ingest_batch(site, payload: dict, ip: str | None) -> dict: + now = timezone.now() + ctx = payload.get("context") or {} + events = payload.get("events") or [] + result = {"accepted": 0, "duplicated": 0, "rejected": []} + if not events or len(events) > C.MAX_EVENTS_PER_BATCH: + return {"accepted": 0, "duplicated": 0, + "rejected": [{"reason": "batch_size_invalid"}]} + visitor = _touch_visitor(site, payload["visitor_id"], ctx, ip) + geo = _geo.resolve(ip) if ip else None + ready, session_cache = [], {} + for i, ev in enumerate(events): + reason = validate_event(ev) + if reason: + result["rejected"].append({"index": i, "event_id": ev.get("event_id", ""), + "reason": reason}) + continue + occ = ev["occurred_at"] + sid = payload["session_id"] + sess = session_cache.get(sid) + if not sess: + sess = _get_session(site, visitor, sid, ctx, occ) + session_cache[sess.session_id] = sess + ready.append(Event( + site=site, visitor=visitor, session=sess, + event_id=ev["event_id"], event_name=ev["event_name"], + occurred_at=occ, path=ev.get("path", "")[:500], + title=ev.get("title", "")[:200], + properties=ev.get("properties") or {}, + ip=ip, late=abs((now - occ).total_seconds()) > C.CLOCK_SKEW_LATE_SEC, + country=geo["country"] if geo else "", + province=geo["province"] if geo else "", + city=geo["city"] if geo else "")) + if ready: + before = Event.objects.filter(site=site, + event_id__in=[e.event_id for e in ready]).values_list("event_id", flat=True) + existed = set(before) + created = Event.objects.bulk_create( + [e for e in ready if e.event_id not in existed], ignore_conflicts=True) + # bulk_create ignore_conflicts 不回传；用存在集 + 计数差算 duplicated + result["accepted"] = len(ready) - (len([e for e in ready if e.event_id in existed])) + result["duplicated"] = len([e for e in ready if e.event_id in existed]) + for sess in set(e.session for e in ready): + n = len([e for e in ready if e.session == sess]) + VisitSession.objects.filter(pk=sess.pk).update( + ended_at=max(e.occurred_at for e in ready if e.session == sess), + event_count=sess.event_count + n, + duration_ms=int(sum( + (e.ended and 0) or 0 for e in [] ) or + max(0, (max(x.occurred_at for x in ready if x.session == sess) + - min(x.occurred_at for x in ready if x.session == sess) + ).total_seconds() * 1000)) + return result +``` +**注意**：上面 `duration_ms` 段落含伪代码占位，执行者应实现为"本批事件给该会话追加的时长 = 该会话本批事件 occurred_at 极差毫秒数 + 既有 duration_ms"，即： +```python +sess_events = [e for e in ready if e.session_id_of(sess)] # 用 e.session.pk == sess.pk +span_ms = int((max(x.occurred_at for x in sess_events) + - min(x.occurred_at for x in sess_events)).total_seconds() * 1000) +VisitSession.objects.filter(pk=sess.pk).update( + ended_at=max(x.occurred_at for x in sess_events), + event_count=F("event_count") + len(sess_events), + duration_ms=F("duration_ms") + span_ms) +``` +（`from django.db.models import F`；同批多会话各自算。） + +- [ ] **Step 4: 跑测试通过** → 5 passed。修到绿为止。 + +- [ ] **Step 5: Commit** `feat: 批量入库服务（幂等/首触/会话兜底/聚合更新）` + +### Task 7: 采集接口 POST /api/collect/batch + +**Files:** +- Create: `server/tracker/api/__init__.py`, `server/tracker/api/collect.py`, `server/tracker/tests/test_collect.py`, Modify: `server/config/urls.py`, `server/config/settings.py`（CORS 动态白名单） + +- [ ] **Step 1: 失败测试**（要点：白名单域放行/非白名单 403/限流触发 429/textplain 解析/响应最小化/幂等重放） +```python +import json +from datetime import timezone as tz +from django.utils import timezone +from tracker.models import Site + +NOW = timezone.now() + +def _post(client, site, body, origin="https://www.a.com", extra=None): + return client.post("/api/collect/batch", data=json.dumps(body), + content_type="text/plain", HTTP_ORIGIN=origin, **(extra or {})) + +def _body(n=1, sid="s1", vid="v1"): + return {"site_id": "site_x", "visitor_id": vid, "session_id": sid, + "context": {"referrer_domain": "", "path": "/", "device_type": "PC"}, + "events": [{"event_id": f"e{i}", "event_name": "page_view", + "occurred_at": NOW.isoformat(), "path": "/", "title": "t", + "properties": {}} for i in range(n)]} + +def test_collect_ok_creates_events(db, client): + Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x") + r = _post(client, None, _body(2)) + assert r.status_code == 200 and r.json() == {"code": 200, "data": {"accepted": 2, "duplicated": 0}, "message": "ok"} + +def test_origin_not_whitelisted_rejected(db, client): + Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x") + r = _post(client, None, _body(1), origin="https://evil.com") + assert r.status_code == 403 and r.json()["code"] == 403 + +def test_inactive_site_rejected(db, client): + Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x", is_active=False) + assert _post(client, None, _body(1)).status_code == 403 + +def test_unknown_site_403_not_404(db, client): # 不泄露站点是否存在 + r = _post(client, None, _body(1)) + assert r.status_code == 403 + +def test_replay_duplicated(db, client): + Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x") + _post(client, None, _body(1)) + r = _post(client, None, _body(1)) + assert r.json()["data"]["duplicated"] == 1 + +def test_throttle_429(db, client, settings): + settings.REST_FRAMEWORK = {**settings.REST_FRAMEWORK, + "DEFAULT_THROTTLE_RATES": {"collect": "3/min"}} + Site.objects.create(name="s", domains=["www.a.com"], site_id="site_x") + for _ in range(3): + assert _post(client, None, _body(1)).status_code == 200 + assert _post(client, None, _body(1)).status_code == 429 +``` + +- [ ] **Step 2: 确认失败** → 404（无路由）。 + +- [ ] **Step 3: 实现** + +`server/tracker/api/collect.py`: +```python +import json +from datetime import datetime +from django.utils import timezone +from rest_framework.views import APIView +from rest_framework.response import Response +from rest_framework.permissions import AllowAny +from rest_framework.throttling import AnonRateThrottle +from rest_framework.parsers import BaseParser +from tracker.models import Site +from tracker.services.ingest import ingest_batch + +class PlainJSONParser(BaseParser): + media_type = "text/plain" + def parse(self, stream, *args, **kwargs): + return json.loads(stream.read().decode("utf-8") or "{}") + +class CollectThrottle(AnonRateThrottle): + scope = "collect" + +def _iso(dt): + if isinstance(dt, str): + s = dt.replace("Z", "+00:00") + d = datetime.fromisoformat(s) + return d if d.tzinfo else d.replace(tzinfo=timezone.utc) + return dt + +class CollectBatchView(APIView): + permission_classes = [AllowAny] + parser_classes = [PlainJSONParser] + throttle_classes = [CollectThrottle] + authentication_classes = [] + + def post(self, request): + origin = request.headers.get("Origin") or request.headers.get("Referer") or "" + site = self._match_site(request) + if site is None or not site.is_active: + return self._deny(403, "site_not_allowed") + host = (origin.split("//")[-1].split("/")[0].split(":")[0].lower() + if origin else "") + if host and host not in [d.lower() for d in site.domains]: + return self._deny(403, "origin_not_allowed") + if not host and not site.allow_no_origin: + return self._deny(403, "origin_required") + payload = request.data or {} + for ev in payload.get("events") or []: + if "occurred_at" in ev: + try: + ev["occurred_at"] = _iso(ev["occurred_at"]) + except ValueError: + return self._deny(400, "invalid_occurred_at") + if not payload.get("visitor_id") or not payload.get("session_id"): + return self._deny(400, "invalid_payload") + ip = request.META.get("REMOTE_ADDR") + r = ingest_batch(site, payload, ip) + return Response({"code": 200, + "data": {"accepted": r["accepted"], "duplicated": r["duplicated"]}, + "message": "ok"}) + + def _match_site(self, request): + sid = (request.data or {}).get("site_id") if isinstance(request.data, dict) else None + # site_id 在 body；为匹配 Origin 需先读 body——parser 已解析到 request.data + from tracker.models import Site as S + return S.objects.filter(site_id=sid).first() if sid else None + + def _deny(self, code, reason): + return Response({"code": code, "data": None, "message": reason}, status=code) +``` +`server/config/urls.py`: +```python +from django.urls import path +from tracker.api.collect import CollectBatchView +urlpatterns = [ + path("api/collect/batch", CollectBatchView.as_view()), +] +``` +CORS 动态白名单（`settings.py` 追加 + MIDDLEWARE 顺序保持 corsheaders 在前）： +```python +CORS_ALLOWED_ORIGIN_REGEXES = [] # 不用全局 +``` +并在 `tracker/api/collect.py` 出口处手动加头（简单直接，绕开 cors-headers 的静态配置）： +```python +# post() 成功分支 Response 前： +resp = Response({...}) +if origin: + resp["Access-Control-Allow-Origin"] = origin + resp["Access-Control-Allow-Headers"] = "content-type" +return resp +``` +另加 OPTIONS 预检支持：`CollectBatchView.options()` 返回同头 200。（SDK 用 text/plain 免预检，此处为兜底。） + +- [ ] **Step 4: 跑测试通过** → 6 passed。 + +- [ ] **Step 5: Commit** `feat: 采集接口（域名白名单/限流/textplain解析/幂等）` + +### Task 8: refresh_stats 命令 + stats.py + +**Files:** +- Create: `server/tracker/services/stats.py`, `server/tracker/management/__init__.py`, `server/tracker/management/commands/__init__.py`, `server/tracker/management/commands/refresh_stats.py`, `server/tracker/tests/test_stats.py` + +- [ ] **Step 1: 失败测试**：造 2 天数据（访客 v1 两天访问、v2 一天）→ `refresh(site, date)` 后 DailyStat 的 pv/uv/sessions/new_visitors 正确；uv 按访客去重、new_visitors 按 first_seen_at 当日计数。 +- [ ] **Step 2: 实现** `stats.py`： +```python +def refresh_daily(site, date) -> DailyStat: + from tracker.models import Event, VisitSession, Visitor, DailyStat + from django.db.models import Count + day_start = timezone.make_aware(datetime.combine(date, time.min)) + day_end = day_start + timedelta(days=1) + pv = Event.objects.filter(site=site, occurred_at__gte=day_start, occurred_at__lt=day_end).count() + uv = Visitor.objects.filter(site=site, sessions__started_at__gte=day_start, + sessions__started_at__lt=day_end).distinct().count() + sessions = VisitSession.objects.filter(site=site, started_at__gte=day_start, + started_at__lt=day_end).count() + new_v = Visitor.objects.filter(site=site, first_seen_at__gte=day_start, + first_seen_at__lt=day_end).count() + obj, _ = DailyStat.objects.update_or_create(site=site, date=date, + defaults={"pv": pv, "uv": uv, "sessions": sessions, "new_visitors": new_v}) + return obj +``` +`refresh_stats.py` 命令：`python manage.py refresh_stats --days 3`（回刷近 N 天，含今天）。 +**调度**：生产环境由 1Panel 计划任务每日 00:10 执行 `docker exec python manage.py refresh_stats --days 2`（写入 Task 22 的 README-deploy；不做每日调度则趋势图历史全为零）。 +- [ ] **Step 3: 测试通过，Commit** `feat: 日聚合刷新命令` + +### Task 9: JWT 认证接口 + +**Files:** +- Create: `server/tracker/api/auth.py`, `server/tracker/tests/test_auth.py`, Modify: `config/urls.py` + +- [ ] **Step 1: 失败测试**：`/api/admin/auth/token` 用户名密码 → access/refresh；错密码 401；`/api/admin/auth/me` 带 token 返回 username/is_admin（is_staff）；无 token 401；`/api/admin/users` 仅 is_staff 可 POST 建普通成员。 +- [ ] **Step 2: 实现**：`TokenObtainPairView` 子类（返回 `{code:200,data:{access,refresh},message:"ok"}` 包一层），`me` 返回 `{username, is_admin: user.is_staff}`；`UserCreateView`（IsAdminUser）`username/password` 创建 `is_staff=False` 用户。 +- [ ] **Step 3: 测试通过，Commit** `feat: JWT 登录与用户接口` + +### Task 10: 站点管理 API + +**Files:** +- Create: `server/tracker/api/sites.py`, `server/tracker/tests/test_sites.py`, Modify: `config/urls.py` + +- [ ] **Step 1: 失败测试**：CRUD（列表/新建自动生成 site_id/更新域名白名单/停用）；仅登录成员可访问，删除仅 admin；域名校验（去协议、转小写、非空列表）。 +- [ ] **Step 2: 实现**：DRF `APIView` 或轻量 `ModelViewSet` + 自行包 `{code,data,message}`。序列化输出含 `site_id/domains/is_active/allow_no_origin`。 +- [ ] **Step 3: 测试通过，Commit** `feat: 站点管理接口` + +### Task 11: 总览 Dashboard API + +**Files:** +- Create: `server/tracker/api/dashboard.py`, `server/tracker/tests/test_dashboard.py`, Modify: `config/urls.py` + +- [ ] **Step 1: 失败测试**（造数后断言）： +```python +def test_overview_shape(db, client, django_user_model): + # 造 2 访客/3 会话/10 事件，1 条在 5 分钟内（active_now=1） + r = client.get("/api/admin/dashboard/overview") + assert r.json()["data"]["today"] == {"pv": 10, "uv": 2, "sessions": 3, + "new_visitors": 2, "active_now": 1} + assert len(r.json()["data"]["trend"]) == 30 + assert r.json()["data"]["sources"][0]["source"] == "google.com" + assert r.json()["data"]["top_pages"][0]["path"] == "/pricing" +``` +- [ ] **Step 2: 实现**：`today` 用事件/会话表实时聚合（当天 Asia/Shanghai 边界）；`trend` 取 DailyStat 近 30 天（缺日补零；今天用实时值覆盖最后一位）；`sources` = 当天 VisitSession group by `referrer_domain`（空→"直接访问"）top8；`top_pages` = 当天 page_view group by path top8；`active_now` = Event `received_at > now-5min` distinct visitor。认证 + 成员权限。 +- [ ] **Step 3: 测试通过，Commit** `feat: 总览聚合接口` + +### Task 12: 访客列表/详情/时间线/会话 API + +**Files:** +- Create: `server/tracker/api/visitors.py`, `server/tracker/tests/test_visitors.py`, Modify: `config/urls.py` + +路由： +``` +GET /api/admin/visitors 列表（分页/筛选/搜索） +GET /api/admin/visitors/ 详情（档案+环境+统计+会话摘要列表） +GET /api/admin/visitors//timeline 跨天时间线（按日分组→会话→事件） +GET /api/admin/visitors//sessions/ 单会话页面路径 +``` + +- [ ] **Step 1: 失败测试**： + - 列表字段齐全：visitor_id/country/province/city/device_type/os/browser/first_seen_at/last_seen_at/visit_count/active_days/session_count/total_stay_ms/first_source/last_source/online；筛选 country/device_type/online=true/search=访客ID或IP前缀；按 last_seen_at 倒序分页（page_size 20）。 + - timeline 结构：`[{date:"2026-09-15", sessions:[{session_id, started_at, events:[{time,event_name,path,title,active_ms}]}]}]`，多天数据按日倒序、日内时间正序；默认近 30 天，`?days=` 可调，单次上限 1000 事件。 + - 会话详情：事件按 occurred_at 正序，含 active_ms（page_stay 的 properties.active_ms 就地并入相邻 page_view 或单列，见实现说明）。 +- [ ] **Step 2: 实现**：ORM 查询 + Python 分组；`total_stay_ms` = `Sum("sessions__duration_ms)` 注释；`first_source` = `first_referrer_domain or first_utm.utm_source or "直接访问"`；`last_source` 取最近会话同逻辑；`online` = last_seen_at > now-5min。**page_view 与 page_stay 合并展示**：时间线返回时若 page_stay 的 path 与前一条 page_view 相同，则把 `active_ms` 合入该 page_view（服务端做，前端省事）。 +- [ ] **Step 3: 测试通过，Commit** `feat: 访客列表/详情/时间线/会话接口` + +--- + +# Phase 2 · JS SDK（M1：多页网站） + +### Task 13: identity.ts + context.ts + +**Files:** +- Create: `sdk/src/identity.ts`, `sdk/src/context.ts`, `sdk/test/identity.test.ts`, `sdk/test/context.test.ts` + +- [ ] **Step 1: 失败测试**（happy-dom 提供 localStorage/document；fake timers 测 30 分钟切会话）： +```ts +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { getVisitorId, getSessionId, touchActivity } from "../src/identity"; + +describe("identity", () => { + beforeEach(() => localStorage.clear()); + it("visitor_id 持久化", () => { + expect(getVisitorId()).toBe(getVisitorId()); + expect(getVisitorId()).toMatch(/^[0-9a-f-]{36}$/); + }); + it("30 分钟无活动切新会话", () => { + vi.useFakeTimers(); + const s1 = getSessionId(); + vi.advanceTimersByTime(10 * 60_000); touchActivity(); + vi.advanceTimersByTime(10 * 60_000); + expect(getSessionId()).toBe(s1); // 20min 内有活动→同会话 + vi.advanceTimersByTime(31 * 60_000); // 再过 31min 无活动 + expect(getSessionId()).not.toBe(s1); + }); +}); +``` +context 测试：`parseUtm("?utm_source=google&utm_medium=cpc&other=1")` 只留 utm_*；`sanitizeUrl` 删 email/token 参数；`buildEnvContext()` 返回 device_type（UA 含 "Mobile" → "手机"）/os/browser/language/timezone/screen 键。 +- [ ] **Step 2: 实现** +```ts +// identity.ts +import { uuid } from "./util"; +const K = { vid: "ctk_vid", sid: "ctk_sid", act: "ctk_act" }; +const IDLE = 30 * 60_000; +export function getVisitorId(): string { + let v = localStorage.getItem(K.vid); + if (!v) { v = uuid(); localStorage.setItem(K.vid, v); } + return v; +} +export function touchActivity() { localStorage.setItem(K.act, String(Date.now())); } +export function getSessionId(): string { + const last = +(localStorage.getItem(K.act) || 0); + let s = localStorage.getItem(K.sid); + if (!s || Date.now() - last > IDLE) { s = uuid(); localStorage.setItem(K.sid, s); } + touchActivity(); + return s; +} +``` +`context.ts`：`buildEnvContext()`（UA 正则判 device_type/os/browser；`navigator.language`、`Intl.DateTimeFormat().resolvedOptions().timeZone`、`screen.width+"x"+screen.height`、`navigator.hardwareConcurrency`、`navigator.connection?.effectiveType`）；`parseUtm(search)`；`sanitizeUrl(path)`；`pageContext()` → `{path: sanitized, title: document.title, referrer: document.referrer, referrer_domain, utm}`。 +- [ ] **Step 3: 测试通过，Commit** `feat(sdk): 身份与环境上下文模块` + +### Task 14: transport.ts 批量传输 + +**Files:** +- Create: `sdk/src/transport.ts`, `sdk/src/config.ts`, `sdk/test/transport.test.ts` + +- [ ] **Step 1: 失败测试**（vi.stubGlobal fetch；fake timers）： + - 满 10 条即发；15 秒定时器到即发；payload 含 batch_id/site_id/visitor_id/session_id/events。 + - 500 失败→退避重试（500ms×2^n 抖动）最多 3 次→丢弃计数；400 不重试直接丢。 + - 429 → 重试。 + - `flushBeacon()` 调 `navigator.sendBeacon`（spy 断言 URL 与 body）。 + - 队列上限 200，超限丢最旧并计数。 +- [ ] **Step 2: 实现** +```ts +import { uuid } from "./util"; +import { getVisitorId, getSessionId } from "./identity"; + +export interface CtkConfig { siteId: string; endpoint: string; + batchSize?: number; flushMs?: number; maxRetry?: number; queueMax?: number } + +export class Transport { + private q: any[] = []; private timer: any = null; + private retry = 0; dropped = 0; + constructor(public cfg: Required) {} + push(ev: object) { + if (this.q.length >= this.cfg.queueMax) { this.q.shift(); this.dropped++; } + this.q.push(ev); + if (this.q.length >= this.cfg.batchSize) this.flush(); + else if (!this.timer) + this.timer = setTimeout(() => this.flush(), this.cfg.flushMs); + } + flush(beacon = false) { + if (this.timer) { clearTimeout(this.timer); this.timer = null; } + if (!this.q.length) return; + const events = this.q; this.q = []; + const body = JSON.stringify({ batch_id: uuid(), site_id: this.cfg.siteId, + visitor_id: getVisitorId(), session_id: getSessionId(), + sent_at: new Date().toISOString(), context: (globalThis as any).__ctk_ctx, + events }); + if (beacon && navigator.sendBeacon) { + navigator.sendBeacon(this.cfg.endpoint, new Blob([body], {type: "text/plain"})); + return; + } + this.send(body); + } + private async send(body: string) { + try { + const res = await fetch(this.cfg.endpoint, { method: "POST", + keepalive: true, headers: { "Content-Type": "text/plain" }, body }); + if (res.status === 400) { this.retry = 0; return; } // 参数错不重试 + if (!res.ok) throw new Error(String(res.status)); + this.retry = 0; + } catch { + if (this.retry++ < this.cfg.maxRetry) { + const delay = 500 * 2 ** this.retry * (0.8 + Math.random() * 0.4); + setTimeout(() => this.send(body), delay); + } else { this.retry = 0; this.dropped += 1; } + } + } +} +``` +`config.ts`：`mergeConfig(opts)` 填默认 `{batchSize:10, flushMs:15000, maxRetry:3, queueMax:200}`。 +- [ ] **Step 3: 测试通过，Commit** `feat(sdk): 批量传输（批量/退避/信标/丢包计数）` + +### Task 15: events.ts + index.ts 入口与打包 + +**Files:** +- Create: `sdk/src/events.ts`, `sdk/src/index.ts`, `sdk/test/events.test.ts`, `sdk/demo/index.html`, Modify: `.gitignore`（`sdk/node_modules`, `sdk/dist`） + +- [ ] **Step 1: 失败测试**： + - init 后 `sdk_loaded` 事件入队；`page_view` 含 path/title/referrer_domain/utm（utm 仅首页带）。 + - 有效停留：fake timers 下 `visible` 且有 pointerdown → 3 秒后 active_ms≈3000；`document.hidden=true` 后不再累计；隐藏时自动 flush beacon（page_stay 已入队）。 +- [ ] **Step 2: 实现** +```ts +// events.ts +import { Transport } from "./transport"; +import { pageContext } from "./context"; +import { uuid } from "./util"; + +export function startPageTracking(t: Transport) { + let activeMs = 0, lastAct = Date.now(); + const act = () => { lastAct = Date.now(); }; + ["pointerdown", "keydown", "wheel", "touchstart"].forEach(e => + document.addEventListener(e, act, { passive: true, capture: true })); + setInterval(() => { if (!document.hidden && Date.now() - lastAct < 30_000) activeMs += 1000; }, 1000); + const emit = (event_name: string, properties: any = {}) => { + const pc = pageContext(); + t.push({ event_id: uuid(), event_name, occurred_at: new Date().toISOString(), + path: pc.path, title: pc.title, properties }); + }; + emit("page_view", { referrer: document.referrer, + referrer_domain: pageContext().referrer_domain, utm: pageContext().utm }); + const close = () => { + if (activeMs > 0) emit("page_stay", { active_ms: activeMs, path: location.pathname }); + activeMs = 0; t.flush(true); + }; + document.addEventListener("visibilitychange", () => { if (document.hidden) close(); }); + window.addEventListener("pagehide", close); +} +``` +```ts +// index.ts —— 打包入口（IIFE 副作用） +import { mergeConfig } from "./config"; +import { Transport } from "./transport"; +import { startPageTracking } from "./events"; +import { buildEnvContext } from "./context"; + +const g = globalThis as any; +const q: any[][] = g.ctk?.q || []; +let transport: Transport | null = null; + +function dispatch(cmd: string, arg?: any) { + if (cmd === "init" && !transport) { + const cfg = mergeConfig(arg); + (globalThis as any).__ctk_ctx = buildEnvContext(); + transport = new Transport(cfg); + transport.push({ event_id: uuid(), event_name: "sdk_loaded", + occurred_at: new Date().toISOString(), + path: location.pathname, title: document.title, + properties: { sdk_version: "0.1.0" } }); + startPageTracking(transport); + } else if (cmd === "track" && transport) transport.push(arg); + else if (cmd === "flush" && transport) transport.flush(); +} +g.ctk = dispatch; +q.forEach(([c, a]) => dispatch(c, a)); +``` +（`uuid` import 补上；`sdk_loaded` 后 init 前队列已重放。**整个 dispatch 与 startPageTracking 必须包在 try/catch 中，任何异常静默吞掉（console.debug 级别记录，绝不冒泡到宿主页面）**——这是 spec §4.5 的硬性要求，Task 15 测试补一条：让 localStorage 抛异常时 init 不抛。） + +`sdk/demo/index.html`： +```html +CTK Demo + + +

Demo 页 1

去第 2 页 +``` +（`page2.html` 同构再建一份，标题"Demo 页 2"，验证多页跳转的会话连续性。） +``` +- [ ] **Step 3: 构建 + 体积门禁** +```bash +cd sdk && pnpm build && pnpm test +gzip -c dist/v1.js | wc -c # 必须 ≤ 15360（15KB），超标则精简（去 polyfill/减正则） +``` +- [ ] **Step 4: Commit** `feat(sdk): 页面浏览/有效停留/事件入口，打包≤15KB` + +### Task 16: SDK ↔ 后端本地集成验证（手动脚本） + +- [ ] **Step 1**: 后端起服务（本地 MySQL 或临时 SQLite settings_test 跑 `runserver` 需要落盘库——用环境变量指到开发 MySQL：`.env` 见 deploy README），`createsuperuser` 建 admin，`shell` 里建 Site（domains 含 127.0.0.1，`allow_no_origin=True`）。 +- [ ] **Step 2**: `cd sdk && python -m http.server 8080`，浏览器开 `http://127.0.0.1:8080/demo/`，点击、停留、切后台、跳 page2。 +- [ ] **Step 3**: 断言：`Event` 表出现 page_view/page_stay/sdk_loaded；Visitor.visit_count/active_days 正确；重复刷新不产生重复 event_id；后台 Django shell 查询验证（截图/记录到 PR 描述）。 +- [ ] **Step 4: Commit**（如有修复）`fix: SDK↔后端联调修复` + +--- + +# Phase 3 · Art Design Pro 管理后台 + +### Task 17: 模板落地与请求层适配 + +**Files:** +- Create: `admin/`（模板克隆），Modify: `admin/.env.development`, `admin/vite.config.ts`, `admin/src/utils/request/*`（按模板实际结构定位） + +- [ ] **Step 1**: 克隆模板 +```bash +git clone --depth 1 https://github.com/Daymychen/art-design-pro.git admin +cd admin && rm -rf .git && pnpm install && pnpm dev # 确认模板可跑 +pnpm clean:dev # 一键精简演示页（按提示输入 yes） +``` +- [ ] **Step 2**: `.env.development` 设 `VITE_API_URL=/api`；`vite.config.ts` 加 proxy `"/api": { target: "http://127.0.0.1:8000", changeOrigin: true }`。 +- [ ] **Step 3**: 改造模板 `src/utils/request`（或等价拦截器文件）：响应成功判定改为 `res.data.code === 200` 时 `return res.data.data`；401 跳登录页携带 redirect；token 从登录响应存入（沿用模板的 token 管理，替换其 mock 登录为真实 API）。 +- [ ] **Step 4**: 登录页接 `/api/admin/auth/token`（用户名/密码 → access/refresh），登录成功进布局页。手动验证：错误密码报错、正确 token 进入首页。 +- [ ] **Step 5: Commit** `feat(admin): 模板精简+请求层适配 Django JWT` + +### Task 18: api 模块 + 路由注册 + +**Files:** +- Create: `admin/src/api/tracker/{auth,dashboard,visitors,sites}.ts`, `admin/src/router/modules/tracker.ts`（命名按模板路由规范） + +- [ ] **Step 1**: api 模块（示例 visitors.ts，其余同构）： +```ts +import request from "@/utils/request"; +export interface VisitorRow { visitor_id: string; country: string; province: string; + city: string; device_type: string; os: string; browser: string; + first_seen_at: string; last_seen_at: string; visit_count: number; + active_days: number; session_count: number; total_stay_ms: number; + first_source: string; last_source: string; online: boolean } +export const getVisitors = (params: any) => + request.get<{ list: VisitorRow[]; total: number }>("/admin/visitors", { params }); +export const getVisitor = (vid: string) => request.get(`/admin/visitors/${vid}`); +export const getTimeline = (vid: string, days = 30) => + request.get(`/admin/visitors/${vid}/timeline`, { params: { days } }); +export const getSession = (vid: string, sid: string) => + request.get(`/admin/visitors/${vid}/sessions/${sid}`); +``` +- [ ] **Step 2**: 路由模块注册 4 页（总览/访客列表/访客详情(hidden)/站点管理），菜单名中文，icon 任选模板内置。 +- [ ] **Step 3: Commit** `feat(admin): tracker api 模块与路由` + +### Task 19: 站点管理页 + 总览 Dashboard 页 + +**Files:** +- Create: `admin/src/views/tracker/site/index.vue`, `admin/src/views/tracker/dashboard/index.vue` + +- [ ] **Step 1**: 站点页：Element Plus 表格（名称/site_id 复制按钮/域名列表 tag/状态开关）+ 新建编辑 Dialog（域名校验转小写）+ 停用开关调 PATCH。手动验证增删改。 +- [ ] **Step 2**: Dashboard 页：5 张统计卡（今日 PV/UV/会话/新访客/在线）、ECharts 30 天趋势折线（pv/uv/sessions 三线）、来源饼图 top8、热门页面表格。用模板自带图表组件或直接 ECharts。手动对拍后端返回。 +- [ ] **Step 3: Commit** `feat(admin): 站点管理与总览页` + +### Task 20: 访客列表页 + +**Files:** +- Create: `admin/src/views/tracker/visitor/index.vue` + +- [ ] **Step 1**: 表格列：访客ID(截断+复制)/国旗 emoji+国家·省市/设备(PC·Win·Chrome 一行小字)/首访/最近访问/访问次数/活跃天数/总停留(格式化 x分x秒)/首触来源/最近来源/在线绿点。筛选栏：国家输入/设备下拉/来源输入/在线开关/关键词(访客ID·IP 前缀)/日期范围。分页 20/页，后端排序 last_seen_at desc。行点击 → 详情页。 +- [ ] **Step 2**: 手动验证（用 Task 16 联调产生的真实数据）。 +- [ ] **Step 3: Commit** `feat(admin): 访客列表页` + +### Task 21: 访客详情页（时间线） + +**Files:** +- Create: `admin/src/views/tracker/visitor/detail.vue` + +- [ ] **Step 1**: 布局：顶部档案卡（访客ID/国家·城市/IP/设备环境 grid/首触来源 utm 链/统计 4 数字）+ 时间线主体（`el-timeline`，日期分组倒序；组内会话块：起止时间+来源+时长，事件行：`10:24 · 看了 /pricing · 停留 2分18秒`，事件 icon 按类型区分 page_view/page_stay 合并行/sdk_loaded）+ 顶部"会话"Tab 切换会话列表（含页面路径视图：路径序列 + 各页停留）。 +- [ ] **Step 2**: 停留格式化工具（`formatStay(ms)`）；空态/加载态；days 切换（7/30/90）。 +- [ ] **Step 3**: 手动验证（真实数据跨天回看：Day1 与 Day2 各一组）。**这是 M1 验收页面。** +- [ ] **Step 4: Commit** `feat(admin): 访客详情跨天时间线` + +--- + +# Phase 4 · 部署与验收 + +### Task 22: Docker 部署产物 + +**Files:** +- Create: `server/Dockerfile`, `deploy/docker-compose.yml`, `deploy/nginx.conf`, `deploy/README-deploy.md`, Modify: `.gitignore` + +- [ ] **Step 1**: `server/Dockerfile`： +```dockerfile +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +COPY . . +RUN python manage.py collectstatic --noinput 2>/dev/null || true +EXPOSE 8000 +CMD ["sh", "-c", "python manage.py migrate && gunicorn config.wsgi:application -b 0.0.0.0:8000 -w 3 --forwarded-allow-ips=*"] +``` +- [ ] **Step 2**: `deploy/nginx.conf` 关键段（compose 服务名统一为 `web`）：`location /api/ { proxy_pass http://web:8000; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; }`（配合 DRF `NUM_PROXIES=1` 取真实 IP）；`location /sdk/ { alias /var/www/sdk/; }`（**静态别名**，放 `sdk/dist/v1.js`，不走代理）；`location / { root /var/www/admin; try_files $uri $uri/ /index.html; }`（SPA fallback）。 +- [ ] **Step 3**: `docker-compose.yml`：`web`(build server, env_file .env, 依赖外部 MySQL——1Panel 已有容器，compose 里不建 MySQL，`DB_HOST` 填宿主内网地址)、`nginx`(挂 admin/dist 与 sdk/dist、80/443)。`.env.example` 列全环境变量。 +- [ ] **Step 4**: `README-deploy.md`：1Panel 步骤（建 MySQL 库 → Compose 部署 → 进容器 `createsuperuser` → 上传 xdb → 建站点拿 site_id → 目标网站贴接入代码）；目标网站接入片段（同 sdk/demo，endpoint 换正式域名）。 +- [ ] **Step 5: Commit** `chore: Docker/Nginx 部署产物与 1Panel 指南` + +### Task 23: M1 端到端验收（真实网站） + +- [ ] **Step 1**: 选一个真实网站（用户的 WordPress 站或 demo 站）贴接入代码，白名单填其域名。 +- [ ] **Step 2**: 按验收清单逐项过（写入 PR/交付记录）： + - [ ] 新访客首次访问 → 后台访客列表出现，环境/来源/地理正确 + - [ ] 浏览 3 个页面各停留可感知时长 → 时间线 3 条 page_view 且 active_ms 合理 + - [ ] 隔天（或改本机日期/用当天不同会话模拟）再访 → 同 visitor_id、visit_count+1、时间线新增日期组 + - [ ] 首触来源 Google/带 UTM 链接进入 → first_referrer_domain/first_utm 正确 + - [ ] 直接刷新/断网重试/重复上报 → 无重复事件（event_id 幂等） + - [ ] 非白名单域 curl 伪造上报 → 403；超频 → 429 + - [ ] Dashboard 数字与访客明细对得上；总停留=各会话之和 + - [ ] SDK 对页面加载性能无可感知影响（DevTools Network 确认 async + ≤15KB） +- [ ] **Step 3**: 全部通过 → `git tag m1-beta`，Commit 验收记录 `docs: M1 验收通过` + +--- + +## 执行注意 + +- Task 6 ingest 的 `duration_ms` 以"计划内正确版本"实现（F 表达式），不要照抄占位伪码。 +- Art Design Pro 模板内部组件 API 以克隆下来的实际代码为准（Task 17 后先读 `src/views` 现存示例再写页面）；本计划页面代码是基线，允许按模板约定适配，**不允许**砍功能。 +- ip2region pip 包若与计划 API 不符，参照其 README 调整 `geo.py`，测试断言不变。 +- 所有中文文案直接用于界面；代码注释保持中文、少量、说明意图。 +- 每任务完成即提交；测试红→绿→提交，禁止跳测试。