diff --git a/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md b/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md index b8685b7..2381368 100644 --- a/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md +++ b/docs/superpowers/plans/2026-09-15-m1-data-collection-foundation.md @@ -130,22 +130,26 @@ TEMPLATES = [{ "DIRS": [], "APP_DIRS": True, "OPTIONS": {"context_processors": []}, }] -DATABASES = { - "default": { - "ENGINE": "django.db.backends.mysql", - "NAME": os.environ.get("DB_NAME", "tracker"), - "USER": os.environ.get("DB_USER", "root"), - "PASSWORD": os.environ.get("DB_PASSWORD", ""), - "HOST": os.environ.get("DB_HOST", "127.0.0.1"), - "PORT": os.environ.get("DB_PORT", "3306"), - "OPTIONS": {"charset": "utf8mb4"}, +if os.environ.get("DB_ENGINE", "mysql") == "sqlite": # 本地无 MySQL 的开发模式 + DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3"}} +else: + DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": os.environ.get("DB_NAME", "tracker"), + "USER": os.environ.get("DB_USER", "root"), + "PASSWORD": os.environ.get("DB_PASSWORD", ""), + "HOST": os.environ.get("DB_HOST", "127.0.0.1"), + "PORT": os.environ.get("DB_PORT", "3306"), + "OPTIONS": {"charset": "utf8mb4"}, + } } -} REST_FRAMEWORK = { "DEFAULT_AUTHENTICATION_CLASSES": ("rest_framework_simplejwt.authentication.JWTAuthentication",), "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), - "DEFAULT_THROTTLE_RATES": {"collect": "120/min", "admin": "600/min"}, + "DEFAULT_THROTTLE_RATES": {"collect": "120/min", "admin": "600/min", "login": "10/min"}, "NUM_PROXIES": 1, } SIMPLE_JWT = {"ACCESS_TOKEN_LIFETIME": timedelta(hours=12)} @@ -181,6 +185,7 @@ CLOCK_SKEW_REJECT_SEC = 86400 # 超过即拒绝 SENSITIVE_KEY_PATTERNS = ("password", "passwd", "token", "secret", "card", "verif", "otp", "idcard") URL_PARAM_DROP_PATTERNS = ("email", "phone", "mobile", "token", "key", "password", "verif") SESSION_IDLE_MS = 30 * 60 * 1000 +BOT_UA_KEYWORDS = ("bot", "spider", "crawl", "slurp", "bingpreview", "headless", "lighthouse") ``` - [ ] **Step 3: 建 Django 工程与 app 骨架** @@ -357,6 +362,7 @@ class Visitor(models.Model): visit_count = models.PositiveIntegerField(default=0) # 会话数(服务端按新会话递增) active_days = models.PositiveIntegerField(default=0) # 活跃天数 last_active_date = models.DateField(null=True, blank=True) + is_bot = models.BooleanField(default=False) # UA 爬虫标记,报表默认排除 class Meta: db_table = "visitor" constraints = [models.UniqueConstraint(fields=["site", "visitor_id"], name="uq_site_visitor")] @@ -375,7 +381,8 @@ class VisitSession(models.Model): utm = models.JSONField(default=dict, blank=True) landing_path = models.CharField(max_length=500, blank=True, default="") event_count = models.PositiveIntegerField(default=0) - duration_ms = models.PositiveBigIntegerField(default=0) + duration_ms = models.PositiveBigIntegerField(default=0) # 会话内事件时间跨度 + stay_ms = models.PositiveBigIntegerField(default=0) # 有效停留累计(page_stay active_ms 之和) class Meta: db_table = "visit_session" constraints = [models.UniqueConstraint(fields=["site", "session_id"], name="uq_site_session")] @@ -397,6 +404,7 @@ class Event(models.Model): province = models.CharField(max_length=64, blank=True, default="") city = models.CharField(max_length=64, blank=True, default="") late = models.BooleanField(default=False) + is_bot = models.BooleanField(default=False) class Meta: db_table = "event" constraints = [models.UniqueConstraint(fields=["site", "event_id"], name="uq_site_event")] @@ -706,7 +714,7 @@ def _get_session(site, visitor, sid, ctx, occurred_at): return sess @transaction.atomic -def ingest_batch(site, payload: dict, ip: str | None) -> dict: +def ingest_batch(site, payload: dict, ip: str | None, ua: str = "") -> dict: now = timezone.now() ctx = payload.get("context") or {} events = payload.get("events") or [] @@ -716,6 +724,9 @@ def ingest_batch(site, payload: dict, ip: str | None) -> dict: "rejected": [{"reason": "batch_size_invalid"}]} visitor = _touch_visitor(site, payload["visitor_id"], ctx, ip) geo = _geo.resolve(ip) if ip else None + is_bot = any(k in (ua or "").lower() for k in C.BOT_UA_KEYWORDS) + if is_bot: + Visitor.objects.filter(pk=visitor.pk).update(is_bot=True) ready, session_cache = [], {} for i, ev in enumerate(events): reason = validate_event(ev) @@ -738,7 +749,8 @@ def ingest_batch(site, payload: dict, ip: str | None) -> dict: ip=ip, late=abs((now - occ).total_seconds()) > C.CLOCK_SKEW_LATE_SEC, country=geo["country"] if geo else "", province=geo["province"] if geo else "", - city=geo["city"] if geo else "")) + city=geo["city"] if geo else "", + is_bot=is_bot)) if ready: before = Event.objects.filter(site=site, event_id__in=[e.event_id for e in ready]).values_list("event_id", flat=True) @@ -760,17 +772,24 @@ def ingest_batch(site, payload: dict, ip: str | None) -> dict: ).total_seconds() * 1000)) return result ``` -**注意**:上面 `duration_ms` 段落含伪代码占位,执行者应实现为"本批事件给该会话追加的时长 = 该会话本批事件 occurred_at 极差毫秒数 + 既有 duration_ms",即: +**注意(执行者必读——上面代码 `duration_ms` 段是占位,按以下正确版本实现)**: ```python -sess_events = [e for e in ready if e.session_id_of(sess)] # 用 e.session.pk == sess.pk -span_ms = int((max(x.occurred_at for x in sess_events) - - min(x.occurred_at for x in sess_events)).total_seconds() * 1000) -VisitSession.objects.filter(pk=sess.pk).update( - ended_at=max(x.occurred_at for x in sess_events), - event_count=F("event_count") + len(sess_events), - duration_ms=F("duration_ms") + span_ms) +from django.db.models import F, Greatest + +for sess in {e.session for e in ready}: + se = [e for e in ready if e.session.pk == sess.pk] + new_end = max(x.occurred_at for x in se) + span_ms = int((new_end - min(x.occurred_at for x in se)).total_seconds() * 1000) + stay_add = sum(int(e.properties.get("active_ms", 0) or 0) + for e in se if e.event_name == "page_stay") + VisitSession.objects.filter(pk=sess.pk).update( + ended_at=Greatest(F("ended_at"), new_end), # 只前进不回退,防乱序批次改早结束时间 + event_count=F("event_count") + len(se), + duration_ms=F("duration_ms") + span_ms, + stay_ms=F("stay_ms") + stay_add) # 有效停留单独累计,供访客"总停留"取数 ``` -(`from django.db.models import F`;同批多会话各自算。) +- 同批多会话各自按上式计算。 +- `duplicated` 计数允许并发近似(唯一约束保证绝不重复入库,仅统计口径可能差 1-2,M1 接受)。 - [ ] **Step 4: 跑测试通过** → 5 passed。修到绿为止。 @@ -892,8 +911,8 @@ class CollectBatchView(APIView): return self._deny(400, "invalid_occurred_at") if not payload.get("visitor_id") or not payload.get("session_id"): return self._deny(400, "invalid_payload") - ip = request.META.get("REMOTE_ADDR") - r = ingest_batch(site, payload, ip) + ip = self._client_ip(request) + r = ingest_batch(site, payload, ip, request.headers.get("User-Agent", "")) return Response({"code": 200, "data": {"accepted": r["accepted"], "duplicated": r["duplicated"]}, "message": "ok"}) @@ -904,6 +923,13 @@ class CollectBatchView(APIView): from tracker.models import Site as S return S.objects.filter(site_id=sid).first() if sid else None + def _client_ip(self, request): + """nginx 前置时取 X-Forwarded-For 最后一段(与 DRF NUM_PROXIES=1 一致;套 CDN 时按层数前移)。""" + xff = request.META.get("HTTP_X_FORWARDED_FOR", "") + if xff: + return xff.split(",")[-1].strip() + return request.META.get("REMOTE_ADDR") + def _deny(self, code, reason): return Response({"code": code, "data": None, "message": reason}, status=code) ``` @@ -947,9 +973,10 @@ def refresh_daily(site, date) -> DailyStat: from django.db.models import Count day_start = timezone.make_aware(datetime.combine(date, time.min)) day_end = day_start + timedelta(days=1) - pv = Event.objects.filter(site=site, occurred_at__gte=day_start, occurred_at__lt=day_end).count() - uv = Visitor.objects.filter(site=site, sessions__started_at__gte=day_start, - sessions__started_at__lt=day_end).distinct().count() + pv = Event.objects.filter(site=site, is_bot=False, + occurred_at__gte=day_start, occurred_at__lt=day_end).count() + uv = Event.objects.filter(site=site, is_bot=False, occurred_at__gte=day_start, + occurred_at__lt=day_end).values("visitor").distinct().count() sessions = VisitSession.objects.filter(site=site, started_at__gte=day_start, started_at__lt=day_end).count() new_v = Visitor.objects.filter(site=site, first_seen_at__gte=day_start, @@ -968,7 +995,7 @@ def refresh_daily(site, date) -> DailyStat: - Create: `server/tracker/api/auth.py`, `server/tracker/tests/test_auth.py`, Modify: `config/urls.py` - [ ] **Step 1: 失败测试**:`/api/admin/auth/token` 用户名密码 → access/refresh;错密码 401;`/api/admin/auth/me` 带 token 返回 username/is_admin(is_staff);无 token 401;`/api/admin/users` 仅 is_staff 可 POST 建普通成员。 -- [ ] **Step 2: 实现**:`TokenObtainPairView` 子类(返回 `{code:200,data:{access,refresh},message:"ok"}` 包一层),`me` 返回 `{username, is_admin: user.is_staff}`;`UserCreateView`(IsAdminUser)`username/password` 创建 `is_staff=False` 用户。 +- [ ] **Step 2: 实现**:`TokenObtainPairView` 子类(返回 `{code:200,data:{access,refresh},message:"ok"}` 包一层),`me` 返回 `{username, is_admin: user.is_staff}`;`UserCreateView`(IsAdminUser)`username/password` 创建 `is_staff=False` 用户。登录接口挂 `ScopedRateThrottle` scope=`login`(10/min/IP,防爆破;settings 已含 rate)。 - [ ] **Step 3: 测试通过,Commit** `feat: JWT 登录与用户接口` ### Task 10: 站点管理 API @@ -996,7 +1023,7 @@ def test_overview_shape(db, client, django_user_model): assert r.json()["data"]["sources"][0]["source"] == "google.com" assert r.json()["data"]["top_pages"][0]["path"] == "/pricing" ``` -- [ ] **Step 2: 实现**:`today` 用事件/会话表实时聚合(当天 Asia/Shanghai 边界);`trend` 取 DailyStat 近 30 天(缺日补零;今天用实时值覆盖最后一位);`sources` = 当天 VisitSession group by `referrer_domain`(空→"直接访问")top8;`top_pages` = 当天 page_view group by path top8;`active_now` = Event `received_at > now-5min` distinct visitor。认证 + 成员权限。 +- [ ] **Step 2: 实现**:`today` 用事件/会话表实时聚合(当天 Asia/Shanghai 边界);`trend` 取 DailyStat 近 30 天(缺日补零;今天用实时值覆盖最后一位);`sources` = 当天 VisitSession group by `referrer_domain`(空→"直接访问")top8;`top_pages` = 当天 page_view group by path top8;`active_now` = Event `received_at > now-5min` distinct visitor。认证 + 成员权限。**所有聚合排除 `is_bot=True`(爬虫不进 Dashboard)。** - [ ] **Step 3: 测试通过,Commit** `feat: 总览聚合接口` ### Task 12: 访客列表/详情/时间线/会话 API @@ -1016,7 +1043,7 @@ GET /api/admin/visitors//sessions/ 单会话页面路 - 列表字段齐全:visitor_id/country/province/city/device_type/os/browser/first_seen_at/last_seen_at/visit_count/active_days/session_count/total_stay_ms/first_source/last_source/online;筛选 country/device_type/online=true/search=访客ID或IP前缀;按 last_seen_at 倒序分页(page_size 20)。 - timeline 结构:`[{date:"2026-09-15", sessions:[{session_id, started_at, events:[{time,event_name,path,title,active_ms}]}]}]`,多天数据按日倒序、日内时间正序;默认近 30 天,`?days=` 可调,单次上限 1000 事件。 - 会话详情:事件按 occurred_at 正序,含 active_ms(page_stay 的 properties.active_ms 就地并入相邻 page_view 或单列,见实现说明)。 -- [ ] **Step 2: 实现**:ORM 查询 + Python 分组;`total_stay_ms` = `Sum("sessions__duration_ms)` 注释;`first_source` = `first_referrer_domain or first_utm.utm_source or "直接访问"`;`last_source` 取最近会话同逻辑;`online` = last_seen_at > now-5min。**page_view 与 page_stay 合并展示**:时间线返回时若 page_stay 的 path 与前一条 page_view 相同,则把 `active_ms` 合入该 page_view(服务端做,前端省事)。 +- [ ] **Step 2: 实现**:ORM 查询 + Python 分组;`total_stay_ms` = `Sum("sessions__stay_ms")` 注解(**有效停留口径,不用 duration_ms 跨度**);`first_source` = `first_referrer_domain or first_utm.utm_source or "直接访问"`;`last_source` 取最近会话同逻辑;`online` = last_seen_at > now-5min;列表默认过滤 `is_bot=False`(`?show_bot=1` 可查爬虫)。**page_view 与 page_stay 合并展示**:时间线返回时若 page_stay 的 path 与前一条 page_view 相同,则把 `active_ms` 合入该 page_view(服务端做,前端省事)。 - [ ] **Step 3: 测试通过,Commit** `feat: 访客列表/详情/时间线/会话接口` --- @@ -1221,7 +1248,7 @@ gzip -c dist/v1.js | wc -c # 必须 ≤ 15360(15KB),超标则精简( ### Task 16: SDK ↔ 后端本地集成验证(手动脚本) -- [ ] **Step 1**: 后端起服务(本地 MySQL 或临时 SQLite settings_test 跑 `runserver` 需要落盘库——用环境变量指到开发 MySQL:`.env` 见 deploy README),`createsuperuser` 建 admin,`shell` 里建 Site(domains 含 127.0.0.1,`allow_no_origin=True`)。 +- [ ] **Step 1**: 后端起服务——本地无 MySQL 用开发模式:`DB_ENGINE=sqlite python manage.py runserver`(生成 db.sqlite3);有 MySQL 则按 `.env` 连接。`createsuperuser` 建 admin,`shell` 里建 Site(domains 含 127.0.0.1,`allow_no_origin=True`)。 - [ ] **Step 2**: `cd sdk && python -m http.server 8080`,浏览器开 `http://127.0.0.1:8080/demo/`,点击、停留、切后台、跳 page2。 - [ ] **Step 3**: 断言:`Event` 表出现 page_view/page_stay/sdk_loaded;Visitor.visit_count/active_days 正确;重复刷新不产生重复 event_id;后台 Django shell 查询验证(截图/记录到 PR 描述)。 - [ ] **Step 4: Commit**(如有修复)`fix: SDK↔后端联调修复` @@ -1320,7 +1347,7 @@ CMD ["sh", "-c", "python manage.py migrate && gunicorn config.wsgi:application - ``` - [ ] **Step 2**: `deploy/nginx.conf` 关键段(compose 服务名统一为 `web`):`location /api/ { proxy_pass http://web:8000; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; }`(配合 DRF `NUM_PROXIES=1` 取真实 IP);`location /sdk/ { alias /var/www/sdk/; }`(**静态别名**,放 `sdk/dist/v1.js`,不走代理);`location / { root /var/www/admin; try_files $uri $uri/ /index.html; }`(SPA fallback)。 - [ ] **Step 3**: `docker-compose.yml`:`web`(build server, env_file .env, 依赖外部 MySQL——1Panel 已有容器,compose 里不建 MySQL,`DB_HOST` 填宿主内网地址)、`nginx`(挂 admin/dist 与 sdk/dist、80/443)。`.env.example` 列全环境变量。 -- [ ] **Step 4**: `README-deploy.md`:1Panel 步骤(建 MySQL 库 → Compose 部署 → 进容器 `createsuperuser` → 上传 xdb → 建站点拿 site_id → 目标网站贴接入代码);目标网站接入片段(同 sdk/demo,endpoint 换正式域名)。 +- [ ] **Step 4**: `README-deploy.md`:**前置构建**(`cd admin && pnpm build`、`cd sdk && pnpm build` 产出 dist 供挂载)→ 1Panel 步骤(建 MySQL 库 → Compose 部署 → 进容器 `createsuperuser` → 上传 xdb → 1Panel 网站功能给 API 域名挂 HTTPS 证书 → 建站点拿 site_id → 目标网站贴接入代码)→ 目标网站接入片段(同 sdk/demo,endpoint 换正式域名)→ **若目标站套 CDN/WAF,X-Forwarded-For 层数变化需同步调整 `_client_ip` 与 `NUM_PROXIES`**。 - [ ] **Step 5: Commit** `chore: Docker/Nginx 部署产物与 1Panel 指南` ### Task 23: M1 端到端验收(真实网站) @@ -1329,7 +1356,7 @@ CMD ["sh", "-c", "python manage.py migrate && gunicorn config.wsgi:application - - [ ] **Step 2**: 按验收清单逐项过(写入 PR/交付记录): - [ ] 新访客首次访问 → 后台访客列表出现,环境/来源/地理正确 - [ ] 浏览 3 个页面各停留可感知时长 → 时间线 3 条 page_view 且 active_ms 合理 - - [ ] 隔天(或改本机日期/用当天不同会话模拟)再访 → 同 visitor_id、visit_count+1、时间线新增日期组 + - [ ] 隔天再访(**需真实次日复验**——服务端拒绝 >24h 时间偏移事件,无法伪造跨天数据,这是防篡改设计;当天先验证:等 30 分钟后新访问使 visit_count+1、时间线新增会话组)→ 同 visitor_id、visit_count+1、时间线新增日期组 - [ ] 首触来源 Google/带 UTM 链接进入 → first_referrer_domain/first_utm 正确 - [ ] 直接刷新/断网重试/重复上报 → 无重复事件(event_id 幂等) - [ ] 非白名单域 curl 伪造上报 → 403;超频 → 429