commit 7f4f3be8d8e8a45ae17521ab279b2e95b9e89f0d Author: 浅念 Date: Thu Aug 27 16:12:37 2026 +0800 插件 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cb3cee1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +node_modules/ + +# temporary analysis artifacts +*_resp.json +*.txt +getOp_req_headers.txt +alibaba_login.png +.playwright-mcp/ diff --git a/background.js b/background.js new file mode 100644 index 0000000..a9b14fb --- /dev/null +++ b/background.js @@ -0,0 +1,144 @@ +// background.js — Service Worker：推送队列 worker + 本地数据汇聚（双写） +// 以 chrome.storage 为本地汇聚权威源（MV3 的 SW 会被休眠，内存变量不可靠）； +// 同时通过 alarms + IndexedDB 队列把数据推送到后端 ingest 端点。 +import { addItem, getStats } from './push/queue.js'; +import { processQueue } from './push/worker.js'; +import { loadConfig, saveConfig } from './push/config.js'; + +// ===== 本地数据汇聚（保留，供插件本地导出）===== +const BG_KEY = 'alibaba_bg_details'; + +async function loadMeta() { + const data = await chrome.storage.local.get(BG_KEY); + return data[BG_KEY] || { dedup: true, customerIdMap: {}, detailData: [] }; +} + +async function saveMeta(meta) { + await chrome.storage.local.set({ [BG_KEY]: meta }); +} + +// ===== 推送配置 + alarms ===== +const ALARM = 'push-tick'; + +async function refresh() { + try { + const cfg = await loadConfig(); + if (cfg.enabled) { + await chrome.alarms.create(ALARM, { periodInMinutes: 1 }); + processQueue(); + } else { + await chrome.alarms.clear(ALARM); + } + } catch (e) { + // 配置/alarms 失败不应阻断 SW 启动 + console.error('[bg] refresh error:', e); + } +} + +chrome.runtime.onInstalled.addListener(refresh); +chrome.runtime.onStartup.addListener(refresh); +chrome.alarms.onAlarm.addListener(a => { + if (a && a.name === ALARM) processQueue(); +}); + +// ===== 消息处理 ===== +chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { + // --- 推送相关 --- + if (msg.type === 'enqueue') { + (async () => { + try { + const cfg = await loadConfig(); + if (!cfg.enabled) { sendResponse({ ok: false, reason: 'disabled' }); return; } + const inquiryId = msg.record && msg.record.inquiryId; + const id = await addItem({ + payload: { record: msg.record, messages: msg.messages || [] }, + inquiryId, + }); + sendResponse({ ok: true, id }); + processQueue(); + } catch (e) { + sendResponse({ ok: false, error: String((e && e.message) || e) }); + } + })(); + return true; + } + + if (msg.type === 'getPushStats') { + (async () => { + try { sendResponse({ ok: true, stats: await getStats() }); } + catch (e) { sendResponse({ ok: false, error: String((e && e.message) || e) }); } + })(); + return true; + } + + if (msg.type === 'getPushConfig') { + (async () => { + try { sendResponse({ ok: true, config: await loadConfig() }); } + catch (e) { sendResponse({ ok: false, error: String((e && e.message) || e) }); } + })(); + return true; + } + + if (msg.type === 'setPushConfig') { + (async () => { + try { + await saveConfig(msg.config || {}); + await refresh(); + sendResponse({ ok: true }); + } catch (e) { + sendResponse({ ok: false, error: String((e && e.message) || e) }); + } + })(); + return true; + } + + // --- 本地数据汇聚（保留原逻辑，照搬） --- + if (msg.type === 'storeDetail') { + // 先读后写：SW 休眠唤醒后 DETAIL_STORE 会清空，必须从 storage 恢复再合并 + (async () => { + try { + const meta = await loadMeta(); + // 首条写入时锁定本次抓取的 dedup 策略 + const isEmpty = Object.keys(meta.customerIdMap).length === 0 && meta.detailData.length === 0; + if (isEmpty) meta.dedup = !!msg.dedup; + + if (meta.dedup && msg.customerId) { + // 按 customerId 去重，保留 createTime 更新的那条 + const existing = meta.customerIdMap[msg.customerId]; + if (!existing || (msg.createTime || '') >= (existing.createTime || '')) { + meta.customerIdMap[msg.customerId] = msg.record; + } + } else { + // 无 customerId 或非去重模式：直接追加 + meta.detailData.push(msg.record); + } + meta.lastUpdate = Date.now(); + await saveMeta(meta); + sendResponse({ ok: true }); + } catch (e) { + sendResponse({ ok: false, error: String(e && e.message || e) }); + } + })(); + return true; // 保持消息通道，等待异步 sendResponse + } + + if (msg.type === 'clearBgData') { + (async () => { + await chrome.storage.local.remove(BG_KEY); + sendResponse({ ok: true }); + })(); + return true; + } + + if (msg.type === 'getBgData') { + (async () => { + const meta = await loadMeta(); + // 合并两份数据源，避免去重模式下无 customerId 的记录被丢弃 + const records = meta.dedup + ? [...Object.values(meta.customerIdMap), ...meta.detailData] + : meta.detailData; + sendResponse({ records, dedup: meta.dedup }); + })(); + return true; + } +}); diff --git a/content.js b/content.js new file mode 100644 index 0000000..97a3674 --- /dev/null +++ b/content.js @@ -0,0 +1,1301 @@ +// content.js — 核心抓取逻辑（支持跨页面导航状态持久化） + +(async function () { + // 防止重复注入 + if (window.__alibabaScraperInit) return; + window.__alibabaScraperInit = true; + + // ============ 常量 ============ + const LIST_URL = 'https://message.alibaba.com/message/default.htm#feedback/all'; + const STORAGE_KEY = 'alibaba_scraper_state'; + + // ============ 状态（从 storage 恢复） ============ + let state = { + phase: 'idle', // idle | list | detail | paused | done + mode: 'detail', + dedup: true, + chat: true, + listData: [], + detailData: [], + customerIdMap: {}, + seenChatCustomer: {}, + detailQueue: [], + queueIndex: 0, + currentPage: 0, + totalPages: 0, + totalInquiries: 0, + startTime: '', + }; + + // 从 storage 恢复状态 + async function loadState() { + try { + const stored = await chrome.storage.local.get(STORAGE_KEY); + if (stored[STORAGE_KEY]) { + state = { ...state, ...stored[STORAGE_KEY] }; + log('从缓存恢复状态: ' + state.phase + ', 队列 ' + state.queueIndex + '/' + state.detailQueue.length); + } + } catch (e) { /* 首次安装无缓存 */ } + } + + async function saveState() { + try { + await chrome.storage.local.set({ [STORAGE_KEY]: state }); + } catch (e) { /* 忽略 */ } + } + + // ============ 通信 ============ + function sendToPopup(type, data = {}) { + try { chrome.runtime.sendMessage({ type, ...data }); } catch (e) {} + } + + function log(text, level = '') { + console.log('[阿里抓取]', text); + sendToPopup('log', { text, level }); + } + + function progress(current, total) { + sendToPopup('progress', { current, total }); + } + + // ============ 工具 ============ + const sleep = ms => new Promise(r => setTimeout(r, ms)); +const randSleep = (min, max) => sleep(min + Math.random() * (max - min)); // 随机间隔,降低风控指纹 + + function isDetailPage() { + return window.location.href.includes('maDetail.htm'); + } + + // 提取 customerId(三层兜底:真customerId link href → inquiryId → secTradeId)。 + // 独立出来:fetchChatByApi 调用时 scrapeDetailData 还没跑,需提前用它判断"同客户是否已抓聊天"。 + function extractCustomerId() { + const link = document.querySelector('a.customer-detail-link'); + let cid = ''; + if (link) { + const m = (link.getAttribute('href') || '').match(/customerId=([^&]+)/); + cid = m ? m[1] : ''; + } + if (!cid) { + const iq = location.search.match(/imInquiryId=(\d+)/) || location.hash.match(/detail\/all\/(\d+)\//); + cid = iq ? 'iq_' + iq[1] : ''; + } + if (!cid) { + const sec = location.hash.match(/MC1IDX1[\w-]+/); + cid = sec ? 'sec_' + sec[0] : ''; + } + return cid; + } + +// ==== 聊天时间范围工具(与 scripts/verify-chat-range.mjs 基准逐字同步;改任一侧必须同步另一侧)==== +// 范围过滤(null sendTime 防御性保留 —— 宁多勿漏) +function inChatRange(sendTime, sinceMs, untilMs) { + if (sendTime == null) return true; + if (sinceMs != null && sendTime < sinceMs) return false; + if (untilMs != null && sendTime > untilMs) return false; + return true; +} +// 水位读取兼容:旧=纯 string(sendTime 未知) / 新={id,sendTime} +function parseWatermark(v) { + if (v == null || v === '') return null; + try { + const o = JSON.parse(v); + if (o && typeof o === 'object' && o.id != null) return { id: String(o.id), sendTime: Number(o.sendTime) || null }; + } catch (e) { /* 旧格式继续 */ } + return { id: String(v), sendTime: null }; +} +// 水位只前进写入判定 +function shouldWriteWatermark(oldWm, candId, candSendTime) { + if (candId == null || candSendTime == null) return false; + if (oldWm == null) return true; + if (oldWm.sendTime == null) return true; // legacy 升级,允许覆盖一次 + return candSendTime > oldWm.sendTime; +} +// probe 轮断言分类:'fail'(非对象/code≠200/data=null) | 'empty'(空list) | null(正常)。 +// non-JSON/HTTP/关键词由 postForm 抛 RISK,不经此函数(真风控照抛,不降级) +function probeFailKind(json) { + if (!json || typeof json !== 'object') return 'fail'; + if (json.code !== undefined && Number(json.code) !== 200) return 'fail'; + if (json.data == null) return 'fail'; + const list = json.data && json.data.list; + if (!Array.isArray(list) || list.length === 0) return 'empty'; + return null; +} +// ISO → 毫秒,parse 容错(spec §5:非法/空 → null = 降级不限) +function toMs(v) { const n = v ? Date.parse(v) : NaN; return Number.isFinite(n) ? n : null; } + + // ============ 安全层:风控关键词 ============ + // getOpMessages 响应里命中任一关键词 → throw 'RISK:关键词',由主流程触发暂停报警 + const RISK_KEYWORDS = ['验证码', 'captcha', '滑动验证', '安全验证', 'blocked', '访问频繁', '限制访问', '风控']; + + // 对方账号不可用(自行注销 / 违反平台规则被管控)的提示文案。命中 → 本询盘无法抓取, + // 直接跳过到下一条(不算风控、不暂停、不发聊天 API,彻底绕开风控检测)。 + const SKIP_KEYWORDS = ['对方账号已自行注销', '违反平台规则被管控']; + const hitsSkipKeyword = s => { + const lower = String(s == null ? '' : s).toLowerCase(); + return SKIP_KEYWORDS.find(kw => lower.includes(String(kw).toLowerCase())); + }; + + // ============ 安全层:增量水位(新格式 {id,sendTime};只前进不倒退,spec §4.5) ============ + async function getWatermark(inquiryId) { + if (!inquiryId) return null; + try { + const r = await chrome.storage.local.get(['lastTopMsg_' + inquiryId]); + return parseWatermark(r && r['lastTopMsg_' + inquiryId]); + } catch (e) { return null; } + } + async function trySetWatermark(inquiryId, candId, candSendTime) { + if (!inquiryId) return; + const oldWm = await getWatermark(inquiryId); + if (!shouldWriteWatermark(oldWm, candId, candSendTime)) return; + try { + await chrome.storage.local.set({ ['lastTopMsg_' + inquiryId]: JSON.stringify({ id: candId, sendTime: candSendTime }) }); + } catch (e) { /* 持久化失败不致命 */ } + } + + // ============ 安全层:DOM 增量抓取 lastDomTime 持久化(按 inquiryId) ============ + // 用于 loadAllChatHistory 的 DOM 增量分支:记上次最新 sendTime(ISO 字符串), + // 下次只抓 sendTime > lastDomTime 的新增消息,不往旧滚。ISO 字符串字典序即时间序,可直接比较。 + async function getLastDomTime(inquiryId) { + if (!inquiryId) return null; + try { + const key = 'lastDomTime_' + inquiryId; + const r = await chrome.storage.local.get([key]); + return r && r[key] ? String(r[key]) : null; + } catch (e) { return null; } + } + async function setLastDomTime(inquiryId, ts) { + if (!inquiryId || !ts) return; + try { + const key = 'lastDomTime_' + inquiryId; + await chrome.storage.local.set({ [key]: String(ts) }); + } catch (e) { /* 持久化失败不致命,下一轮可能多抓一些 */ } + } + + // ============ 安全层:作息时段(8-12 / 14-18,边界 ±30 分钟会话级随机) ============ + // “最像人”:每个工作时段在本会话内有一次随机的开始/结束偏移,避免每天卡整点。 + const WORK_RANGES = [{ start: 8, end: 12 }, { start: 14, end: 18 }]; + const sessionWorkOffsets = WORK_RANGES.map(() => ({ + // ±30 分钟(单位:分钟) + startOffset: Math.round((Math.random() - 0.5) * 2 * 30), + endOffset: Math.round((Math.random() - 0.5) * 2 * 30), + })); + // 作息时段开关:true=按 8-12/14-18(生产,降低非工作时段高速请求的风控风险); + // false=总是允许(仅测试用,避免非工作时段卡住等待) + const WORK_TIME_ENABLED = false; // 临时本地测试态(18:47 非工作时段会卡);本地验证完恢复 true + function isWorkTime() { + if (!WORK_TIME_ENABLED) return true; + const now = new Date(); + const mod = now.getHours() * 60 + now.getMinutes(); // 当日分钟数 0-1439 + for (let i = 0; i < WORK_RANGES.length; i++) { + const r = WORK_RANGES[i]; + const startMin = r.start * 60 + sessionWorkOffsets[i].startOffset; + const endMin = r.end * 60 + sessionWorkOffsets[i].endOffset; + if (mod >= startMin && mod < endMin) return true; + } + return false; + } + // 暂停轮询直到进入作息时段;用户手动暂停(state.phase==='paused')或完成则提前返回 false + async function waitUntilWorkTime(reason) { + if (isWorkTime()) return true; + log(`非作息时段,暂停等待 (${reason})`, 'info'); + while (!isWorkTime()) { + if (state.phase === 'paused') return false; + await randSleep(30000, 60000); + } + log(`进入作息时段,继续 (${reason})`, 'info'); + return true; + } + + // ============ 监听 popup 指令 ============ + chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { + switch (msg.action) { + case 'start': + state = { + phase: 'list', mode: msg.mode || 'detail', dedup: msg.dedup !== false, + chat: msg.chat !== false, limit: msg.limit || 0, chatLimit: msg.chatLimit ?? 0, + chatSince: msg.chatSince ?? null, chatUntil: msg.chatUntil ?? null, // 时间范围(spec §4.2);显式列入重建清单 + listData: [], detailData: [], + customerIdMap: {}, seenChatCustomer: {}, detailQueue: [], queueIndex: 0, + currentPage: 0, totalPages: 0, totalInquiries: 0, + startTime: new Date().toISOString(), + }; + saveState(); + if (!isDetailPage()) { + runListPhase(); + } else { + window.location.href = LIST_URL; + } + sendResponse({ ok: true }); + break; + case 'pause': + state.phase = 'paused'; + saveState(); + log('已暂停', 'info'); + sendToPopup('log', { text: '已暂停（刷新页面后状态保留）', level: 'info' }); + sendResponse({ ok: true }); + break; + case 'resume': + state.phase = state.detailQueue.length > state.queueIndex ? 'detail' : 'list'; + saveState(); + if (!isDetailPage()) { + if (state.detailQueue.length > state.queueIndex) { + runDetailPhase(); + } else { + runListPhase(); + } + } else { + // 当前在详情页：跳转到 queueIndex 对应的详情页再抓，避免用上一条 DOM 造成数据错位 + if (state.detailQueue.length > state.queueIndex) { + const it = state.detailQueue[state.queueIndex]; + const u = it.dataHref.startsWith('http') ? it.dataHref : 'https://message.alibaba.com' + it.dataHref; + window.location.href = u; + } else { + finishAll(); + } + } + sendResponse({ ok: true }); + break; + case 'export': { + // 回调式获取 background 数据（避免在非 async 监听器里用 await 导致整文件加载失败） + chrome.runtime.sendMessage({ type: 'getBgData' }, bgResult => { + let exportData = state.detailData; + // 列表模式没有详情数据时，回退到 listData（否则列表模式导出空文件） + if (exportData.length === 0 && state.listData.length > 0) { + exportData = state.listData; + } + if (bgResult && bgResult.records && bgResult.records.length > exportData.length) { + exportData = bgResult.records; + } + const csv = buildCSV(exportData, state.mode); + const count = exportData.length; + sendResponse({ success: true, csv, count }); + }); + break; + } + case 'clear': + state = { + phase: 'idle', mode: 'detail', dedup: true, chat: true, + listData: [], detailData: [], customerIdMap: {}, seenChatCustomer: {}, + detailQueue: [], queueIndex: 0, currentPage: 0, totalPages: 0, + totalInquiries: 0, startTime: '', + }; + chrome.storage.local.remove(STORAGE_KEY); + // 清增量标记(lastTopMsgId/lastDomTime):旧版 sendTime null 400 可能留下 lastTopMsgId + // 但后端没存,导致新版增量命中后只抓 0 条新增、永远补不回。clear 时一并清,让下次全量重抓。 + chrome.storage.local.get(null, all => { + const keys = Object.keys(all || {}).filter(k => k.startsWith('lastTopMsg_') || k.startsWith('lastDomTime_')); + if (keys.length) chrome.storage.local.remove(keys); + }); + chrome.runtime.sendMessage({ type: 'clearBgData' }); + sendResponse({ ok: true }); + break; + case 'getState': + sendResponse({ + running: ['list', 'detail'].includes(state.phase), + phase: state.phase, + listCount: state.listData.length, + detailCount: state.detailData.length, + queueIndex: state.queueIndex, + queueTotal: state.detailQueue.length, + }); + break; + } + return true; + }); + + // ============ 列表页抓取 ============ + function scrapeListPage() { + const items = document.querySelectorAll('a.inquiry-item'); + const list = []; + for (const item of items) { + const idEl = item.querySelector('.spec-inquiry-id'); + const inquiryId = idEl ? (idEl.innerText.match(/\d+/) || [''])[0] : ''; + if (!inquiryId) continue; + + const timeEl = idEl ? idEl.nextElementSibling : null; + let createTime = '', updateTime = ''; + if (timeEl) { + const ct = timeEl.innerText.match(/创建时间[：:]?\s*(\S+)/); + const ut = timeEl.innerText.match(/更新时间[：:]?\s*(\S+)/); + if (ct) createTime = ct[1]; + if (ut) updateTime = ut[1]; + } + + const nameEl = item.querySelector('.aui2-grid-name'); + const flagEl = item.querySelector('[class*="ui2-flag"]'); + const cc = (flagEl ? flagEl.className : '').match(/ui2-flag-(\w+)/); + + const msgEl = item.querySelector('.buyer-chat-msg'); + const ownerEl = item.querySelector('.aui-grid-owner-name'); + const statusEl = item.querySelector('.aui2-grid-quo-status-col'); + + const avatarEl = item.querySelector('.sender-avatar'); + let avatarUrl = ''; + if (avatarEl) { + // .sender-avatar 实测无 style 属性，需尝试 background-image / 子 img / 自身 src + const bg = (avatarEl.getAttribute('style') || '').match(/url\(["']?([^"')]+)/); + if (bg) avatarUrl = bg[1].replace(/&/g, '&'); + else { + const img = avatarEl.tagName === 'IMG' ? avatarEl : avatarEl.querySelector('img'); + if (img) avatarUrl = (img.getAttribute('src') || '').replace(/&/g, '&'); + } + } + + list.push({ + inquiryId, createTime, updateTime, + buyerName: nameEl ? nameEl.innerText.trim() : '', + countryCode: cc ? cc[1] : '', + msgPreview: msgEl ? msgEl.innerText.trim().substring(0, 300) : '', + manager: ownerEl ? ownerEl.innerText.trim() : '', + status: statusEl ? statusEl.innerText.trim().replace('查看详情', '').trim() : '', + hasBuyerLevel: !!item.querySelector('[class*="buyer-level"]'), + avatarUrl, + isFlagged: !!item.querySelector('[class*="flag-filled"]'), + dataHref: item.getAttribute('data-href') || '', + }); + } + return list; + } + + function getTotalPages() { + // 页码在 .ui2-pagination-pages 容器内，每页条数在 .ui2-pagination-show + const pagContainer = document.querySelector('.ui2-pagination-pages'); + if (pagContainer) { + // 找所有 data-page 属性的最大值 + const links = pagContainer.querySelectorAll('a[data-page]'); + let maxPage = 1; + for (const link of links) { + const n = parseInt(link.getAttribute('data-page'), 10); + if (!isNaN(n) && n > maxPage) maxPage = n; + } + return maxPage; + } + // 备用 + const el = document.querySelector('.ui2-pagination'); + if (!el) return 1; + const pages = el.innerText.match(/(\d+)\s*$/); + return pages ? parseInt(pages[1], 10) : 1; + } + + function setPageSize(n) { + const btn = document.querySelector(`a[data-page-size="${n}"]`); + if (btn) btn.click(); + } + + // 等待列表数据加载完成 + function waitForListLoaded() { + return new Promise(resolve => { + let tries = 0; + const t = setInterval(() => { + tries++; + const items = document.querySelectorAll('a.inquiry-item'); + if (items.length > 0 || tries > 20) { + clearInterval(t); + resolve(); + } + }, 500); + }); + } + + function clickNextPage() { + const btn = document.querySelector('[mc-qa="list-page-next"]') || + document.querySelector('[mc-role="btn-next-page"]'); + if (btn) btn.click(); + } + + // ============ 列表阶段 ============ + async function runListPhase() { + state.phase = 'list'; + saveState(); + log(`模式: ${state.mode === 'detail' ? '列表+详情' : '仅列表'} — 开始抓取列表`, 'info'); + + try { + setPageSize(100); + // 等待列表刷新 + await waitForListLoaded(); + await randSleep(1200, 3000); + + state.totalPages = getTotalPages(); + state.totalInquiries = state.totalPages * 100; + log(`共 ${state.totalPages} 页，约 ${state.totalInquiries} 条`, 'info'); + saveState(); + + for (let page = 1; page <= state.totalPages; page++) { + if (state.phase === 'paused') return; + + if (page > 1) { + clickNextPage(); + await waitForListLoaded(); + await randSleep(1200, 3000); + } + + state.currentPage = page; + const items = scrapeListPage(); + log(`第 ${page}/${state.totalPages} 页: ${items.length} 条`, ''); + + // limit：只取需要的数量 + const remaining = state.limit > 0 ? state.limit - state.listData.length : items.length; + const trimmed = remaining > 0 ? items.slice(0, remaining) : []; + state.listData.push(...trimmed); + + if (state.mode === 'detail') { + // 去重改由后端按 customerId 合并(同客户详情/聊天一致),扩展全推不丢数据。 + state.detailQueue.push(...trimmed); + } + + // limit 检查：达到上限就停止 + if (state.limit > 0 && state.listData.length >= state.limit) { + log(`已达到限制 ${state.limit} 条，停止列表抓取`, 'info'); + break; + } + + progress(state.listData.length, state.totalInquiries); + saveState(); + await randSleep(800, 1800); + } + + log(`列表抓取完成，共 ${state.listData.length} 条`, 'success'); + + if (state.mode === 'list' || state.phase === 'paused') { + state.phase = 'done'; + saveState(); + sendToPopup('done', {}); + return; + } + + // 进入详情阶段 + runDetailPhase(); + + } catch (e) { + log('列表抓取出错: ' + e.message, 'error'); + state.phase = 'idle'; + saveState(); + sendToPopup('error', { text: e.message }); + } + } + + // ============ 详情阶段（导航到详情页前触发） ============ + async function runDetailPhase() { + if (state.queueIndex >= state.detailQueue.length) { + finishAll(); + return; + } + + // 安全层 #4:进入下一个详情前做作息时段检测;非作息时段挂起等待,用户暂停则提前退出 + if (!await waitUntilWorkTime('详情')) return; + if (state.phase === 'paused') return; + + state.phase = 'detail'; + saveState(); + log(`开始抓取详情页 (${state.queueIndex + 1}/${state.detailQueue.length})`, 'info'); + + const item = state.detailQueue[state.queueIndex]; + const fullUrl = item.dataHref.startsWith('http') ? item.dataHref : + 'https://message.alibaba.com' + item.dataHref; + + // 导航到详情页（content script 会被销毁） + window.location.href = fullUrl; + } + + // ============ 在详情页加载后被调用 ============ + async function processCurrentDetailPage() { + if (state.phase !== 'detail') return; + + try { + // 等待页面加载 + await waitForLoad(); + + // 对方账号注销/被管控:页面提示"对方账号已自行注销.../违反平台规则被管控",本询盘不可抓。 + // 命中即 throw SKIP → catch 跳过下一条(在调聊天 API 之前,彻底绕开风控检测、不暂停) + if (hitsSkipKeyword(document.body.innerText || '')) throw new Error('SKIP:账号不可用'); + + // 优先用 API 直拉聊天（快且完整），失败回退 wheel DOM 滚动 + let apiMsgs = null; + let apiMeta = null; + if (state.chat) { + // 同客户去重:同买家多询盘是同一聊天会话(后端 ingest 已按 customerId 归并到代表 inquiry 一份), + // 扩展侧每个询盘都翻页抓一遍是纯浪费(多余 API 请求 + 风控风险)。故同 customer 本次只抓首个遇到的询盘, + // 后续跳过(API/DOM 都不调),该询盘聊天留空 → 后端自动归代表(代表已有完整聊天)。 + const custId = extractCustomerId(); + if (custId && state.seenChatCustomer[custId]) { + log(`同客户 ${custId} 聊天已抓,跳过本询盘聊天抓取(后端已归代表)`, 'info'); + state.chatSkipped = true; // 标记 scrapeDetailData 不再 DOM fallback,避免抓出重复的 dom- 消息 + } else { + const r = await fetchChatByApi(); + if (r) { apiMsgs = r.messages; apiMeta = r.meta; if (custId) state.seenChatCustomer[custId] = true; } + else await loadAllChatHistory(); + } + } + + // 等行为卡(含异步加载的「被加为黑名单数」)渲染完再抓,否则黑名单字段会漏抓 + await waitForText('被加为黑名单数', 5000); + // 注:客户资料折叠面板(手机/座机/商业类型/销售平台/公司官网)无法用 JS 展开—— + // alibaba 检查 event.isTrusted,只接受真实鼠标点击;扩展(content script)的 + // .click()/dispatchEvent 都 isTrusted=false 被忽略。这些折叠字段抓不到,保留其他。 + // 等客户信息卡(.contact-base-info + 买家名 .name-text)渲染完再抓, + // 否则买家名/国家/公司/邮箱/电话/注册时间等会因 DOM 没加载而漏抓(空)。 + for (let i = 0; i < 30; i++) { + if (document.querySelector('.contact-base-info') && document.querySelector('.name-text')) break; + await randSleep(150, 400); + } + // 抓取数据(行为数据异步加载,可能漏;抓后验证 productViews 有值,否则重等重抓,最多 3 次) + // DOM handoff 在重试循环外取一次传入:若在 scrapeDetailData 内消费,首次调用即清空, + // 重试调用会落穿到未过滤全量 DOM 提取 → 时间模式下范围外消息泄漏(seam 修复) + const domHandoff = state._incDomMessages || null; + const domFiltered = !!state._incDomFiltered; + state._incDomMessages = null; state._incDomFiltered = false; + let detail = scrapeDetailData(apiMsgs, domHandoff, domFiltered); + for (let t = 0; t < 3 && !detail.behavior?.productViews; t++) { + await randSleep(800, 1500); + detail = scrapeDetailData(apiMsgs, domHandoff, domFiltered); + } + // 输出抓取结果日志,便于在 popup 确认是否抓全(客户ID/手机/座机/商业类型/行为等) + const it0 = state.detailQueue[state.queueIndex]; + log(`抓取[${state.queueIndex + 1}/${state.detailQueue.length}] ${it0?.inquiryId || '?'} → 客户ID:${detail.customerId ? '✓' : '空'} 买家:${detail.detailBuyerName || '空'} 手机:${detail.phone || '空'} 座机:${detail.landline || '空'} 商业类型:${detail.businessType || '空'} 官网:${detail.website || '空'} 行为浏览:${detail.behavior?.productViews || '空'} 店内产品:${detail.storeInquiryProduct ? '✓' : '无'}`, detail.customerId && detail.behavior?.productViews ? 'success' : 'error'); + // 把 API 直拉的会话级时间挂到 detail，供 payload.js 作为优先 createTime/updateTime + if (apiMeta) { + detail.createTimeFromApi = apiMeta.gmtCreate; + detail.updateTimeFromApi = apiMeta.gmtModified; + } + const item = state.detailQueue[state.queueIndex]; + + // 去重：通过 background.js 存储（同步返回，不依赖异步 storage） + const merged = { ...item, ...detail }; + try { + const stored = await new Promise(resolve => { + chrome.runtime.sendMessage({ + type: 'storeDetail', + record: merged, + customerId: detail.customerId || '', + createTime: item.createTime || '', + dedup: state.dedup, + }, resp => resolve(resp)); + }); + } catch (e) { + // background 不可用时回退到本地存储 + if (state.dedup && detail.customerId) { + const existing = state.customerIdMap[detail.customerId]; + if (!existing || item.createTime >= (existing.createTime || '')) { + state.customerIdMap[detail.customerId] = merged; + } + } else { + state.detailData.push(merged); + } + } + + // 推送到后端（Plan 2）：发 record + messages 给 SW 入队。fire-and-forget， + // 失败不影响本地抓取流程；SW 会根据 push_config.enabled 决定是否真正入队。 + try { + chrome.runtime.sendMessage( + { type: 'enqueue', record: merged, messages: detail.chatMessages || [] }, + () => { /* 最后错误仅诊断用 */ void chrome.runtime.lastError; } + ); + } catch (e) { /* SW 不可用时静默 */ } + + state.queueIndex++; + progress(state.queueIndex, state.detailQueue.length); + await saveState(); + + if ((state.queueIndex) % 10 === 0) { + log(`详情进度: ${state.queueIndex}/${state.detailQueue.length}`, ''); + } + + if (state.phase === 'paused') return; + + // 继续下一条 + if (state.queueIndex < state.detailQueue.length) { + // 先回列表页再进详情页（更稳定） + // 直接跳转下一条详情页 + const nextItem = state.detailQueue[state.queueIndex]; + const nextUrl = nextItem.dataHref.startsWith('http') ? nextItem.dataHref : + 'https://message.alibaba.com' + nextItem.dataHref; + // 额外延迟确保 storage 写入完成 + await randSleep(800, 2000); + window.location.href = nextUrl; + } else { + finishAll(); + } + + } catch (e) { + // 安全层 #6:RISK: 前缀错误 → 触发风控报警并暂停(不跳过,等用户处理后恢复) + if (e && typeof e.message === 'string' && e.message.startsWith('RISK:')) { + state.phase = 'paused'; + await saveState(); + log('⚠️ 疑似触发风控,已暂停: ' + e.message, 'error'); + sendToPopup('risk', { msg: e.message }); + return; + } + // SKIP: 对方账号注销/管控 → 友好跳过下一条,不算异常、不暂停 + if (e && typeof e.message === 'string' && e.message.startsWith('SKIP:')) { + const it = state.detailQueue[state.queueIndex]; + log(`对方账号不可用(${e.message.slice(5)}),跳过本询盘 ${it?.inquiryId || ''}`, 'info'); + state.queueIndex++; + await saveState(); + await randSleep(800, 2000); + runDetailPhase(); + return; + } + log('详情抓取异常: ' + e.message, 'error'); + // 跳过这条继续 + state.queueIndex++; + await saveState(); + await randSleep(800, 2000); + runDetailPhase(); + } + } + + // ============ API 直拉聊天（替代 wheel：秒级拉全量，能取到 wheel 取不到的早期历史）============ + // 取 cookie 值,两层:① 顶层 cookie(标准,值在 ^ 或 ; 边界之后,遇 ; 截断); + // ② 兜底——有的 token 不是顶层 cookie,而是嵌在别的 cookie 的 query-string 值里 + // (ctoken 就藏在 xman_us_t 的 "l_source=..&ctoken=xxx&.." 值中),故再按 name= 全局取、遇 & 截断。 + function getCookie(name) { + const dc = document.cookie; + let m = dc.match(new RegExp('(?:^|;\\s*)' + name + '=([^;]+)')); + if (m) return m[1]; + m = dc.match(new RegExp(name + '=([^;&]+)')); + return m ? m[1] : ''; + } + function decodeHtml(s) { + // textarea 是 RCDATA 元素，只解码字符引用（实体）不解析标签，能正确处理 á ñ ' 等所有命名/数字实体 + const t = document.createElement('textarea'); + t.innerHTML = String(s == null ? '' : s); + return t.value || ''; + } + // 卡片消息(询盘/目录,type=6/2008)的 API content 是 workspace.alibaba.com/card URL。 + // 阿里前端会调内部 API 把它渲染成卡片正文(询盘标题/采购数量/详细需求 等)放进 .session-rich-content。 + // API 拿不到正文,所以详情页 DOM 渲染后,按 data-original(URL)对应回 wrapper,抓 .session-rich-content 正文。 + function enrichCardFromDom(m) { + const c = m.content || ''; + if (!/workspace\.alibaba\.com\/card\?type=/.test(c)) return; + const ext = (c.match(/[?&]extensionType=([^&]+)/) || [])[1] || ''; + // 图片卡片(type=13)不走 DOM enrich:实测卡片 URL 的 secOperateAliId 也能构造可加载的 clouddisk 图片直链, + // 由 apiMsgToChat 直接构造,不依赖 DOM 渲染(避免历史图片懒加载 → [图片]) + if (/^(jpg|jpeg|png|gif|webp|bmp)$/i.test(ext)) return; + if (/type=14\b/.test(c)) return; // 视频卡片:apiMsgToChat 从卡片 URL 构造 clouddisk,不依赖 DOM 懒加载 + try { + for (const w of document.querySelectorAll('.message-item-wrapper')) { + if ((w.getAttribute('data-original') || '') === c) { + // 询盘/目录卡片:.session-rich-content 正文(标题/采购/需求) + const card = w.querySelector('.session-rich-content'); + const text = card ? card.innerText.trim() : ''; + if (text) m.content = text; + return; + } + } + } catch (e) { /* DOM 不可用则保留 URL,apiMsgToChat 兜底标记 */ } + } + function apiMsgToChat(m) { + const direction = m.messageType === 'send' ? 'outgoing' : (m.messageType === 'rec' ? 'incoming' : 'system'); + let content = decodeHtml(m.content || ''); + if (/^,白名单 alibaba 域防 XSS) + const src = (content.match(/src=["']([^"']+)["']/i) || [])[1] || ''; + content = src ? ('img:' + src) : '[图片]'; + } + else if (/extensionType=/.test(content) && /workspace\.alibaba\.com\/card\b/.test(content)) { + const ext = (content.match(/[?&]extensionType=([^&]+)/) || [])[1] || ''; + if (/^(jpg|jpeg|png|gif|webp|bmp)$/i.test(ext)) { + // 图片卡片(type=13):从卡片 URL 构造 clouddisk 图片直链(实测卡片 secOperateAliId 也能加载, + // 不依赖 DOM 懒加载,避免历史图片显示 [图片]) + const fid = (content.match(/[?&]id=([^&]+)/) || [])[1] || ''; + const fparent = (content.match(/[?&]parentId=([^&]+)/) || [])[1] || ''; + const fsec = (content.match(/secOperateAliId=([^&]+)/) || [])[1] || ''; + content = (fid && fparent && fsec) + ? ('img:https://clouddisk.alibaba.com/file/redirectFileUrl.htm?appkey=OneChat&id=' + fid + '&parentId=' + fparent + '&secOperateAliId=' + fsec + '&fileAction=imagePreview') + : '[图片]'; + } else if (/type=14\b/.test(content)) { + // 视频卡片(type=14):从卡片 URL 构造 clouddisk 视频直链(同图片,卡片 secOperateAliId 有效) + const fid = (content.match(/[?&]id=([^&]+)/) || [])[1] || ''; + const fparent = (content.match(/[?&]parentId=([^&]+)/) || [])[1] || ''; + const fsec = (content.match(/secOperateAliId=([^&]+)/) || [])[1] || ''; + const fact = (content.match(/[?&]fileAction=([^&]+)/) || [])[1] || 'download'; + content = (fid && fparent && fsec) + ? ('video:https://clouddisk.alibaba.com/file/redirectFileUrl.htm?appkey=OneChat&id=' + fid + '&parentId=' + fparent + '&secOperateAliId=' + fsec + '&fileAction=' + fact) + : '[视频]'; + } else { + // 文件卡片(PDF/Word/Excel):提取 clouddisk 直链(预览/下载用),用卡片 URL 打不开(需 alibaba 登录态) + const name = (content.match(/[?&]name=([^&]+)/) || [])[1] || ''; + let fileName = name ? decodeURIComponent(name).replace(/\+/g, ' ') : ''; + if (!fileName) fileName = ext ? ext.toUpperCase() + ' 文件' : '文件'; + // 真实文件 URL 在 url= 参数里(clouddisk redirectFileUrl,含 secOperateAliId + fileAction) + const clouddisk = (content.match(/url=(https:\/\/clouddisk\.alibaba\.com\/file\/redirectFileUrl\.htm\?[^]*?fileAction=\w+)/) || [])[1] || ''; + content = 'file:' + fileName + '|' + (clouddisk || content); + } + } + else if (/workspace\.alibaba\.com\/card\?type=/.test(content)) { + // 询盘/目录等卡片(API content 是 URL)。优先由 enrichCardFromDom 用 DOM 正文替换; + // 若 DOM 没抓到(回退场景),按 type 标记,避免把整段 URL 当文本显示。 + const type = (content.match(/type=(\d+)/) || [])[1] || ''; + content = type === '6' ? '[询盘卡片]' : (type === '2008' ? '[目录卡片]' : '[卡片消息]'); + } + else if (/<[^>]+>/.test(content)) { + const t = content.replace(//gi, '\n').replace(/<[^>]+>/g, '').trim(); + content = t || '[卡片消息]'; + } + // sendTime：API 返回毫秒数 → ISO 字符串（payload.js 直接消费） + const sendTime = m.sendTime ? new Date(m.sendTime).toISOString() : null; + // translation：从 m.extInfo.icbuData.extend（JSON 字符串）双重 parse 取 + // extend.extParams.inputTranslateInfo.translate；任何步骤失败留空，不抛 + let translation = ''; + try { + const ext = m.extInfo && m.extInfo.icbuData && m.extInfo.icbuData.extend; + if (ext) { + const e1 = JSON.parse(ext); + const info = e1 && e1.extParams && e1.extParams.inputTranslateInfo; + if (info) { + const e2 = JSON.parse(info); + translation = (e2 && e2.translate) || ''; + } + } + } catch (e) { /* translation 缺失不致命 */ } + // owner 固定是卖家(己方)、contact 固定是买家(对方)，方向由 messageType 决定 + const sender = direction === 'outgoing' + ? ((m.owner && m.owner.name) || '卖家') + : ((m.contact && m.contact.name) || '买家'); + return { + messageId: String(m.messageId), + sendTime, + sender, + original: content, + translation, + direction, + }; + } + async function fetchChatByApi() { + // 注意:本函数内的 RISK: 前缀错误必须向上抛出(不能被这里的 catch 吞掉回退 DOM), + // 由 processCurrentDetailPage 的 catch 捕获并触发风控暂停报警。 + // 其它普通异常(网络/解析/结构不完整)仍回退到 DOM 滚动。 + try { + // URL hash 末尾的 secTradeId token（列表 dataHref 自带） + const hashMatch = location.hash.match(/(MC1IDX1[\w-]+)\/?\s*$/); + if (!hashMatch) { log('聊天 API 跳过: URL hash 无 secTradeId token,回退 DOM 滚动', 'info'); return null; } + const hashSecTradeId = hashMatch[1]; + const ctoken = getCookie('ctoken'), tbToken = getCookie('_tb_token_'), xsrf = getCookie('XSRF-TOKEN'); + if (!ctoken || !tbToken || !xsrf) { log('聊天 API 跳过: 缺 cookie(ctoken/_tb_token_/XSRF-TOKEN),回退 DOM', 'info'); return null; } + // 安全层 #5:postForm 加风控检测 —— HTTP 非 200 / 关键词 / 结构异常 → throw 'RISK:*' + // ★ 绝不扫描 data 内的消息正文:getOpMessages 的 data.list[].content 是真实聊天, + // 客户消息含 "captcha/blocked/验证码/风控" 是正常外贸对话。旧实现对整段 text 扫关键词, + // 会把正常聊天误判为风控 → 整任务误暂停(命中任一历史消息即停)。 + // 现分两种响应形态分别检测: + // ① 非 JSON(HTML 验证页/滑块/登录页):整段 text 扫关键词(这才是风控页特征) + // ② JSON:只浅扫顶层 msg/message/errMsg/statue 等风控字段(data 是对象会被跳过,不进 list) + const RISK_TOP_FIELDS = new Set(['msg', 'message', 'errmsg', 'statue', 'errormessage', 'err']); + const hitsRiskKeyword = s => { + const lower = String(s == null ? '' : s).toLowerCase(); + return RISK_KEYWORDS.find(kw => lower.includes(String(kw).toLowerCase())); + }; + const postForm = async (url, params) => { + const resp = await fetch(url, { + method: 'POST', body: new URLSearchParams(params), credentials: 'include', + headers: { 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' } + }); + if (resp.status !== 200) throw new Error('RISK:HTTP(' + resp.status + ')'); + const text = await resp.text(); + let j; + try { j = JSON.parse(text); } + catch (e) { + // 非 JSON(HTML 风控/登录页)才扫整段 text + const kw = hitsRiskKeyword(text); + if (kw) throw new Error('RISK:关键词(' + kw + ')'); + throw new Error('RISK:结构异常(non-JSON)'); + } + // JSON:只浅扫顶层字符串类型的风控字段,不递进 data + if (j && typeof j === 'object') { + for (const f of Object.keys(j)) { + if (RISK_TOP_FIELDS.has(f.toLowerCase()) && typeof j[f] === 'string') { + const kw = hitsRiskKeyword(j[f]); + if (kw) throw new Error('RISK:关键词(' + kw + '@' + f + ')'); + } + } + } + return j; + }; + // 1) querySummary 拿加密会话参数(数据不完整视为「API 不可用」,回退 DOM,不算风控) + const qs = await postForm( + `https://message.alibaba.com/message/ajax/feedback/querySummary.htm?ctoken=${encodeURIComponent(ctoken)}&_tb_token_=${encodeURIComponent(tbToken)}`, + { _csrf_token_: xsrf, params: JSON.stringify({ secTradeId: hashSecTradeId }) } + ); + const d = qs && qs.data; + if (!d || !d.owner || !d.contact || !d.owner.accountIdEncrypt || !d.contact.accountIdEncrypt || !d.secTradeId) { log('聊天 API 跳过: querySummary 返回数据不完整,回退 DOM', 'info'); return null; } + // meta：从 querySummary 提取会话级时间（毫秒）→ ISO，挂到 detail 供 payload.js 用作 + // createTime/updateTime 优先值（优于列表页的本地时分显示） + const meta = { + gmtCreate: d.gmtCreate ? new Date(d.gmtCreate).toISOString() : null, + gmtModified: d.gmtModified ? new Date(d.gmtModified).toISOString() : null, + }; + const secOwnerAccountId = d.owner.accountIdEncrypt; + const secTargetAccountId = d.contact.accountIdEncrypt; + const secTradeId = d.secTradeId; + const targetAliId = d.cid ? String(d.cid).split('-')[0] : ''; + + // 安全层 #1:增量抓取。按 inquiryId 读上次最后一条(最新)messageId。 + // 命中则提前停止翻页,只抓新增;首轮(无记录)→ 全量。 + const itForId = state.detailQueue[state.queueIndex]; + const inquiryId = itForId ? itForId.inquiryId : ''; + const sinceMs = toMs(state.chatSince), untilMs = toMs(state.chatUntil); // 时间范围毫秒(§4.2);null=不限 + const lastTopWm = await getWatermark(inquiryId); + const lastTopId = lastTopWm ? lastTopWm.id : ''; + if (lastTopId) log(`聊天 API 增量模式: 上次水位 ${lastTopId},遇此停止翻页`); + if (sinceMs != null || untilMs != null) { + log(`聊天范围: ${sinceMs != null ? new Date(sinceMs).toLocaleString() : '最早'} ~ ${untilMs != null ? new Date(untilMs).toLocaleString() : '至今'}`); + } + + // 2) 循环 getOpMessages(spec 附录A:跳游标/probe/过滤/顺序/停止/水位) + const jump = untilMs != null && !state.cursorJumpUnsupported; + let cursor = jump ? untilMs + 1 : Date.now(); // +1ms:实测语义=严格小于,恰好覆盖 <= untilMs(附录B) + let probeRound = jump; + let probeDowngraded = false; // 局部瞬态:probe 降级过(仅本调用内有效 —— 不放 state,防止中途退出污染下个询盘) + let downgradeKind = null; // 降级原因:'fail'=结构异常 / 'empty'=空 list(多为真空,附录B组形态) + const raw = []; + const MAX_ROUNDS = state.chatLimit > 0 ? Math.ceil(state.chatLimit / 20) : 2000; + let candId = null, candSendTime = null; // 候选水位:逐轮取最大 sendTime(spec §4.5) + let stopReason = '轮数上限'; + for (let i = 0; i < MAX_ROUNDS; i++) { + if (state.phase === 'paused') return null; + if (!await waitUntilWorkTime('聊天翻页')) return null; + if (state.phase === 'paused') return null; + const json = await postForm( + `https://onetalk.alibaba.com/message/getOpMessages.htm?ctoken=${encodeURIComponent(ctoken)}&_tb_token_=${encodeURIComponent(tbToken)}`, + { params: JSON.stringify({ scene: '', timeSlide: { timeStamp: cursor, forward: false, pageSize: 20 }, secOwnerAccountId, secTradeId, secTargetAccountId, openRealTimeTranslation: true, targetAliId }), _csrf: xsrf } + ); + if (probeRound) { // 首轮探测:仅降级集走 fallback(真风控由 postForm 抛出,spec §4.3) + probeRound = false; + const kind = probeFailKind(json); + if (kind) { + log(`until 跳游标首轮 ${kind === 'empty' ? '空 list' : '结构异常'},降级 cursor=now 重试`); + probeDowngraded = true; + downgradeKind = kind; + cursor = Date.now(); + await randSleep(2000, 5000); + continue; + } + } else { + if (!json || typeof json !== 'object') throw new Error('RISK:结构异常(非对象)'); + if (json.code !== undefined && Number(json.code) !== 200) throw new Error('RISK:结构异常(code=' + json.code + ')'); + if (json.data == null) throw new Error('RISK:结构异常(data=null)'); + } + const list = (json.data && json.data.list) || []; + if (!list.length) { + // 降级重试也空 = 真空,不设 unsupported(probeDowngraded 为局部变量,随调用结束自然消亡) + stopReason = '到顶(空列表)'; break; + } + // 仅 'fail'(结构异常)且降级重试首批非空才实锤"不支持";'empty' 多为会话在 until 前本无消息 + // (真空,附录B组形态),据此设标志会把跳游标误禁一整个任务(final review Important#2 收紧) + if (probeDowngraded && downgradeKind === 'fail') { + probeDowngraded = false; + state.cursorJumpUnsupported = true; saveState(); + } + for (const m of list) { // 候选水位逐轮 max(未过滤口径,spec §4.5) + if (m.sendTime != null && (candSendTime == null || m.sendTime > candSendTime)) { + candId = String(m.messageId); candSendTime = m.sendTime; + } + } + let hitLastTop = false; + for (const m of list) { // 顺序:水位命中查未过滤 list,再范围过滤(spec §4.3.2) + if (lastTopId && String(m.messageId) === lastTopId) { hitLastTop = true; break; } + if (inChatRange(m.sendTime, sinceMs, untilMs)) raw.push(m); + } + let minT = Infinity; + for (const m of list) if (m.sendTime != null && m.sendTime < minT) minT = m.sendTime; // null 跳过(防游标跳1970) + if (minT !== Infinity) cursor = minT; + if (raw.length % 400 < 20) log(`聊天 API 拉取中: ${raw.length} 条`); + if (hitLastTop) { stopReason = '增量命中'; break; } + if (sinceMs != null && cursor < sinceMs) { stopReason = '时间边界'; break; } + if (list.length < 20) { stopReason = '到顶'; break; } + await randSleep(2000, 5000); + } + log(`聊天 API 完成: ${raw.length} 条(停:${stopReason})`, 'success'); + await trySetWatermark(inquiryId, candId, candSendTime); // 只前进写入(spec §4.5) + return { messages: raw.map(m => { enrichCardFromDom(m); return apiMsgToChat(m); }), meta }; + } catch (e) { + // RISK: 前缀错误向上抛,不吞掉 —— 让 processCurrentDetailPage 触发风控暂停报警 + if (e && typeof e.message === 'string' && e.message.startsWith('RISK:')) throw e; + log('API 拉取聊天失败，回退 DOM: ' + e.message, 'error'); + return null; + } + } + + // ============ DOM 聊天消息提取(可复用) ============ + // 从 .message-item-wrapper 抓消息;sendTime 取法与原 scrapeDetailData 内联逻辑完全一致 + // (item-base-info 下匹配 \d{4}-\d{2}-\d{2} 的 span),供 scrapeDetailData 和 loadAllChatHistory 增量分支共用。 + function extractDomChatMessages() { + const wrappers = document.querySelectorAll('.message-item-wrapper'); + const msgs = []; + for (const mw of wrappers) { + const isR = mw.className.includes('item-right'); + const isL = mw.className.includes('item-left'); + const ne = mw.querySelector('.name'); + const sender = ne ? ne.innerText.trim() : (isR ? '卖家' : ''); + const te = mw.querySelector('.item-base-info'); + let time = ''; + if (te) { + for (const sp of te.querySelectorAll('span')) { + if (sp.innerText.match(/\d{4}-\d{2}-\d{2}/)) { time = sp.innerText.trim(); break; } + } + } + // 原文：data-original 实际在 wrapper 自身属性上，其次子元素，最后 rich-content 兜底 + let orig = mw.getAttribute('data-original') || ''; + if (!orig) { + const oe = mw.querySelector('[data-original]'); + orig = oe ? (oe.getAttribute('data-original') || '') : ''; + } + if (!orig) { + const richEl = mw.querySelector('.session-rich-content'); + orig = richEl ? richEl.innerText.trim() : ''; + } + const trEl = mw.querySelector('.target-content'); + const trans = trEl ? trEl.innerText.trim() : ''; + if (orig || time) { + msgs.push({ + messageId: mw.getAttribute('data-message-id') || `dom-${wrappers.length}-${msgs.length}`, + sendTime: time ? new Date(time.replace(/(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})/, '$1T$2')).toISOString() : new Date().toISOString(), + // domTimed:是否有真实 DOM 时间戳(时间戳稀疏渲染,连续消息常无自己的时间,sendTime 为伪造 now)。 + // 范围过滤对无真实时间的消息保留(宁多勿漏),否则 untilMs 模式会系统性丢弃它们 + domTimed: !!time, + sender, original: orig, translation: trans, direction: isR ? 'outgoing' : (isL ? 'incoming' : 'system'), + }); + } + } + return msgs; + } + + // ============ 加载全部聊天历史（DOM wheel 回退方案，API 失败时使用）============ + async function loadAllChatHistory() { + const sinceMs = toMs(state.chatSince), untilMs = toMs(state.chatUntil); // 时间范围毫秒(§4.2);null=不限 + // 增量分支:有 lastDomTime(非首次)→ 页面默认已显示最新 N 条(含新增), + // 只从 DOM 抓 sendTime > lastDomTime 的新增消息,推入 detail.chatMessages(经 state._incDomMessages + // 暂存,scrapeDetailData 消费),不往旧滚,直接结束。 + const it = state.detailQueue[state.queueIndex]; + const inquiryId = it ? it.inquiryId : ''; + if (inquiryId) { + const lastDomTime = await getLastDomTime(inquiryId); + if (lastDomTime) { + const all = extractDomChatMessages(); + const inc = all.filter(m => m.sendTime && m.sendTime > lastDomTime) + .filter(m => !m.domTimed || inChatRange(Date.parse(m.sendTime), sinceMs, untilMs)); // 无真实时间戳的保留(宁多勿漏) + log(`DOM 增量:抓到 ${inc.length} 条新增(sendTime > 上次)`); + if (inc.length) { + state._incDomMessages = inc; + // 更新 lastDomTime = 本批最新 sendTime(max) + let maxT = lastDomTime; + for (const m of inc) if (m.sendTime > maxT) maxT = m.sendTime; + await setLastDomTime(inquiryId, maxT); + } else if (sinceMs != null || untilMs != null) { + // 范围模式下 0 新增也是决定性结果(无新的范围内消息),防止消费点穿透到未过滤全量提取(同全量分支语义) + state._incDomMessages = []; + state._incDomFiltered = true; + } + return; // 不往旧滚,只增量 + } + } + + // === 全量分支(首次,无 lastDomTime):走原 wheel 滚动逻辑 === + // 阿里消息历史通过“向上滚动”懒加载（wheel 事件触发），不是按钮。 + // 注意：.ui-header-message-box-list-more（“查看更多>”）是跳转到旧版消息中心(atmgateway)的链接， + // 点击会打开新标签页，绝不能当作“加载历史”按钮。 + const container = document.querySelector('.common-load-more'); + if (!container) { + log('未找到聊天滚动容器 .common-load-more，跳过历史加载', 'info'); + return; + } + const MAX_ROUNDS = state.chatLimit > 0 ? Math.ceil(state.chatLimit / 20) : 300; // chatLimit 来自 popup;0=全量(300 轮≈6000 条) + // 阿里历史加载存在“平台期”（连续多轮不变后又突增），STABLE_LIMIT 必须 ≥10 才不会误判截断。 + // 实测 13776617920 加载到 2635+ 条仍未到顶，旧值 3 会严重截断。 + // 安全层 #2:STABLE_LIMIT 12 → 18,容忍更长的平台期。 + const STABLE_LIMIT = 18; // 连续 N 轮不变才认定到顶 + let prevCount = document.querySelectorAll('.message-item-wrapper').length; + let stable = 0; + let lastLog = prevCount; + + for (let i = 0; i < MAX_ROUNDS; i++) { + if (state.phase === 'paused') return; // 支持暂停中断 + container.scrollTop = 0; + container.dispatchEvent(new WheelEvent('wheel', { deltaY: -500, bubbles: true, cancelable: true })); + // 安全层 #2:每次滚动 2-6 秒大范围随机,替代原 500-1200ms + await randSleep(2000, 6000); + // 安全层 #2:每 15 次滚动插一个超长停顿(30-90 秒,模拟走神) + if ((i + 1) % 15 === 0) { + log('聊天历史滚动: 超长停顿(模拟走神)'); + await randSleep(30000, 90000); + } + + // DOM 时间边界:顶部向下找第一个带时间 span 的 wrapper(时间戳稀疏渲染,非每条都有),早于 since → 停滚(spec §4.4) + if (sinceMs != null) { + let oldestT = null; + for (const mw of document.querySelectorAll('.message-item-wrapper')) { + const te = mw.querySelector('.item-base-info'); + let hit = false; + if (te) for (const sp of te.querySelectorAll('span')) { + const m = sp.innerText.match(/(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})/); + if (m) { oldestT = new Date(m[1] + 'T' + m[2]).getTime(); hit = true; break; } + } + if (hit) break; + } + if (oldestT != null && oldestT < sinceMs) { log('DOM 时间边界,停止滚动'); break; } + } + + const newCount = document.querySelectorAll('.message-item-wrapper').length; + if (newCount === prevCount) { + stable++; + if (stable >= STABLE_LIMIT) break; + } else { + stable = 0; + prevCount = newCount; + } + if (newCount - lastLog >= 100) { + log(`聊天历史加载中: ${newCount} 条`); + lastLog = newCount; + } + } + log(`聊天记录加载完成: ${prevCount} 条`); + + // 全量抓完后:水位用未过滤集合的 max(水位=见到的最新);挂载给消费方的是过滤后的集合 + if (inquiryId) { + const all = extractDomChatMessages(); + let maxT = null; + for (const m of all) if (m.sendTime && (!maxT || m.sendTime > maxT)) maxT = m.sendTime; + if (maxT) await setLastDomTime(inquiryId, maxT); + state._incDomMessages = all.filter(m => !m.domTimed || inChatRange(Date.parse(m.sendTime), sinceMs, untilMs)); // 无真实时间戳的保留(宁多勿漏) + state._incDomFiltered = (sinceMs != null || untilMs != null); // 空数组也具决定性(防消费点穿透到未过滤全量) + } + } + + // 等待 body 文本出现指定内容(用于等异步加载的行为卡字段),超时返回 false + function waitForText(text, timeout = 5000) { + const start = Date.now(); + return new Promise(resolve => { + (function check() { + if (document.body.innerText.includes(text)) return resolve(true); + if (Date.now() - start > timeout) return resolve(false); + setTimeout(check, 200); + })(); + }); + } + + // ============ 详情页数据提取 ============ + function scrapeDetailData(apiMsgs, domHandoff = null, domFiltered = false) { + const data = {}; + const bodyText = document.body.innerText; + + // customerId 提取已抽到 extractCustomerId()(processCurrentDetailPage 也用它做同客户聊天去重) + data.customerId = extractCustomerId(); + + // 买家信息 + const nt = document.querySelector('.name-text'); + data.detailBuyerName = nt ? nt.innerText.trim() : ''; + const cl = document.querySelector('.country-flag-label'); + data.detailCountry = cl ? cl.innerText.trim() : ''; + const av = document.querySelector('.alicrm-customer-detail-card-header-avatar'); + data.detailAvatarUrl = av ? av.getAttribute('src') : ''; + + // 基本信息(需先在 processCurrentDetailPhase 展开折叠面板,才能看到手机/座机/商业类型/销售平台/公司官网) + const ci = document.querySelector('.contact-base-info'); + if (ci) { + const t = ci.innerText || ''; + const cleanV = v => { const x = (v || '').trim(); return (x === '' || x === '-') ? '' : x; }; + data.company = cleanV((t.match(/公司名称\s*\n?\s*([^\n]+)/) || [])[1]); + data.email = cleanV((t.match(/邮箱\s*\n?\s*([^\n]+)/) || [])[1]); + data.registerTime = cleanV((t.match(/注册时间\s*\n?\s*([^\n]+)/) || [])[1]); + data.phone = cleanV((t.match(/手机\s*\n?\s*([+\d][\d\s\-()]{4,30})/) || [])[1]); + data.landline = cleanV((t.match(/座机\s*\n?\s*([+\d][\d\s\-()]{4,30})/) || [])[1]); + data.businessType = cleanV((t.match(/商业类型\s*\n?\s*([^\n]+)/) || [])[1]); + data.salesPlatform = cleanV((t.match(/销售平台\s*\n?\s*([^\n]+)/) || [])[1]); + data.website = cleanV((t.match(/公司官网\s*\n?\s*([^\n]+)/) || [])[1]); + } + + // 标签 + const tagList = ['流失预警', '高潜复购', '复购客户', 'TM沟通', '新客户', '大客户', '高价值客户', '待跟进']; + data.buyerTags = tagList.filter(t => bodyText.includes(t)).join(', '); + + // 特征（按真实阿里国际站买家特征词表更新） + const featList = ['工厂偏好', '物流商偏好', '轻定制偏好', 'logo定制偏好', '其他沟通工具偏好', '货代服务偏好', + '付款方式偏好', '组货方式偏好', '高质量偏好', '新品偏好', '原装偏好', '商品价格敏感']; + data.buyerFeatures = featList.filter(f => bodyText.includes(f)).join(', '); + + // 行为数据 + const b = {}; + // alibaba 行为卡 DOM 顺序是「数字 + 标签」(如 "318 产品浏览数"), + // 正则按「数字 + 标签」匹配,否则会错位抓到下一个指标的值。 + b.productViews = (bodyText.match(/(\d+)\s*\n?\s*产品浏览数/) || [])[1] || ''; + b.validInquiries = (bodyText.match(/(\d+)\s*\n?\s*有效询盘/) || [])[1] || ''; + b.validRFQ = (bodyText.match(/(\d+)\s*\n?\s*有效RFQ数/) || [])[1] || ''; + b.loginDays = (bodyText.match(/(\d+)\s*\n?\s*登录天数/) || [])[1] || ''; + b.spamInquiries = (bodyText.match(/(\d+)\s*\n?\s*垃圾询盘数/) || [])[1] || ''; + b.blacklistCount = (bodyText.match(/(\d+)\s*\n?\s*被加为黑名单数/) || [])[1] || ''; + b.topIndustry = (bodyText.match(/最常采购行业\s*\n?\s*(.+?)(?:\n|$)/) || [])[1] || ''; + data.behavior = b; + + // 最近询盘产品(产品图片 URL 列表,alibaba「最近询盘产品」区是图片) + const recentImgs = document.querySelectorAll('.alicrm-customer-behavioral-card-recent-inquries img'); + data.recentInquiryProducts = Array.from(recentImgs).map(i => i.src).filter(Boolean); + + // 店内行为(店铺内:询盘产品名+数量 + 浏览产品图/链接)。部分询盘才有这个板块。 + const minisite = document.querySelector('.alicrm-customer-behavioral-card-minisite-behavior'); + if (minisite) { + const inq = minisite.querySelector('.alicrm-customer-behavioral-card-minisite-behavior-inquiry'); + const inqText = inq ? inq.innerText : ''; + const pm = inqText.match(/询盘产品\s*([\s\S]*?)\s*×\s*\d/); + data.storeInquiryProduct = pm ? pm[1].trim().replace(/\n+/g, ' ') : ''; + const qm = inqText.match(/×\s*(\d+)/); + data.storeInquiryQty = qm ? qm[1] : ''; + const vis = minisite.querySelector('.alicrm-customer-behavioral-card-minisite-behavior-visit'); + data.storeVisitProducts = []; + if (vis) { + for (const vp of vis.querySelectorAll('.visit-product-container')) { + const img = vp.querySelector('img'); + const a = vp.querySelector('a[href]'); + let href = a ? (a.getAttribute('href') || '') : ''; + if (href.startsWith('//')) href = 'https:' + href; + const pid = href.match(/_(\d+)\.html/); + data.storeVisitProducts.push({ img: img ? img.src : '', url: href, productId: pid ? pid[1] : '' }); + } + } + } + + // 聊天记录（优先 API 结果，回退 DOM 抓取） + if (state.chat) { + let msgs; + if (state.chatSkipped) { + // 同客户去重跳过:不抓 chat(后端已归代表),避免 DOM fallback 抓出重复的 dom- 消息 + msgs = []; + state.chatSkipped = false; + } else { + // loadAllChatHistory 的 DOM handoff 由调用方(processCurrentDetailPage)在重试循环外 + // 取出并传入(避免重试调用清空后落穿未过滤提取)。优先级:apiMsgs > 增量暂存 > 全量 DOM 抓取。 + const incMsgs = domHandoff; + const incFiltered = domFiltered; // 空数组+过滤标记 → 空也有效(范围内0条,spec §4.4 降级) + // apiMsgs != null 表示 API 成功(含增量命中返回空数组 []=无新增);只有 API 失败(apiMsgs=null)才回退 DOM。 + msgs = apiMsgs != null ? apiMsgs : null; + if (!msgs) msgs = (incMsgs && (incMsgs.length || incFiltered)) ? incMsgs : null; + if (!msgs) msgs = extractDomChatMessages(); + } + data.chatCount = msgs.length; + data.chatMessages = msgs; + data.chatText = msgs.map(m => JSON.stringify(m)).join('|||'); + } + + return data; + } + + // 等待详情页加载 + function waitForLoad() { + return new Promise(resolve => { + let tries = 0; + const t = setInterval(() => { + tries++; + const txt = document.body.innerText || ''; + if (document.querySelector('a.customer-detail-link') || + document.querySelector('.contact-base-info') || + document.querySelector('.message-item-wrapper') || + hitsSkipKeyword(txt) || tries > 60) { // 30s，也认账号注销/管控提示 → 快速放行给 SKIP 检测 + clearInterval(t); + resolve(); + } + }, 500); + }); + } + + // ============ 完成 ============ + async function finishAll() { + // 从 background.js 获取所有已存储的详情数据 + try { + const bgResult = await new Promise(resolve => { + chrome.runtime.sendMessage({ type: 'getBgData' }, resp => resolve(resp)); + }); + if (bgResult && bgResult.records) { + state.detailData = bgResult.records; + } else { + // 回退：用本地数据 + if (state.dedup) { + state.detailData = Object.values(state.customerIdMap); + } + } + } catch (e) { + if (state.dedup) { + state.detailData = Object.values(state.customerIdMap); + } + } + + log(`最终 ${state.detailData.length} 条详情数据（原始 ${state.listData.length} 条）`, 'success'); + state.phase = 'done'; + await saveState(); + await randSleep(800, 2000); + log('全部抓取完成！', 'success'); + sendToPopup('done', {}); + window.location.href = LIST_URL; + } + + // ============ CSV 导出 ============ + function buildCSV(data, mode) { + const rows = data || state.detailData; + if (!rows.length) return ''; + const m = mode || state.mode; + // 仅 detail 模式且记录含详情字段时才输出详情列，避免列表模式多出 17 个空列 + const hasDetail = m === 'detail' && rows.some(r => 'customerId' in r || 'detailBuyerName' in r); + + const headers = [ + '询价单号', '创建时间', '更新时间', '买家名称', '国家代码', + '消息预览', '负责人', '状态', '是否标记', + ]; + if (hasDetail) { + headers.push( + 'customerId', '详情买家名', '详情国家', '头像URL', + '公司名称', '邮箱', '注册时间', '买家标签', '买家特征', + '产品浏览数', '有效询盘', '有效RFQ', '登录天数', + '垃圾询盘数', '最常采购行业', '聊天消息数', '聊天记录' + ); + } + + const esc = v => { + if (v == null) return ''; + const s = String(v); + return (s.includes(',') || s.includes('"') || s.includes('\n')) + ? '"' + s.replace(/"/g, '""') + '"' + : s; + }; + + let csv = headers.map(esc).join(',') + '\n'; + for (const row of rows) { + const line = [ + row.inquiryId, row.createTime, row.updateTime, row.buyerName, + row.countryCode, row.msgPreview, row.manager, row.status, + row.isFlagged ? '是' : '否', + ]; + if (hasDetail) { + const b = row.behavior || {}; + line.push( + row.customerId || '', row.detailBuyerName || '', row.detailCountry || '', + row.detailAvatarUrl || '', row.company || '', row.email || '', + row.registerTime || '', row.buyerTags || '', row.buyerFeatures || '', + b.productViews || '', b.validInquiries || '', b.validRFQ || '', + b.loginDays || '', b.spamInquiries || '', b.topIndustry || '', + row.chatCount || '', row.chatText || '' + ); + } + csv += line.map(esc).join(',') + '\n'; + } + return csv; + } + + // ============ 初始化：页面加载后判断当前阶段 ============ + await loadState(); + + const pageType = isDetailPage() ? '详情页' : '列表页'; + log(`插件已就绪 (${pageType})，当前阶段: ${state.phase}`, 'info'); + + if (state.phase === 'detail' && isDetailPage()) { + // 上次正在抓详情页，页面导航后恢复 + await randSleep(1500, 3000); // 等页面完全加载 + processCurrentDetailPage(); + } else if (state.phase === 'detail' && !isDetailPage()) { + // 在详情阶段但当前在列表页，继续 + await randSleep(800, 1800); + runDetailPhase(); + } else if (state.phase === 'list') { + // 列表阶段（从详情页点 start 跳转过来后），自动开始，否则会卡死 + await randSleep(1200, 3000); + runListPhase(); + } + // 其他状态（idle/done/paused）等待用户操作 + +})(); diff --git a/docs/DEPLOY-ALIBB.md b/docs/DEPLOY-ALIBB.md new file mode 100644 index 0000000..b00849f --- /dev/null +++ b/docs/DEPLOY-ALIBB.md @@ -0,0 +1,260 @@ +# alibb 生产部署文档(1Panel + OpenResty + MySQL 8.4) + +> 基于 2026-08-27 实际生产部署整理(服务器重装后完整恢复验证通过)。 +> 与 2026-07-15 首次部署的差异见 §8。恢复/迁移专用 runbook 见 +> [2026-08-25-server-restore-redeploy.md](superpowers/plans/2026-08-25-server-restore-redeploy.md)。 + +## 1. 架构 + +``` +浏览器扩展(抓 alibaba 询盘) + │ POST /api/ingest(Bearer INGEST_TOKEN) + ▼ +http://219.157.255.213:23010 ── OpenResty 站点「alibaba」(1Panel 面板管理,host 网络) + ├─ / → 前端 dist 静态(/opt/1panel/www/sites/alibaba/index) + └─ /api/* → http://127.0.0.1:8090(反代,client_max_body_size 50m) + │ + ▼ + alibb-backend 容器(Express + Prisma,127.0.0.1:8090→8080) + │ + ▼ + 1Panel MySQL 8.4(容器 1Panel-mysql-YL0F,库 alibaba) +``` + +管理后台:浏览器开 http://219.157.255.213:23010 → admin 登录(密码见服务器 `~/alibb-secrets.txt`)。 + +## 2. 环境与信息总表 + +| 项 | 值 | +|---|---| +| 服务器 | 阿里云 ECS `219.157.255.213`,Ubuntu 26.04 LTS | +| 登录用户 | `kt`(sudo 免密);SSH 密钥 `C:\Users\30997\.ssh\alibb_deploy` | +| SSH(本机) | `/c/Windows/System32/OpenSSH/ssh.exe -i 'C:\Users\30997\.ssh\alibb_deploy' kt@219.157.255.213`(Git Bash 自带 ssh 因私钥权限 644 不可用) | +| Docker 网络 | `1panel-network`(bridge,1Panel 安装时自动创建) | +| MySQL | 容器 `1Panel-mysql-YL0F`,镜像 mysql:8.4.11,`127.0.0.1:3306`,root 密码见容器 env `MYSQL_ROOT_PASSWORD` | +| 业务库/账号 | 库 `alibaba`(utf8mb4),用户 `alibaba@%`,密码 `wecbni5zRRnZrMMh`,授权 `ALL ON alibaba.*` | +| 后端 | 容器 `alibb-backend`,代码 `~/alibb-server/backend`,`127.0.0.1:8090→8080`,restart unless-stopped | +| OpenResty | 容器 `1Panel-openresty-Seym`,**host 网络模式**,管理 80/443/23010 | +| 站点 | 1Panel 面板管理,代号 **alibaba**;目录 `/opt/1panel/www/sites/alibaba/`(index/ 前端、log/ 日志) | +| 站点 conf | `/opt/1panel/www/conf.d/alibaba.conf`(面板生成的 server 块 + 手工并入的 SPA fallback 与 /api 反代) | +| 对外端口 | 23010(云安全组已放行);8090/3306 仅本机 | +| 生产密钥 | JWT_SECRET / INGEST_TOKEN / ADMIN_USERNAME / ADMIN_PASSWORD 内嵌于 `~/alibb-server/backend/docker-compose.yml`;明文副本 `~/alibb-secrets.txt`(600) | +| 本机代码 | 扩展 `d:\alibb`(master);后端 `d:\alibb-server\backend`(master,compose 为 **dev 版**);前端 `d:\alibb-server\frontend`(main) | +| 备份 | 本机 `D:\alibb-server\backups\`(权威);服务器副本 `~/alibb-backup-20260825/` | + +## 3. 从零部署(完整步骤) + +> 以下命令全部在 2026-08-27 实际验证。分两段:本机(Windows/Git Bash)与服务器(SSH)。 + +### 3.1 SSH 准备 + +```bash +# 首次:云控制台 VNC 或 1Panel 面板「主机→终端」,root 执行(公钥替换成自己的) +useradd -m -s /bin/bash kt; echo "kt ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/kt; \ +mkdir -p /home/kt/.ssh; echo "<本机公钥内容>" > /home/kt/.ssh/authorized_keys; \ +chown -R kt:kt /home/kt/.ssh; chmod 700 /home/kt/.ssh; chmod 600 /home/kt/.ssh/authorized_keys + +# 本机清除旧 host key(重装后必做,否则报 HOST IDENTIFICATION HAS CHANGED) +/c/Windows/System32/OpenSSH/ssh-keygen.exe -R 219.157.255.213 +``` + +### 3.2 系统基础 + +```bash +# Docker +curl -fsSL https://get.docker.com | sudo sh +sudo tee /etc/docker/daemon.json <<'EOF' +{ "registry-mirrors": ["https://docker.1panel.live"] } +EOF +sudo systemctl restart docker +# 1Panel(官方脚本,装时设面板密码) +curl -sSL https://resource.fit2cloud.com/1panel/package/quick_start.sh -o quick_start.sh && sudo bash quick_start.sh +``` + +### 3.3 1Panel 应用 + +面板 → 应用商店安装: + +- **OpenResty**(1.31+,host 网络模式) +- **MySQL 8.4**(装完记容器名,形如 `1Panel-mysql-XXXX`,下称 ``) + +### 3.4 数据库 + +方式 A(面板):数据库 → 创建,库名 `alibaba`,用户 `alibaba`,权限 `alibaba.*`(密码设 `wecbni5zRRnZrMMh` 与 compose 一致,或建完后 ALTER)。 + +方式 B(命令,推荐): +```bash +M= +R=$(sudo docker inspect $M --format '{{range .Config.Env}}{{println .}}{{end}}' | grep MYSQL_ROOT_PASSWORD | cut -d= -f2) +sudo docker exec -i -e MYSQL_PWD=$R $M mysql -uroot <<'SQL' +CREATE DATABASE alibaba CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +CREATE USER 'alibaba'@'%' IDENTIFIED BY 'wecbni5zRRnZrMMh'; +GRANT ALL PRIVILEGES ON `alibaba`.* TO 'alibaba'@'%' WITH GRANT OPTION; +FLUSH PRIVILEGES; +SQL +# 验证 +sudo docker exec -e MYSQL_PWD=wecbni5zRRnZrMMh $M mysql -ualibaba -e "SELECT 1;" +``` + +导入存量数据(有备份时): +```bash +sudo docker exec -i -e MYSQL_PWD=$R $M mysql -uroot < alibaba-db.sql +``` + +### 3.5 后端 + +```bash +# 本机推代码(排除 node_modules/dist/.git) +cd /d && tar czf /tmp/alibb-server.tgz --exclude='alibb-server/*/node_modules' --exclude='*/dist' --exclude='*/.git' alibb-server +/c/Windows/System32/OpenSSH/scp.exe -i 'C:\Users\30997\.ssh\alibb_deploy' /tmp/alibb-server.tgz kt@219.157.255.213:/tmp/ +# 服务器解包 +ssh ... 'tar xzf /tmp/alibb-server.tgz -C ~' +``` + +**compose 关键项**(服务器生产版,本地 dev compose 上线前必须对齐): +```yaml +services: + backend: + container_name: alibb-backend + build: . + ports: ["127.0.0.1:8090:8080"] + environment: + DATABASE_URL: "mysql://alibaba:wecbni5zRRnZrMMh@:3306/alibaba?charset=utf8mb4&timezone=Z" + INGEST_TOKEN / JWT_SECRET / ADMIN_USERNAME / ADMIN_PASSWORD: 生产值(强随机,首次生成见下) + NODE_ENV: production + restart: unless-stopped +networks: + 1panel-network: + external: true +``` + +首次生成密钥(已有备份则沿用旧值,扩展/老 JWT 不失效): +```bash +openssl rand -hex 24 # INGEST_TOKEN +openssl rand -hex 32 # JWT_SECRET +openssl rand -hex 16 # ADMIN_PASSWORD +``` + +构建启动(注意 MySQL 容器名替换): +```bash +cd ~/alibb-server/backend +sed -i "s/<旧MYSQL容器名>/<新MYSQL容器名>/g" docker-compose.yml +sudo docker compose build backend && sudo docker compose up -d +sudo docker compose logs backend | tail # 期望:No pending migrations + listening on 8080 +``` + +### 3.6 站点 + 前端 + +**先在 1Panel 面板建站**(网站 → 创建网站):类型**静态网站**,主域名 `219.157.255.213`,端口 `23010`,代号 `alibaba`。 +> 注意:面板检查端口占用——若 conf.d 里有手工 conf 占着 23010 会报"端口已被占用",先移除再建。 + +面板会生成 `/opt/1panel/www/sites/alibaba/` 和 `/opt/1panel/www/conf.d/alibaba.conf`。然后合并前端与反代: + +```bash +S=/opt/1panel/www/sites/alibaba +# 前端 dist(备包直解最快;index 子目录在面板建站时已建) +sudo tar xzf alibb-frontend-dist.tgz -C $S/ && sudo chmod -R a+rX $S/index +# 或源码构建: +# cd ~/alibb-server/frontend && sudo docker run --rm -v "$PWD":/app -w /app node:22-alpine sh -c 'npm ci && npm run build' +# sudo rm -rf $S/index/* && sudo cp -r dist/. $S/index/ && sudo chmod -R a+rX $S/index + +# 在 conf 的 root 行后并入 SPA fallback + /api 反代(host 网络 → 必须宿主端口,不能用容器名) +sudo sed -i "/root \/www\/sites\/alibaba\/index;/a\\ + location / { try_files \$uri \$uri/ /index.html; }\\ + location ^~ /api/ {\\ + proxy_pass http://127.0.0.1:8090;\\ + client_max_body_size 50m;\\ + proxy_set_header Host \$host;\\ + proxy_set_header X-Real-IP \$remote_addr;\\ + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;\\ + }" /opt/1panel/www/conf.d/alibaba.conf +``` + +重载 OpenResty: +```bash +sudo docker exec 1Panel-openresty-Seym openresty -t && sudo docker exec 1Panel-openresty-Seym openresty -s reload +``` + +### 3.7 安全组 + +云控制台放行 **TCP 23010** 入方向(源 0.0.0.0/0 或收窄)。8090/3306 不放行。 + +### 3.8 验证 + +```bash +curl http://219.157.255.213:23010/ # 200 index.html +curl http://219.157.255.213:23010/inquiries # 200(SPA fallback) +curl http://219.157.255.213:23010/api/health # {"ok":true} +``` +浏览器登录 admin / ADMIN_PASSWORD,核对询盘/客户/消息数据。 + +### 3.9 扩展配置 + +popup:serverUrl `http://219.157.255.213:23010` + INGEST_TOKEN(~/alibb-secrets.txt)→ 授权 → 启用推送 → 抓取。 + +## 4. 增量更新(改代码后) + +| 改动 | 操作 | +|---|---| +| 后端 | 传变更文件 → `cd ~/alibb-server/backend && sudo docker compose build backend && sudo docker compose up -d`(migrate 自动跑) | +| 前端 | 传变更 → 服务器 build(§3.6 方式 B)→ 复制 dist → 无需 reload(静态文件即时生效) | +| 扩展 | 本机改 → `chrome://extensions` 刷新扩展(用户操作) | + +## 5. 常用运维 + +```bash +# 后端日志 / 重启 +sudo docker compose -f ~/alibb-server/backend/docker-compose.yml logs -f backend +sudo docker compose -f ~/alibb-server/backend/docker-compose.yml restart backend + +# 查库 +sudo docker exec -e MYSQL_PWD=wecbni5zRRnZrMMh 1Panel-mysql-YL0F mysql -ualibaba alibaba + +# 行数快照 +sudo docker exec -e MYSQL_PWD=wecbni5zRRnZrMMh 1Panel-mysql-YL0F mysql -ualibaba alibaba -e \ +"SELECT (SELECT COUNT(*) FROM inquiries) i,(SELECT COUNT(*) FROM customers) c,(SELECT COUNT(*) FROM chat_messages) m;" + +# 清询盘重抓(customers 保留,重抓 upsert 更新) +... mysql ... -e "SET FOREIGN_KEY_CHECKS=0; TRUNCATE chat_messages; TRUNCATE inquiries; SET FOREIGN_KEY_CHECKS=1;" + +# 重载 OpenResty(改 conf 后) +sudo docker exec 1Panel-openresty-Seym openresty -t && sudo docker exec 1Panel-openresty-Seym openresty -s reload +``` + +## 6. 备份与恢复 + +- **备份**(本机执行,产物存 `D:\alibb-server\backups\`): +```bash +/c/Windows/System32/OpenSSH/ssh.exe -i 'C:\Users\30997\.ssh\alibb_deploy' kt@219.157.255.213 \ +'sudo docker exec 1Panel-mysql-YL0F mysqldump -ualibaba -pwecbni5zRRnZrMMh --single-transaction --no-tablespaces --routines --triggers --databases alibaba' > D:\alibb-server\backups\alibaba-db-$(date +%Y%m%d).sql +``` +- 完整备份方案:[2026-08-25-server-reinstall-backup.md](superpowers/plans/2026-08-25-server-reinstall-backup.md) +- 服务器重装恢复 runbook:[2026-08-25-server-restore-redeploy.md](superpowers/plans/2026-08-25-server-restore-redeploy.md)(含 2026-08-27 实操记录) + +## 7. 故障排查 + +| 症状 | 原因 → 处理 | +|---|---| +| `/api` 502 | ① backend 挂了:`docker ps` / `logs backend`;② 反代写成了 `http://backend:8080`——host 网络模式解析不了容器名,必须 `http://127.0.0.1:8090` | +| openresty -t 报 host not found in upstream | 同上,proxy_pass 用容器名导致 | +| 后端起不来,连接拒绝 | DATABASE_URL 里 MySQL 容器名不对(重装/重建后容器名后缀会变) | +| 管理后台登录 401 | ADMIN_USERNAME/PASSWORD 与 compose 不符;扩展推送 401 → INGEST_TOKEN 不符 | +| migrate 报表不存在 | 库导错/导到别的库,重做 §3.4 | +| MySQL 8.x 用户连不上(老客户端) | `ALTER USER 'alibaba'@'%' IDENTIFIED WITH caching_sha2_password BY '...'`(Prisma 支持 caching_sha2,一般无需处理) | +| 换服务器/IP | 改 3 处:扩展 serverUrl、站点 conf `server_name`、云安全组 | + +## 8. 与 2026-07-15 旧环境差异(重要) + +| 项 | 旧(7 月) | 新(2026-08-27) | +|---|---|---| +| MySQL | 5.7.44(容器 1Panel-mysql-sLiG,0.0.0.0:3306) | 8.4.11(容器 1Panel-mysql-YL0F,**127.0.0.1:3306 已收紧**) | +| OpenResty | bridge 网络,反代可用容器名 `http://backend:8080` | **host 网络,反代必须 `http://127.0.0.1:8090`** | +| 1Panel 路径 | /opt/1panel/apps/openresty/openresty/www/sites/... | **/opt/1panel/www/sites/...**(容器内仍挂 /www,旧 conf 兼容) | +| 系统 | Ubuntu 24.04(kt-W580G20) | Ubuntu 26.04(xyh-W580G20) | +| 站点管理 | 1Panel 面板建站(站点 alibb) | 1Panel 面板建站(代号 **alibaba**,2026-08-27 起面板可见可管理) | + +## 9. 已知限制(业务) + +- **折叠面板字段抓不到**:客户资料折叠面板里的手机/座机/商业类型等,alibaba 校验 `event.isTrusted`(只认真实鼠标点击),扩展无法展开;这些字段多数买家本就是"-"。 +- **风控**:抓取间隔已随机化(randSleep);大量抓取分批 + 中途暂停。 +- 公网明文 HTTP(23010):内网/测试用途可接受;若需公网长期使用,建议套域名 + HTTPS(1Panel 可申请 Let's Encrypt)。 diff --git a/docs/PROJECT.md b/docs/PROJECT.md new file mode 100644 index 0000000..2c73eb7 --- /dev/null +++ b/docs/PROJECT.md @@ -0,0 +1,175 @@ +# 阿里国际站询盘数据采集与管理系统 · 项目介绍 + +> 基于 2026-08-27 代码现状整理(扩展 `d:\alibb` + 服务端 `d:\alibb-server`)。 +> 部署细节见 [DEPLOY-ALIBB.md](DEPLOY-ALIBB.md),后端设计见 [superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md](superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md)。 + +## 1. 项目概述 + +本项目是一套**阿里国际站(alibaba.com)询盘数据的采集与管理系统**,由三个部分组成: + +| 组成部分 | 位置 | 技术 | 角色 | +|---|---|---|---| +| **采集端** Chrome 扩展 | `d:\alibb`(本仓库) | Chrome MV3、原生 JS | 抓取询盘列表、客户资料、行为数据、聊天记录 | +| **服务端** 后端 API | `d:\alibb-server\backend` | Node.js + Express + TypeScript + Prisma + MySQL | 接收推送、幂等入库、提供查询/统计/导出 API | +| **服务端** 管理后台 | `d:\alibb-server\frontend` | Vue3 + Element Plus + Pinia + ECharts | 询盘列表/详情/聊天查看/统计报表 | + +核心链路: + +``` +浏览器(登录阿里主账号) + Chrome 扩展 content.js + ├─ 列表抓取(DOM 翻页)→ 详情抓取(DOM)→ 聊天抓取(阿里内部 API 直拉) + ├─ 双写 A(本地):chrome.storage 汇聚 + customerId 去重 → CSV 导出 + └─ 双写 B(推送):IndexedDB 队列 → alarm 每分钟消费 → POST /api/ingest(Bearer Token) + ↓ 失败指数退避(≤8 次),成功出队 +服务器(阿里云 ECS,1Panel + OpenResty + MySQL 8.4,:23010) + 后端 zod 校验 → 幂等入库(同客户合并 / 消息级去重 / 早返回) + ↓ +管理员浏览器 → Vue3 后台(JWT 登录):列表筛选 / 聊天详情 / 统计 / CSV 导出 +``` + +使用模式:主管理员账号一人操作扩展手动触发抓取(非实时同步),后台数据 = 最近一次抓取的快照;询盘记录的 `manager` 字段天然区分业务员,无需多人登录。 + +--- + +## 2. 采集端:Chrome MV3 扩展(本仓库) + +### 2.1 文件结构 + +| 文件 | 行数/大小 | 职责 | +|---|---|---| +| [manifest.json](../manifest.json) | — | MV3 清单:权限 `storage/activeTab/alarms/unlimitedStorage/scripting`;host 覆盖 `message/alicrm/onetalk.alibaba.com`;`optional_host_permissions` 运行时授权任意后端地址 | +| [content.js](../content.js) | ~1300 行 | 核心:注入 `message.alibaba.com` 的抓取状态机 | +| [background.js](../background.js) | 145 行 | Service Worker:推送队列调度 + 本地数据汇聚(双写) | +| [push/queue.js](../push/queue.js) | — | IndexedDB 持久化队列(库 `inquiry_push`) | +| [push/worker.js](../push/worker.js) | — | 队列消费:POST `/api/ingest`,指数退避,8 次后标 failed | +| [push/payload.js](../push/payload.js) | — | 页面数据 → 后端 zod schema 的字段适配层 | +| [push/config.js](../push/config.js) | — | 推送配置(`chrome.storage.local`) | +| [popup.html](../popup.html) / [popup.js](../popup.js) | — | 操作界面:抓取控制 + 后端推送配置 | +| [tests/](../tests/) | 5 个文件 | vitest 单测(config/queue/payload/sync/worker,fake-indexeddb) | +| [scripts/](../scripts/) | — | `e2e-push.mjs`(推送端到端)、`verify-chat-range.mjs`、`verify-risk-detect.mjs` | +| [使用手册.html](../使用手册.html) | — | 面向操作者的图文五步指南 | + +### 2.2 抓取状态机与断点续抓 + +[content.js](../content.js) 的核心是**跨页面持久化的状态机**:`state.phase ∈ idle | list | detail | paused | done`,存储于 `chrome.storage.local`(键 `alibaba_scraper_state`)。每跳转一个详情页 content script 就被销毁重建,靠恢复 `queueIndex` 继续队列 —— 整轮抓取可跨导航、可暂停恢复、可扩展重载续跑。 + +抓取编排由 `processCurrentDetailPage()` 完成:等待渲染 → 聊天抓取(API 优先)→ 详情提取 → **双写**(本地 `storeDetail` + 推送 `enqueue`)→ 跳下一条。 + +### 2.3 数据抓取能力 + +- **列表页**:DOM 解析 `a.inquiry-item`(询价单号/时间/买家/国家/消息预览/负责人/状态等 13 字段),100 条/页,自动翻页; +- **详情页**:客户资料(邮箱/电话/公司/注册时间等)、买家标签(8 词表)、买家特征(12 词表)、近 90 天行为数据(产品浏览/有效询盘/RFQ/登录天数/垃圾询盘/黑名单数)、店内行为与最近询盘产品; +- **聊天记录**(最复杂模块): + - **API 直拉主路径**:`querySummary.htm` 拿会话加密参数 → `getOpMessages.htm` 游标循环翻页(复用页面 cookie 凭据),单询盘实测 8000+ 条;实测全量 4576 询盘 + 68058 条消息零风控; + - **DOM 滚动回退**:API 失败时模拟 wheel 事件滚动 `.common-load-more` 懒加载; + - **同客户去重**:同买家多询盘共享同一会话,只抓首个(`seenChatCustomer`); + - **消息归一化**(`apiMsgToChat`):方向(incoming/outgoing/system)、富媒体(`img:`/`file:`/`video:` 前缀 + clouddisk 直链构造)、翻译(嵌套 JSON 双重 parse)。 + +### 2.4 反风控设计(贯穿全文件) + +| 机制 | 说明 | +|---|---| +| 随机时延 | 翻页 1.2–3s、聊天每轮 2–5s、每 15 轮插入 30–90s"模拟走神" | +| 作息时段 | 默认 8–12 / 14–18 点工作,会话级 ±30 分钟抖动,非工作时段自动挂起 | +| 风控检测 | `RISK_KEYWORDS`(验证码/captcha/blocked 等)命中 → 暂停 + popup 红色横幅报警;只浅扫 API 响应顶层字段,不扫消息正文防误判 | +| 跳过策略 | 注销/管控账号(`SKIP_KEYWORDS`)直接跳过不暂停 | +| 增量水位 | 按 inquiryId 记录已抓最新消息(`lastTopMsg_*`),重抓只拉增量,只前进不倒退 | +| 时间范围 | 近 1/2/3 月或自定义起止;`until` 跳游标直接从截止时间往回翻,大幅减少请求量 —— 同时是风控减压手段 | + +### 2.5 推送链路(可靠性) + +``` +content.js 抓完一条 → sendMessage 'enqueue' → SW 写 IndexedDB(不丢) + → 立即 + alarm 'push-tick'(每 1 分钟)触发 processQueue() + → 每批 3 条,POST {serverUrl}/api/ingest(Bearer Token) + → 成功标 sent;失败指数退避(2^n 秒,封顶 1h),8 次后标 failed(popup 红字提示) +``` + +关键设计:worker **每次自己 `loadConfig()`**(无状态),因为 MV3 SW 休眠重启会清空模块级变量 —— 缓存配置会导致永不推送。 + +### 2.6 popup 界面 + +抓取模式(仅列表/列表+详情)、条数(全部/前10/前50)、customerId 去重开关、聊天上限、**聊天时间范围**(全部/近1-3月/自定义日期)、后端推送设置(服务器 URL + Token + 授权域名 + 启用开关,默认指向生产 `http://219.157.255.213:23010`)、实时推送状态条(已推/待推/失败,2s 刷新)、开始/暂停/恢复/导出 CSV/清空数据。 + +--- + +## 3. 服务端后端(d:\alibb-server\backend) + +### 3.1 技术栈与结构 + +Express 4 + TypeScript + Prisma 5 + MySQL 8,zod 校验,Docker 两阶段构建(node:20-slim),容器启动自动 `prisma migrate deploy`。目录:`src/{routes,services,middleware,utils,config,lib}` + `prisma/` + `tests/`(vitest + testcontainers 真库集成测试)。 + +### 3.2 API 清单 + +| 方法 + 路径 | 鉴权 | 功能 | +|---|---|---| +| `GET /api/health` | 无 | 探活(DB 连通性) | +| `POST /api/ingest/` | Ingest Token + 60次/min 限流 | 扩展推送入口,幂等入库 | +| `POST /api/auth/login` / `refresh` | 无 / JWT | 管理员登录(bcrypt)、续签 | +| `GET /api/inquiries` | JWT | 列表:分页 + 国家/业务员/状态/时间筛选 + 关键词搜索 + 排序 | +| `GET /api/inquiries/:id` | JWT | 详情 + 客户 + 行为 + 消息数 | +| `GET /api/inquiries/:id/messages` | JWT | 聊天游标分页(复合键 `(sendTime, messageId)`,微信式向上翻页) | +| `GET /api/inquiries/:id/messages/export` | JWT | 单询盘聊天 CSV 流式导出(预留) | +| `GET /api/stats/overview` | JWT | 总数 / 国家 Top10 / 业务员分布 / 近 12 月趋势 | +| `GET /api/export` | JWT | 询盘列表 CSV 导出(21 列中文表头,与列表筛选一致) | + +### 3.3 ingest 幂等核心(亮点) + +1. **同客户合并**:同一 customerId 只保留一条"代表询盘"(最早创建),后续询盘追加进 `allInquiries` JSON 数组,消息统一归入代表 —— 不产生冗余行; +2. **消息级幂等**:按 `message_id` 唯一约束 `createMany skipDuplicates`,500 条/批,单批失败只计数不回滚(`has_failed_batches` 标记,下次推送强制重插); +3. **早返回优化**:"消息数 + 最大 sendTime + 无失败批次" 三项指纹未变 → 跳过批量插入直接返回,避免重复重写几千条消息; +4. customer upsert 用 `skipNulls`,本次没抓到的字段不覆盖已落库值。 + +### 3.4 数据模型(4 张表) + +- **customers**:`customer_id` 唯一;资料 + 行为七字段(可空,区分"无数据"与"为 0")+ 店内行为 JSON 列; +- **inquiries**:`inquiry_id` 唯一;`all_inquiries`(同客户合并)、`messages_count`/`last_send_time`(幂等指纹)、`has_failed_batches`; +- **chat_messages**:`message_id` 唯一(幂等键);`content MEDIUMTEXT`;复合索引 `(inquiry_id, send_time, message_id)` 恰好覆盖游标分页; +- **users**:管理员(bcrypt,空表时按环境变量播种)。 + +### 3.5 鉴权 + +两套并行:扩展 → 静态 `INGEST_TOKEN`(`timingSafeEqual` 常量时间比较防时序攻击);后台 → JWT(HS256,7 天,登录签发)。 + +--- + +## 4. 管理后台前端(d:\alibb-server\frontend) + +### 4.1 页面 + +| 路由 | 页面 | 功能要点 | +|---|---|---| +| `/login` | 登录 | JWT + open-redirect 防护 | +| `/inquiries` | 询盘列表 | 筛选栏(国家/业务员/状态/时间范围/搜索/排序)、表格分页、**CSV 导出复用当前筛选**;列表状态存 sessionStorage,进详情返回后恢复 | +| `/inquiries/:id` | 询盘详情 | 客户信息卡 + 行为数据卡(复刻阿里详情页行为区)+ **同会话多询盘列表** + 完整聊天 | +| `/stats` | 统计 | 数字卡(总数/本月新增)+ ECharts:国家 Top10 环形图、业务员柱状图、12 月趋势折线 | + +### 4.2 聊天组件(核心体验) + +- **虚拟滚动**(vue-virtual-scroller):几千条消息流畅渲染,可变高度气泡自动测量; +- **游标分页**:首屏拉最新一页自动滚到底,顶部按钮向上加载历史,prepend 后保持视觉位置不跳动; +- **微信式气泡**:incoming 左 / outgoing 右 / system 居中;消息类型协议(`img:`/`file:`/`video:`/纯链接/纯文本); +- **XSS 防护**:文本一律插值不用 v-html;富媒体 URL 走 alibaba 域名白名单,纯 URL 只许 http/https。 + +### 4.3 工程细节 + +axios 封装(401 全局跳登录、错误脱敏不含 token)、Pinia auth store(JWT 存 localStorage,MVP 可接受)、路由守卫、全套 vitest 单测、双 lint(oxlint + eslint)。 + +--- + +## 5. 部署与运维 + +生产环境(详见 [DEPLOY-ALIBB.md](DEPLOY-ALIBB.md)): + +- 阿里云 ECS `219.157.255.213`(Ubuntu 26.04),对外端口 **23010**; +- 1Panel 面板管理:OpenResty(host 网络,站点 `alibaba`)→ SPA 静态 + `/api/*` 反代 → `alibb-backend` 容器(127.0.0.1:8090)→ 1Panel MySQL 8.4(库 `alibaba`,utf8mb4,仅本机可达); +- `client_max_body_size 50m` 配合大聊天 payload;备份:mysqldump → 本机 `D:\alibb-server\backups\`(权威)+ 服务器副本; +- 三份代码位置:扩展 `d:\alibb`(master)、后端 `d:\alibb-server\backend`(master)、前端 `d:\alibb-server\frontend`(main);服务器重装恢复 runbook 见 [superpowers/plans/](superpowers/plans/)。 + +## 6. 已知限制 + +- 客户资料**折叠面板字段抓不到**(阿里校验 `event.isTrusted` 只认真实鼠标点击),好在多数买家本就是 "-"; +- 公网明文 HTTP(23010),内网/测试可接受,长期公网建议套域名 + HTTPS; +- 数据时效 = 上次手动抓取快照,需浏览器保持阿里页打开; +- 同 messageId 内容被上游修订时早返回会跳过更新(MVP 接受,重推主场景是新增消息)。 diff --git a/docs/superpowers/plans/2026-07-13-backend-mvp.md b/docs/superpowers/plans/2026-07-13-backend-mvp.md new file mode 100644 index 0000000..8027d4e --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-backend-mvp.md @@ -0,0 +1,1106 @@ +# 阿里询盘后台 - 后端 MVP 实施计划 (Plan 1, v2) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 搭建询盘后台的 Node.js + TypeScript + Express + Prisma + MySQL 后端,实现 `/api/ingest`(插件推送幂等入库) + 管理员 JWT 登录 + 询盘列表/详情/消息/单询盘导出/统计/列表导出 API,可用 supertest / curl 完整验证。 + +**Architecture:** 分层(routes / services / middleware / utils)。幂等靠唯一键 + 早返回(`messagesCount` 存**总量** + `has_failed_batches` 防失败批次被跳过);消息批量用 `createMany({ skipDuplicates: true })` 分批 500、失败批次隔离。PrismaClient **惰性初始化**(适配 testcontainers)。ESM(`"type":"module"`)。所有时间存 UTC。 + +**Tech Stack:** Node 18, TypeScript 5(ESM), Express 4, Prisma 5, MySQL 8, zod, jsonwebtoken, bcryptjs, express-rate-limit, vitest, supertest, testcontainers(mysql)。 + +**关联 spec:** `docs/superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md`(v3) + +**v2 修订说明:** 修正 plan review 发现的 4 致命(ESM type / messagesCount 用总量 / 增量回归测试 / 惰性 Prisma)+ 关键重要项(.env 不被覆盖、Task9 单一实现、vitest singleFork、单询盘导出 API、customerUpdate 不覆盖 null、统一测试脚手架、游标 zod 校验、显式 equals、docker exec -i)+ 高价值建议(SIGTERM、buildWhere 复用、e2e 补全、DoD 增量测试)。 + +**前提:** 工作目录 `d:\alibb-server\backend\`,在该目录 `git init`。Docker Desktop 已装。 + +--- + +## 文件结构 + +``` +d:\alibb-server\backend\ + prisma\schema.prisma # 4 表 + 索引(手写,不用 prisma init) + prisma\migrations\ # prisma migrate 生成 + src\ + config\env.ts # loadEnv() 函数(延迟校验,适配测试覆写) + lib\prisma.ts # getPrisma() 惰性单例 + middleware\{jwtAuth,ingestAuth,rateLimit,error}.ts + routes\{auth,ingest,inquiries,stats,export}.ts + services\{auth,ingest,inquiry,stats}.service.ts + utils\{batchInsert,time}.ts + app.ts # createApp() 装配 + server.ts # listen + bootstrapAdmin + SIGTERM + tests\ + setup.ts # setupApp(): 起 testcontainer + migrate + 返回 {app, prisma, resetDb} + fixtures.ts # makePayload / seedInquiries + {health,auth,ingest,batchInsert,inquiries,stats,export}.test.ts + Dockerfile docker-compose.yml scripts\e2e.sh + package.json(type:module) tsconfig.json .env.example .env vitest.config.ts(singleFork) +``` + +--- + +## Task 1: 项目骨架(ESM + 依赖 + vitest singleFork) + +**Files:** `package.json`, `tsconfig.json`, `.env.example`, `.gitignore`, `vitest.config.ts` + +- [ ] **Step 1: 初始化** +```bash +mkdir -p d:/alibb-server/backend && cd d:/alibb-server/backend +git init && npm init -y +``` + +- [ ] **Step 2: 装依赖** +```bash +npm install express @prisma/client prisma zod jsonwebtoken bcryptjs express-rate-limit dotenv cors morgan mysql2 +npm install -D typescript tsx @types/express @types/node @types/jsonwebtoken @types/bcryptjs @types/cors @types/morgan vitest supertest @types/supertest testcontainers @testcontainers/mysql +``` + +- [ ] **Step 3: `package.json` 关键字段**(必须含 `"type":"module"`,否则 ESM build 跑不起来 —— 修正 F1) +```jsonc +{ + "name": "inquiry-backend", + "type": "module", + "scripts": { + "dev": "tsx watch src/server.ts", + "build": "tsc", + "start": "node dist/server.js", + "test": "vitest run", + "db:migrate": "prisma migrate dev", + "db:deploy": "prisma migrate deploy", + "db:generate": "prisma generate" + } +} +``` + +- [ ] **Step 4: `tsconfig.json`** +```jsonc +{ "compilerOptions": { "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", + "strict": true, "esModuleInterop": true, "outDir": "dist", "rootDir": "src", + "resolveJsonModule": true, "skipLibCheck": true, "types": ["vitest/globals"] }, + "include": ["src/**/*"] } +``` +> NodeNext 要求 import 路径带 `.js` 后缀(写 `.js`,tsc 输出 `.js`)。测试文件由 vitest(tsx)跑,不受 tsc 管。 + +- [ ] **Step 5: `.gitignore`** +``` +node_modules/ dist/ .env *.log +``` + +- [ ] **Step 6: `.env.example`**(并 `cp .env.example .env` —— 修正 I2,避免后续被覆盖) +``` +DATABASE_URL="mysql://root:root@localhost:3306/inquiry" +INGEST_TOKEN="dev-ingest-token-1234" +JWT_SECRET="dev-jwt-secret-1234" +JWT_EXPIRES_IN="7d" +ADMIN_USERNAME="admin" +ADMIN_PASSWORD="admin-password-1234" +PORT=8080 +NODE_ENV=development +``` +```bash +cp .env.example .env +``` +> 注:不再有 `JWT_REFRESH_SECRET`(MVP 单密钥,refresh 用 JWT_SECRET;修正 I7)。`TEST_DATABASE_URL` 由测试运行时注入,不入 env。 + +- [ ] **Step 7: `vitest.config.ts`**(`singleFork` 串行,避免并行起多个 mysql 容器 —— 修正 I4) +```ts +import { defineConfig } from 'vitest/config'; +export default defineConfig({ + test: { globals: true, testTimeout: 90_000, hookTimeout: 90_000, + poolOptions: { forks: { singleFork: true } } } +}); +``` + +- [ ] **Step 8: 临时 `src/server.ts`** = `console.log('placeholder');` + +- [ ] **Step 9: Commit** +```bash +git add -A && git commit -m "chore: scaffold ESM backend project" +``` + +--- + +## Task 2: Prisma Schema(手写,跳过 prisma init) + +**Files:** `prisma/schema.prisma`(手写);起本地 mysql 容器;首次迁移 + +- [ ] **Step 1: 起本地 MySQL 容器**(开发用) +```bash +docker run -d --name inquiry-mysql -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=inquiry -p 3306:3306 mysql:8 --max_allowed_packet=128M --character-set-server=utf8mb4 +``` + +- [ ] **Step 2: 手写 `prisma/schema.prisma`**(不用 `prisma init`,它会覆盖 .env —— 修正 I2)。完整对应 spec §6(含 v3 `has_failed_batches`、`Direction.system`、`MEDIUMTEXT content`、复合索引) +```prisma +generator client { provider = "prisma-client-js" } +datasource db { provider = "mysql"; url = env("DATABASE_URL") } + +enum Direction { incoming outgoing system } + +model Customer { + id BigInt @id @default(autoincrement()) + customerId String @unique @map("customer_id") @db.VarChar(64) + name String? @db.VarChar(128) + country String? @index @db.VarChar(64) + email String? @db.VarChar(128) + company String? @db.VarChar(255) + registerTime DateTime? @map("register_time") + avatarUrl String? @map("avatar_url") @db.VarChar(512) + buyerTags String? @map("buyer_tags") @db.VarChar(255) + buyerFeatures String? @map("buyer_features") @db.VarChar(255) + bProductViews Int? @map("behavior_product_views") + bValidInquiries Int? @map("behavior_valid_inquiries") + bValidRfq Int? @map("behavior_valid_rfq") + bLoginDays Int? @map("behavior_login_days") + bSpamInquiries Int? @map("behavior_spam_inquiries") + bTopIndustry String? @map("behavior_top_industry") @db.VarChar(255) + inquiries Inquiry[] + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + @@map("customers") +} + +model Inquiry { + id BigInt @id @default(autoincrement()) + inquiryId String @unique @map("inquiry_id") @db.VarChar(32) + customerId String? @index @map("customer_id") @db.VarChar(64) + manager String? @index @db.VarChar(128) + createTime DateTime @map("create_time") + updateTime DateTime @map("update_time") + status String? @db.VarChar(64) + msgPreview String? @map("msg_preview") @db.VarChar(512) + isFlagged Boolean @default(false) @map("is_flagged") + messagesCount Int @default(0) @map("messages_count") + lastSendTime DateTime? @map("last_send_time") + hasFailedBatches Boolean @default(false) @map("has_failed_batches") + customer Customer? @relation(fields: [customerId], references: [customerId]) + messages ChatMessage[] + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + @@map("inquiries") +} + +model ChatMessage { + id BigInt @id @default(autoincrement()) + messageId String @unique @map("message_id") @db.VarChar(64) + inquiryId String @index @map("inquiry_id") @db.VarChar(32) + sender String? @db.VarChar(128) + direction Direction + sendTime DateTime @map("send_time") + content String @db.MediumText + translation String? @db.Text + inquiry Inquiry @relation(fields: [inquiryId], references: [inquiryId]) + createdAt DateTime @default(now()) @map("created_at") + @@index([inquiryId, sendTime, messageId]) + @@map("chat_messages") +} + +model User { + id BigInt @id @default(autoincrement()) + username String @unique + passwordHash String @map("password_hash") + createdAt DateTime @default(now()) @map("created_at") + @@map("users") +} +``` + +- [ ] **Step 3: 迁移 + 生成 client** +```bash +npx prisma migrate dev --name init +npx prisma generate +``` + +- [ ] **Step 4: 验证表结构**(Git Bash 用 `-i` 不用 `-it` —— 修正 I9) +```bash +docker exec -i inquiry-mysql mysql -uroot -proot -e "USE inquiry; SHOW TABLES; DESC chat_messages;" +``` +Expected:4 表;`chat_messages.content` 为 `MEDIUMTEXT`;`message_id` UNIQUE;索引含 `inquiry_id_send_time_message_id_idx`。 + +- [ ] **Step 5: Commit** +```bash +git add -A && git commit -m "feat(db): prisma schema (4 tables, has_failed_batches, system enum, composite index)" +``` + +--- + +## Task 3: 惰性 env + 惰性 Prisma(修正 F4) + +**Files:** `src/config/env.ts`, `src/lib/prisma.ts` + +- [ ] **Step 1: `src/config/env.ts`**(函数化 `loadEnv`,延迟校验,让测试能先覆写 `process.env`) +```ts +import { z } from 'zod'; + +const schema = z.object({ + DATABASE_URL: z.string().regex(/^mysql:\/\//, 'must start with mysql://'), + INGEST_TOKEN: z.string().min(8), + JWT_SECRET: z.string().min(8), + JWT_EXPIRES_IN: z.string().default('7d'), + ADMIN_USERNAME: z.string(), + ADMIN_PASSWORD: z.string().min(8), + PORT: z.coerce.number().default(8080), + NODE_ENV: z.enum(['development', 'production']).default('development'), +}); + +let _env: z.infer | null = null; +export function loadEnv() { + if (_env) return _env; + // 首次调用时加载 .env(server 用);测试会在调用前手动设 process.env + if (!process.env.DATABASE_URL) { + const dotenv = await import('dotenv'); await dotenv.config(); + } + ... +} +``` +> 注:`loadEnv` 不能用顶层 `import 'dotenv/config'`(会立即固化),改用动态 import + 顶层 await 函数。简化写法: +```ts +import { z } from 'zod'; +import { config as dotenvConfig } from 'dotenv'; +dotenvConfig(); // 仅设置 process.env,不校验 +const schema = z.object({ /* 同上,无 JWT_REFRESH_SECRET */ }); +let _env: z.infer | null = null; +export function loadEnv() { + if (_env) return _env; + const parsed = schema.safeParse(process.env); + if (!parsed.success) { + console.error('❌ invalid env:', parsed.error.flatten().fieldErrors); + process.exit(1); + } + _env = parsed.data; + return _env; +} +export const env = new Proxy({} as z.infer, { get: (_, k) => (loadEnv() as any)[k] }); +``` +> `env` 用 Proxy 延迟:模块加载时不校验,首次访问才 `loadEnv()`。测试在 import app 前覆写 `process.env`,Proxy 自然取到测试值。 + +- [ ] **Step 2: `src/lib/prisma.ts`**(惰性单例,修正 F4) +```ts +import { PrismaClient } from '@prisma/client'; +let _prisma: PrismaClient | null = null; +export function getPrisma(): PrismaClient { + if (!_prisma) _prisma = new PrismaClient(); + return _prisma; +} +export async function disconnectPrisma() { if (_prisma) { await _prisma.$disconnect(); _prisma = null; } } +``` +> 所有 service 用 `getPrisma()`,**不在模块顶层实例化**。测试在 setupApp 里先覆写 `process.env.DATABASE_URL` 再首次调 `getPrisma()`。 + +- [ ] **Step 3: Commit** +```bash +git add -A && git commit -m "feat(config): lazy env proxy and prisma singleton (testcontainer-friendly)" +``` + +--- + +## Task 4: Express app + 健康检查 + 错误中间件 + SIGTERM + +**Files:** `src/middleware/error.ts`, `src/app.ts`, `src/server.ts` + +- [ ] **Step 1: `src/middleware/error.ts`** +```ts +import type { ErrorRequestHandler } from 'express'; +export const errorHandler: ErrorRequestHandler = (err, _req, res, _next) => { + console.error(err); + res.status(err.status ?? 500).json({ error: err.message ?? 'internal error' }); +}; +``` + +- [ ] **Step 2: `src/app.ts`** +```ts +import express from 'express'; +import cors from 'cors'; +import morgan from 'morgan'; +import { errorHandler } from './middleware/error.js'; + +export function createApp() { + const app = express(); + // MVP:CORS 全开(ingest 靠 token 鉴权;后台 Plan 3 联调时收窄到前端域名) + app.use(cors({ origin: '*' })); + app.use(express.json({ limit: '50mb' })); + if (process.env.NODE_ENV !== 'test') app.use(morgan('dev')); + app.get('/api/health', (_req, res) => res.json({ ok: true })); + app.use(errorHandler); + return app; +} +``` + +- [ ] **Step 3: `tests/health.test.ts`**(纯单元,不起容器) +```ts +import { describe, it, expect } from 'vitest'; +import request from 'supertest'; +import { createApp } from '../src/app.js'; +describe('health', () => { + it('ok', async () => { + const res = await request(createApp()).get('/api/health'); + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true }); + }); +}); +``` + +- [ ] **Step 4: 跑测试通过** +```bash +npm test -- health # 应 1 passed +``` + +- [ ] **Step 5: `src/server.ts`**(含 SIGTERM 优雅关停 —— S9) +```ts +import { createApp } from './app.js'; +import { loadEnv } from './config/env.js'; +import { bootstrapAdmin } from './services/auth.service.js'; +import { disconnectPrisma } from './lib/prisma.js'; + +async function main() { + loadEnv(); + const { env } = await import('./config/env.js'); + await bootstrapAdmin(); + const app = createApp(); + const server = app.listen(env.PORT, () => console.log(`listening on ${env.PORT}`)); + process.on('SIGTERM', async () => { server.close(); await disconnectPrisma(); process.exit(0); }); +} +main(); +``` + +- [ ] **Step 6: Commit** `feat(app): express app, health, error handler, SIGTERM` + +--- + +## Task 5: 管理员 bootstrap + JWT 登录 + 统一测试脚手架 + +**Files:** `src/services/auth.service.ts`, `src/routes/auth.ts`, `tests/setup.ts`, `tests/auth.test.ts`;Modify `src/app.ts` + +- [ ] **Step 1: `tests/setup.ts`**(统一脚手架 —— 修正 I8/F4。所有集成测试都用 `setupApp()`) +```ts +import { MySqlContainer } from '@testcontainers/mysql'; +import type { StartedTestContainer } from 'testcontainers'; +import { execSync } from 'node:child_process'; +import { getPrisma, disconnectPrisma } from '../src/lib/prisma.js'; +import { createApp } from '../src/app.js'; +// 注:MySqlContainer 默认 user=root password=test + +let container: StartedTestContainer | null = null; + +export async function setupApp() { + if (!container) { + container = await new MySqlContainer('mysql:8').withDatabase('t').start(); + process.env.DATABASE_URL = `mysql://root:test@${container.getHost()}:${container.getMappedPort(3306)}/t`; + process.env.INGEST_TOKEN = 'dev-ingest-token-1234'; + process.env.JWT_SECRET = 'dev-jwt-secret-1234'; + process.env.ADMIN_USERNAME = 'admin'; + process.env.ADMIN_PASSWORD = 'admin-password-1234'; + execSync('npx prisma migrate deploy', { stdio: 'inherit', env: process.env }); + } + const prisma = getPrisma(); + const app = createApp(); + return { app, prisma }; +} + +export async function resetDb() { + const prisma = getPrisma(); + await prisma.$executeRawUnsafe('SET FOREIGN_KEY_CHECKS=0'); + for (const t of ['chat_messages', 'inquiries', 'customers', 'users']) { + await prisma.$executeRawUnsafe(`DELETE FROM ${t}`); + } + await prisma.$executeRawUnsafe('SET FOREIGN_KEY_CHECKS=1'); +} + +export async function teardown() { await disconnectPrisma(); await container?.stop(); container = null; } +``` +> 关键:先起容器 → 覆写 `process.env.DATABASE_URL` → 才首次 `getPrisma()`(惰性,取到容器地址)。`vitest singleFork` 保证一个进程一个容器。 + +- [ ] **Step 2: `tests/auth.test.ts`** +```ts +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import request from 'supertest'; +import { setupApp, resetDb, teardown } from './setup.js'; + +let app: any; +beforeAll(async () => { ({ app } = await setupApp()); }, 90000); +afterAll(async () => { await teardown(); }); + +describe('auth', () => { + it('login with admin env creds returns token', async () => { + const res = await request(app).post('/api/auth/login').send({ username: 'admin', password: 'admin-password-1234' }); + expect(res.status).toBe(200); + expect(res.body.token).toMatch(/^[A-Za-z0-9._-]+$/); + }); + it('wrong password 401', async () => { + const res = await request(app).post('/api/auth/login').send({ username: 'admin', password: 'wrong' }); + expect(res.status).toBe(401); + }); +}); +``` + +- [ ] **Step 3: `src/services/auth.service.ts`** +```ts +import bcrypt from 'bcryptjs'; +import jwt from 'jsonwebtoken'; +import { getPrisma } from '../lib/prisma.js'; +import { env } from '../config/env.js'; + +export async function bootstrapAdmin() { + const prisma = getPrisma(); + if (await prisma.user.count() === 0) { + await prisma.user.create({ data: { username: env.ADMIN_USERNAME, passwordHash: await bcrypt.hash(env.ADMIN_PASSWORD, 10) } }); + console.log(`seeded admin "${env.ADMIN_USERNAME}"`); + } +} + +export async function login(username: string, password: string) { + const prisma = getPrisma(); + const user = await prisma.user.findUnique({ where: { username } }); + if (!user || !(await bcrypt.compare(password, user.passwordHash))) + throw Object.assign(new Error('invalid credentials'), { status: 401 }); + return { token: jwt.sign({ sub: user.id, u: user.username }, env.JWT_SECRET, { expiresIn: env.JWT_EXPIRES_IN }), + expiresIn: env.JWT_EXPIRES_IN }; +} +``` +> 注:测试里 `bootstrapAdmin` 由 `/api/auth/login` 首次调用时触发?不——改为在 `createApp` 或 setupApp 末尾调一次。**实现时在 `setupApp()` 末尾加 `await bootstrapAdmin();`**(或在 server.ts 启动调;测试环境 setupApp 调)。 + +- [ ] **Step 4: `src/routes/auth.ts`** +```ts +import { Router } from 'express'; +import jwt from 'jsonwebtoken'; +import { login } from '../services/auth.service.js'; +import { env } from '../config/env.js'; +import { jwtAuth } from '../middleware/jwtAuth.js'; + +export const authRouter = Router(); +authRouter.post('/login', async (req, res, next) => { + try { res.json(await login(req.body.username, req.body.password)); } catch (e) { next(e); } +}); +// refresh 用同一个 JWT_SECRET(MVP 单密钥) +authRouter.post('/refresh', jwtAuth, (req, res) => { + const u = (req as any).user; + res.json({ token: jwt.sign({ sub: u.sub, u: u.u }, env.JWT_SECRET, { expiresIn: env.JWT_EXPIRES_IN }), expiresIn: env.JWT_EXPIRES_IN }); +}); +``` + +- [ ] **Step 5: `src/middleware/jwtAuth.ts`** +```ts +import type { RequestHandler } from 'express'; +import jwt from 'jsonwebtoken'; +import { env } from '../config/env.js'; +export const jwtAuth: RequestHandler = (req, res, next) => { + const h = req.headers.authorization || ''; + const token = h.startsWith('Bearer ') ? h.slice(7) : ''; + try { (req as any).user = jwt.verify(token, env.JWT_SECRET); next(); } + catch { res.status(401).json({ error: 'unauthorized' }); } +}; +``` + +- [ ] **Step 6: `src/app.ts` 挂载** +```ts +import { authRouter } from './routes/auth.js'; +// health 之后、errorHandler 之前: +app.use('/api/auth', authRouter); +``` + +- [ ] **Step 7: 测试通过 + Commit** `feat(auth): admin bootstrap, login, refresh, jwt middleware` + +--- + +## Task 6: /api/ingest — 鉴权 + 限流 + payload zod 校验 + +**Files:** `src/middleware/{ingestAuth,rateLimit}.ts`, `src/routes/ingest.ts`;Modify `src/app.ts` + +- [ ] **Step 1: `src/middleware/ingestAuth.ts`**(timingSafeEqual —— S1) +```ts +import type { RequestHandler } from 'express'; +import { timingSafeEqual } from 'node:crypto'; +import { env } from '../config/env.js'; +export const ingestAuth: RequestHandler = (req, res, next) => { + const h = req.headers.authorization || ''; + const token = h.startsWith('Bearer ') ? h.slice(7) : ''; + const want = env.INGEST_TOKEN; + const ok = token.length === want.length && timingSafeEqual(Buffer.from(token), Buffer.from(want)); + if (!ok) return res.status(401).json({ error: 'invalid ingest token' }); + next(); +}; +``` + +- [ ] **Step 2: `src/middleware/rateLimit.ts`**(MVP IP 维度 —— S3) +```ts +import rateLimit from 'express-rate-limit'; +// 注:spec 写 60/min/token,MVP 用 IP 维度(单操作者够用);多操作者后切 key +export const ingestLimiter = rateLimit({ windowMs: 60_000, max: 60, standardHeaders: true, legacyHeaders: false }); +``` + +- [ ] **Step 3: `tests/fixtures.ts`** +```ts +export function makePayload(o: any = {}) { + return { + inquiry: { inquiryId: '13776617920', manager: 'cara Hu', createTime: '2026-03-13T10:11:50Z', + updateTime: '2026-07-13T10:11:50Z', status: '洽谈中', msgPreview: 'hi', isFlagged: false, ...o.inquiry }, + customer: { customerId: 'cid-1', name: 'Carlos', country: 'PE', email: null, company: 'World Shop', + registerTime: '2025-06-23T00:00:00Z', avatarUrl: '', buyerTags: 'TM沟通', buyerFeatures: '', + behavior: { productViews: null, validInquiries: null, validRfq: 39 }, ...o.customer }, + messages: [ + { messageId: 'm1', sendTime: '2026-07-13T02:11:50Z', sender: 'cara Hu', direction: 'outgoing', content: 'hello', translation: '' }, + { messageId: 'm2', sendTime: '2026-07-13T02:12:00Z', sender: 'Carlos', direction: 'incoming', content: 'hi', translation: '' }, + ], + ...o, + }; +} +``` + +- [ ] **Step 4: `src/routes/ingest.ts`**(zod schema + 路由;`messages.min(1)` 防 S4 空数组边界) +```ts +import { Router } from 'express'; +import { z } from 'zod'; +import { ingestAuth } from '../middleware/ingestAuth.js'; +import { ingestLimiter } from '../middleware/rateLimit.js'; +import { ingest } from '../services/ingest.service.js'; + +const MessageSchema = z.object({ + messageId: z.string().min(1), + sendTime: z.string().datetime(), + sender: z.string().optional(), + direction: z.enum(['incoming', 'outgoing', 'system']), + content: z.string(), + translation: z.string().optional().default(''), +}); +const BehaviorSchema = z.object({ + productViews: z.number().nullable().optional(), validInquiries: z.number().nullable().optional(), + validRfq: z.number().nullable().optional(), loginDays: z.number().nullable().optional(), + spamInquiries: z.number().nullable().optional(), topIndustry: z.string().nullable().optional(), +}).optional(); +const PayloadSchema = z.object({ + inquiry: z.object({ + inquiryId: z.string().min(1), manager: z.string().optional(), + createTime: z.string().datetime(), updateTime: z.string().datetime(), + status: z.string().optional(), msgPreview: z.string().optional(), isFlagged: z.boolean().optional().default(false), + }), + customer: z.object({ + customerId: z.string().min(1), name: z.string().optional(), country: z.string().nullable().optional(), + email: z.string().nullable().optional(), company: z.string().nullable().optional(), + registerTime: z.string().datetime().nullable().optional(), avatarUrl: z.string().optional(), + buyerTags: z.string().optional(), buyerFeatures: z.string().optional(), behavior: BehaviorSchema, + }).nullable(), + messages: z.array(MessageSchema).min(1), +}); + +export const ingestRouter = Router(); +ingestRouter.post('/', ingestLimiter, ingestAuth, async (req, res, next) => { + const parsed = PayloadSchema.safeParse(req.body); + if (!parsed.success) return res.status(400).json({ error: 'invalid payload', issues: parsed.error.flatten() }); + try { res.json(await ingest(parsed.data)); } catch (e) { next(e); } +}); +``` + +- [ ] **Step 5: 占位 `src/services/ingest.service.ts`** = `export async function ingest(p:any){ return {ok:true,inserted:0,failedBatches:0,earlyReturn:false}; }` + +- [ ] **Step 6: `src/app.ts` 挂 `app.use('/api/ingest', ingestRouter)`** + +- [ ] **Step 7: 写 `tests/ingest.test.ts` 的鉴权/校验部分(用 setupApp 模板)** +```ts +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import request from 'supertest'; +import { setupApp, resetDb, teardown } from './setup.js'; +import { makePayload } from './fixtures.js'; +let app: any; +beforeAll(async () => { ({ app } = await setupApp()); }, 90000); +afterAll(teardown); + +describe('ingest auth/validation', () => { + it('401 without token', async () => { + expect((await request(app).post('/api/ingest').send(makePayload())).status).toBe(401); + }); + it('400 bad payload', async () => { + expect((await request(app).post('/api/ingest').set('Authorization', 'Bearer dev-ingest-token-1234').send({ inquiry: {} })).status).toBe(400); + }); +}); +``` + +- [ ] **Step 8: 测试通过 + Commit** `feat(ingest): auth, rate-limit, payload validation` + +--- + +## Task 7: ingest 早返回 + customer/inquiry upsert(customerUpdate 不覆盖 null) + +**Files:** `src/services/ingest.service.ts`, `src/utils/time.ts` + +- [ ] **Step 1: `src/utils/time.ts`** +```ts +export const maxSendTime = (msgs: { sendTime: string }[]) => + msgs.length ? new Date(Math.max(...msgs.map(m => Date.parse(m.sendTime)))) : null; +``` + +- [ ] **Step 2: `src/services/ingest.service.ts`**(早返回 + 主表 upsert;customerUpdate 用 `Prisma.skip` 不覆盖 null —— 修正 I6) +```ts +import { Prisma } from '@prisma/client'; +import { getPrisma } from '../lib/prisma.js'; +import { batchInsertMessages } from '../utils/batchInsert.js'; +import { maxSendTime } from '../utils/time.js'; + +type Payload = { + inquiry: any; customer: any | null; + messages: { messageId: string; sendTime: string; sender?: string; direction: string; content: string; translation?: string }[]; +}; + +export async function ingest(p: Payload) { + const prisma = getPrisma(); + const { inquiry: iq, customer: cu, messages } = p; + const newCount = messages.length; + const newLast = maxSendTime(messages); + + // 早返回:存在 + 无失败批次 + count + lastSendTime 均一致 + const existing = await prisma.inquiry.findUnique({ where: { inquiryId: iq.inquiryId } }); + if (existing && !existing.hasFailedBatches + && existing.messagesCount === newCount + && newLast && existing.lastSendTime + && existing.lastSendTime.getTime() === newLast.getTime()) { + return { ok: true, inserted: existing.messagesCount, failedBatches: 0, earlyReturn: true }; + } + + // 主表事务 + await prisma.$transaction(async (tx) => { + if (cu) { + const data = mapCustomer(cu); + const update = skipNulls(data); // 不用 null 覆盖已落库值 —— 修正 I6 + await tx.customer.upsert({ + where: { customerId: cu.customerId }, + create: data, + update, + }); + } + await tx.inquiry.upsert({ + where: { inquiryId: iq.inquiryId }, + create: { inquiryId: iq.inquiryId, customerId: cu?.customerId ?? null, manager: iq.manager, + createTime: new Date(iq.createTime), updateTime: new Date(iq.updateTime), + status: iq.status, msgPreview: iq.msgPreview, isFlagged: iq.isFlagged }, + update: { customerId: cu?.customerId ?? null, manager: iq.manager, + createTime: new Date(iq.createTime), updateTime: new Date(iq.updateTime), + status: iq.status, msgPreview: iq.msgPreview, isFlagged: iq.isFlagged }, + }); + }); + + // 批量消息(Task 8) + const { failedBatches } = await batchInsertMessages(iq.inquiryId, messages); + + // 用"总量"(批量后该询盘的消息总数),不是 createMany 的本次 delta —— 修正 F2 + const total = await prisma.chatMessage.count({ where: { inquiryId: iq.inquiryId } }); + await prisma.inquiry.update({ + where: { inquiryId: iq.inquiryId }, + data: { messagesCount: total, lastSendTime: newLast, hasFailedBatches: failedBatches > 0 }, + }); + + return { ok: true, inserted: total, failedBatches, earlyReturn: false }; +} + +function mapCustomer(c: any) { + return { customerId: c.customerId, name: c.name, country: c.country, email: c.email, company: c.company, + registerTime: c.registerTime ? new Date(c.registerTime) : null, avatarUrl: c.avatarUrl, + buyerTags: c.buyerTags, buyerFeatures: c.buyerFeatures, + bProductViews: c.behavior?.productViews ?? null, bValidInquiries: c.behavior?.validInquiries ?? null, + bValidRfq: c.behavior?.validRfq ?? null, bLoginDays: c.behavior?.loginDays ?? null, + bSpamInquiries: c.behavior?.spamInquiries ?? null, bTopIndustry: c.behavior?.topIndustry ?? null }; +} +// update 时跳过值为 null 的字段(避免覆盖已落库的有效值) +function skipNulls(data: Record) { + const out: Record = {}; + for (const [k, v] of Object.entries(data)) if (v != null || k === 'customerId') out[k] = v; + return out; +} +``` + +- [ ] **Step 3: Commit** `feat(ingest): early-return + upsert (customerUpdate skips nulls)` + +--- + +## Task 8: 批量消息写入(单一 createMany 实现 + 增量回归测试) + +**Files:** `src/utils/batchInsert.ts`;`tests/ingest.test.ts` 加增量测试 + +- [ ] **Step 1: `src/utils/batchInsert.ts`**(只留 createMany skipDuplicates,删掉 raw SQL 双实现 —— 修正 I1) +```ts +import { getPrisma } from '../lib/prisma.js'; +import type { Direction } from '@prisma/client'; + +const BATCH = 500; + +export async function batchInsertMessages(inquiryId: string, messages: any[]) { + const prisma = getPrisma(); + let failedBatches = 0; + for (let i = 0; i < messages.length; i += BATCH) { + const batch = messages.slice(i, i + BATCH); + try { + // createMany + skipDuplicates:MySQL 下生成 INSERT IGNORE,重复 messageId 跳过。 + // MVP 接受"重推时已存在的消息不更新 content"(spec R6 背书);需要更新 content 再换 raw ON DUPLICATE KEY。 + await prisma.chatMessage.createMany({ + skipDuplicates: true, + data: batch.map(m => ({ + messageId: m.messageId, inquiryId, + sender: m.sender ?? null, + direction: m.direction as Direction, + sendTime: new Date(m.sendTime), + content: m.content, + translation: m.translation ?? '', + })), + }); + } catch (e) { + console.error(`batch at offset ${i} failed:`, e); + failedBatches++; + } + } + return { failedBatches }; +} +``` + +- [ ] **Step 2: `tests/ingest.test.ts` 加幂等 + 增量回归测试(修正 F3 —— 这个测试会逼出 F2 修复)** +```ts +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import request from 'supertest'; +import { setupApp, resetDb, teardown } from './setup.js'; +import { makePayload } from './fixtures.js'; +let app: any; const TOKEN = 'Bearer dev-ingest-token-1234'; +beforeAll(async () => { ({ app } = await setupApp()); }, 90000); +afterAll(teardown); + +describe('ingest idempotency + increment', () => { + it('push N → push N again early-returns', async () => { + await resetDb(); + const p = makePayload(); + const r1 = await request(app).post('/api/ingest').set('Authorization', TOKEN).send(p); + expect(r1.status).toBe(200); + expect(r1.body.inserted).toBe(2); + expect(r1.body.earlyReturn).toBe(false); + const r2 = await request(app).post('/api/ingest').set('Authorization', TOKEN).send(p); + expect(r2.body.earlyReturn).toBe(true); // 同样 payload → 早返回 + }); + + it('push N → push N+1 → push N+1 again early-returns (regression for messagesCount=total)', async () => { + await resetDb(); + const base = makePayload(); + // 推 2 条 + await request(app).post('/api/ingest').set('Authorization', TOKEN).send(base); + // 推 3 条(新增 m3) + const withOneMore = makePayload({ messages: [ + ...base.messages, + { messageId: 'm3', sendTime: '2026-07-13T02:13:00Z', sender: 'x', direction: 'incoming', content: 'c3', translation: '' }, + ]}); + const r3 = await request(app).post('/api/ingest').set('Authorization', TOKEN).send(withOneMore); + expect(r3.body.inserted).toBe(3); // 总量 3,不是 delta 1 + expect(r3.body.earlyReturn).toBe(false); + // 再推同样 3 条 → 早返回(若 messagesCount 退化成 delta,这里会失败) + const r4 = await request(app).post('/api/ingest').set('Authorization', TOKEN).send(withOneMore); + expect(r4.body.earlyReturn).toBe(true); + expect(r4.body.inserted).toBe(3); + }); +}); +``` + +- [ ] **Step 3: 跑测试** +```bash +npm test -- ingest +``` +Expected:增量回归测试通过(证明 messagesCount 用 total)。 + +- [ ] **Step 4: Commit** `feat(ingest): batched createMany skipDuplicates + idempotency/increment regression tests` + +--- + +## Task 9: 询盘列表(筛选/搜索/排序/分页)+ buildInquiryWhere 复用 + +**Files:** `src/services/inquiry.service.ts`, `src/routes/inquiries.ts`;Modify `src/app.ts` + +- [ ] **Step 1: `src/services/inquiry.service.ts`**(`buildInquiryWhere` 导出复用 —— S6;显式 equals —— I10) +```ts +import { Prisma } from '@prisma/client'; +import { getPrisma } from '../lib/prisma.js'; + +export function buildInquiryWhere(q: any): Prisma.InquiryWhereInput { + const where: Prisma.InquiryWhereInput = {}; + if (q.country) where.customer = { is: { country: { equals: q.country } } }; + if (q.manager) where.manager = { equals: q.manager }; + if (q.status) where.status = { equals: q.status }; + if (q.dateFrom || q.dateTo) { + where.createTime = {}; + if (q.dateFrom) where.createTime.gte = new Date(q.dateFrom); + if (q.dateTo) where.createTime.lte = new Date(q.dateTo); + } + if (q.q) where.OR = [ + { inquiryId: { contains: q.q } }, + { msgPreview: { contains: q.q } }, + { customer: { is: { name: { contains: q.q } } } }, + { customer: { is: { company: { contains: q.q } } } }, + ]; + return where; +} + +export async function listInquiries(q: any) { + const prisma = getPrisma(); + const size = Math.min(Number(q.size) || 20, 100); + const page = Number(q.page) || 1; + const where = buildInquiryWhere(q); + const orderBy = q.sort === 'update_time_desc' ? { updateTime: 'desc' as const } + : q.sort === 'create_time_asc' ? { createTime: 'asc' as const } + : { createTime: 'desc' as const }; + const [items, total] = await Promise.all([ + prisma.inquiry.findMany({ where, orderBy, skip: (page - 1) * size, take: size, include: { customer: true } }), + prisma.inquiry.count({ where }), + ]); + return { items, total, page, size }; +} +``` + +- [ ] **Step 2: `src/routes/inquiries.ts`**(骨架,本 Task 只挂 list) +```ts +import { Router } from 'express'; +import { jwtAuth } from '../middleware/jwtAuth.js'; +import { listInquiries, getInquiryDetail, listMessages } from '../services/inquiry.service.js'; +export const inquiriesRouter = Router(); +inquiriesRouter.get('/', jwtAuth, async (req, res, next) => { + try { res.json(await listInquiries(req.query)); } catch (e) { next(e); } +}); +``` + +- [ ] **Step 3: `src/app.ts` 挂 `app.use('/api/inquiries', inquiriesRouter)`** + +- [ ] **Step 4: 测试 `tests/inquiries.test.ts`(list 筛选;seedInquiries fixture 自行构造或复用 makePayload 推几条)** +```ts +it('lists filtered by country', async () => { + await resetDb(); + await pushInquiry(app, makePayload({ customer: { country: 'PE' } })); // helper: POST /api/ingest + await pushInquiry(app, makePayload({ inquiry: { inquiryId: 'X2' }, customer: { country: 'US' } })); + const token = (await request(app).post('/api/auth/login').send({ username: 'admin', password: 'admin-password-1234' })).body.token; + const res = await request(app).get('/api/inquiries?country=PE').set('Authorization', 'Bearer ' + token); + expect(res.status).toBe(200); + expect(res.body.items.length).toBe(1); + expect(res.body.items[0].customer.country).toBe('PE'); +}); +``` + +- [ ] **Step 5: 测试通过 + Commit** `feat(api): inquiry list with filters/search/pagination, shared buildInquiryWhere` + +--- + +## Task 10: 询盘详情 API + +**Files:** Modify `src/services/inquiry.service.ts`, `src/routes/inquiries.ts` + +- [ ] **Step 1: 服务方法** +```ts +export async function getInquiryDetail(inquiryId: string) { + return getPrisma().inquiry.findUnique({ + where: { inquiryId }, include: { customer: true, _count: { select: { messages: true } } }, + }); +} +``` +- [ ] **Step 2: 路由 `inquiriesRouter.get('/:id', jwtAuth, ...)`** → `res.json(await getInquiryDetail(req.params.id))` +- [ ] **Step 3: 测试(推一条 → GET /:id → 200 含 customer + _count.messages) + Commit** `feat(api): inquiry detail` + +--- + +## Task 11: 聊天游标分页(复合游标 + zod 校验) + +**Files:** Modify `src/services/inquiry.service.ts`, `src/routes/inquiries.ts` + +- [ ] **Step 1: 服务方法**(`(sendTime, messageId)` 复合游标) +```ts +export async function listMessages(inquiryId: string, q: { afterSendTime?: string; afterMessageId?: string; size?: string }) { + const prisma = getPrisma(); + const size = Math.min(Number(q.size) || 50, 200); + const where: Prisma.ChatMessageWhereInput = { inquiryId }; + if (q.afterSendTime && q.afterMessageId) { + const after = new Date(q.afterSendTime); + where.OR = [{ sendTime: { gt: after } }, { sendTime: after, messageId: { gt: q.afterMessageId } }]; + } + const items = await prisma.chatMessage.findMany({ + where, orderBy: [{ sendTime: 'asc' }, { messageId: 'asc' }], take: size, + }); + const last = items[items.length - 1]; + return { items, nextCursor: items.length === size && last + ? { afterSendTime: last.sendTime.toISOString(), afterMessageId: last.messageId } : null }; +} +``` +- [ ] **Step 2: 路由 + zod query 校验(半残游标 400 —— 修正 I3)** +```ts +const CursorQuery = z.object({ + afterSendTime: z.string().datetime().optional(), + afterMessageId: z.string().optional(), + size: z.string().optional(), +}).refine(d => (!!d.afterSendTime && !!d.afterMessageId) || (!d.afterSendTime && !d.afterMessageId), + { message: 'cursor fields must be both present or both absent' }); + +inquiriesRouter.get('/:id/messages', jwtAuth, async (req, res, next) => { + const parsed = CursorQuery.safeParse(req.query); + if (!parsed.success) return res.status(400).json({ error: parsed.error.flatten() }); + try { res.json(await listMessages(req.params.id, parsed.data)); } catch (e) { next(e); } +}); +``` +- [ ] **Step 3: 测试(插 50 条 → 首页 size=20 → 带 nextCursor 取下页;另测只传 afterSendTime → 400) + Commit** `feat(api): chat cursor pagination (composite key, zod-validated)` + +--- + +## Task 12: 统计 API + +**Files:** `src/services/stats.service.ts`, `src/routes/stats.ts`;Modify `src/app.ts` + +- [ ] **Step 1: 服务**(`_count` 重命名为 count —— S5) +```ts +import { getPrisma } from '../lib/prisma.js'; +export async function overview() { + const prisma = getPrisma(); + const total = await prisma.inquiry.count(); + const countryTop10 = (await prisma.customer.groupBy({ + by: ['country'], _count: { id: true }, orderBy: { _count: { id: 'desc' } }, take: 10, where: { country: { not: null } }, + })).map(r => ({ country: r.country, count: r._count.id })); + const byManager = (await prisma.inquiry.groupBy({ + by: ['manager'], _count: { id: true }, orderBy: { _count: { id: 'desc' } }, where: { manager: { not: null } }, + })).map(r => ({ manager: r.manager, count: r._count.id })); + const monthly = await prisma.$queryRaw<{ ym: string; c: bigint }[]>` + SELECT DATE_FORMAT(create_time, '%Y-%m') ym, COUNT(*) c FROM inquiries GROUP BY ym ORDER BY ym DESC LIMIT 12`; + return { total, countryTop10, byManager, monthly: monthly.map(r => ({ ym: r.ym, count: Number(r.c) })) }; +} +``` +- [ ] **Step 2: 路由 `/api/stats/overview`(jwtAuth)+ 挂载** +- [ ] **Step 3: 测试 + Commit** `feat(api): stats overview` + +--- + +## Task 13: 导出(列表 CSV + 单询盘聊天 CSV) + +**Files:** `src/routes/export.ts`, Modify `src/routes/inquiries.ts`(单询盘导出);`src/app.ts` + +- [ ] **Step 1: `src/routes/export.ts`**(列表 CSV stream,复用 buildInquiryWhere —— S6) +```ts +import { Router } from 'express'; +import { getPrisma } from '../lib/prisma.js'; +import { jwtAuth } from '../middleware/jwtAuth.js'; +import { buildInquiryWhere } from '../services/inquiry.service.js'; +export const exportRouter = Router(); +exportRouter.get('/', jwtAuth, async (req, res, next) => { + try { + res.setHeader('Content-Type', 'text/csv; charset=utf-8'); + res.setHeader('Content-Disposition', 'attachment; filename="inquiries.csv"'); + res.write('询价单号,业务员,国家,买家,公司,状态,创建时间,消息数\n'); + const rows = await getPrisma().inquiry.findMany({ + where: buildInquiryWhere(req.query), include: { customer: true }, orderBy: { createTime: 'desc' } }); + for (const r of rows) { + res.write([r.inquiryId, r.manager, r.customer?.country, r.customer?.name, r.customer?.company, + r.status, r.createTime.toISOString(), r.messagesCount].map(csvEscape).join(',') + '\n'); + } + res.end(); + } catch (e) { next(e); } +}); +function csvEscape(v: any) { const s = v == null ? '' : String(v); return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s; } +``` +- [ ] **Step 2: 单询盘聊天导出**(挂到 inquiriesRouter —— 修正 I5) +```ts +// src/routes/inquiries.ts +inquiriesRouter.get('/:id/messages/export', jwtAuth, async (req, res, next) => { + try { + res.setHeader('Content-Type', 'text/csv; charset=utf-8'); + res.setHeader('Content-Disposition', `attachment; filename="chat_${req.params.id}.csv"`); + res.write('时间,方向,发送者,原文,翻译\n'); + const msgs = await getPrisma().chatMessage.findMany({ + where: { inquiryId: req.params.id }, orderBy: [{ sendTime: 'asc' }, { messageId: 'asc' }] }); + for (const m of msgs) { + res.write([m.sendTime.toISOString(), m.direction, m.sender, m.content, m.translation ?? ''].map(csvEscape).join(',') + '\n'); + } + res.end(); + } catch (e) { next(e); } +}); +``` +- [ ] **Step 3: 挂 `/api/export`(exportRouter)+ 测试(列表导出 + 单询盘导出,验证 CSV 含表头且不含/含聊天) + Commit** `feat(api): csv export (list + single-inquiry chat)` + +--- + +## Task 14: Docker 开发环境 + 端到端 e2e + +**Files:** `Dockerfile`, `docker-compose.yml`, `scripts/e2e.sh` + +- [ ] **Step 1: `Dockerfile`**(多阶段;`prisma generate` 在 build 阶段;运行时 `migrate deploy`) +```dockerfile +FROM node:18-bookworm-slim AS build +WORKDIR /app +COPY package*.json ./ && RUN npm ci +COPY . . && RUN npx prisma generate && npm run build +FROM node:18-bookworm-slim +WORKDIR /app +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +COPY --from=build /app/prisma ./prisma +COPY --from=build /app/package.json . +EXPOSE 8080 +CMD ["sh","-c","npx prisma migrate deploy && node dist/server.js"] +``` + +- [ ] **Step 2: `docker-compose.yml`**(mysql `--max_allowed_packet=128M` —— spec v3) +```yaml +services: + mysql: + image: mysql:8 + environment: { MYSQL_ROOT_PASSWORD: root, MYSQL_DATABASE: inquiry } + command: --max_allowed_packet=128M --character-set-server=utf8mb4 + volumes: [mysql_data:/var/lib/mysql] + ports: ["3306:3306"] + healthcheck: { test: ["CMD","mysqladmin","ping","-h","localhost","-uroot","-proot"], interval: 5s, retries: 20 } + backend: + build: . + environment: + DATABASE_URL: mysql://root:root@mysql:3306/inquiry + INGEST_TOKEN: dev-ingest-token-1234 + JWT_SECRET: dev-jwt-secret-1234 + ADMIN_USERNAME: admin + ADMIN_PASSWORD: admin-password-1234 + NODE_ENV: production + PORT: "8080" + depends_on: { mysql: { condition: service_healthy } } + ports: ["8080:8080"] +volumes: { mysql_data: {} } +``` + +- [ ] **Step 3: `scripts/e2e.sh`**(补全 detail/messages/stats/export 断言 —— S7) +```bash +#!/usr/bin/env bash +set -e +BASE=http://localhost:8080; ITOK=dev-ingest-token-1234 +JWT=$(curl -s -XPOST $BASE/api/auth/login -H 'Content-Type: application/json' \ + -d '{"username":"admin","password":"admin-password-1234"}' | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.parse(s).token))") +echo "auth ok" +curl -sf -XPOST $BASE/api/ingest -H "Authorization: Bearer $ITOK" -H 'Content-Type: application/json' \ + -d '{"inquiry":{"inquiryId":"E2E-1","createTime":"2026-07-13T10:00:00Z","updateTime":"2026-07-13T10:00:00Z","manager":"tester","status":"new"},"customer":{"customerId":"c-e2e","name":"Test Buyer","country":"US"},"messages":[{"messageId":"e2e-m1","sendTime":"2026-07-13T10:00:00Z","direction":"outgoing","content":"hello e2e"}]}' >/dev/null +echo "ingest ok" +curl -sf "$BASE/api/inquiries/E2E-1" -H "Authorization: Bearer $JWT" | grep -q E2E-1 && echo "detail ok" +curl -sf "$BASE/api/inquiries/E2E-1/messages" -H "Authorization: Bearer $JWT" | grep -q e2e-m1 && echo "messages ok" +curl -sf "$BASE/api/stats/overview" -H "Authorization: Bearer $JWT" | grep -q total && echo "stats ok" +curl -sf "$BASE/api/export" -H "Authorization: Bearer $JWT" | grep -q E2E-1 && echo "export ok" +``` + +- [ ] **Step 4: 起服务跑 e2e** +```bash +docker compose up -d --build +sleep 30; bash scripts/e2e.sh +``` +Expected:`auth/ingest/detail/messages/stats/export ok` 全部打印。 + +- [ ] **Step 5: 跑全量测试** +```bash +npm test +``` +Expected:全部 pass。 + +- [ ] **Step 6: Commit** `feat: docker dev env + e2e; Plan 1 v2 complete` + +--- + +## 完成标准(DoD) + +- [ ] 4 张表按 schema 建好(`has_failed_batches`、`MEDIUMTEXT content`、`Direction` 含 `system`、复合索引) +- [ ] `/api/ingest`:zod 校验 + token 鉴权(timingSafeEqual)+ 限流 + 早返回(`!hasFailedBatches` + count + lastSendTime) + 主表 upsert(customerUpdate 不覆盖 null)+ 批量消息(createMany skipDuplicates 分批 500、失败隔离)+ **messagesCount 用总量** +- [ ] `/api/auth/login` + `/refresh`:env bootstrap admin + JWT +- [ ] `/api/inquiries`(列表筛选/搜索/分页)+ `/:id`(详情)+ `/:id/messages`(复合游标分页,zod 校验)+ `/:id/messages/export`(单询盘聊天 CSV) +- [ ] `/api/stats/overview`(`_count` → `count` 重命名) +- [ ] `/api/export`(列表 CSV stream,不含聊天) +- [ ] **增量幂等回归测试**:推 N → 推 N+1 → 再推 N+1 必须早返回(S11) +- [ ] `docker compose up` 一键起,e2e 六步全 ok +- [ ] 全量 vitest 通过(惰性 Prisma + testcontainers + singleFork) + +## 后续 Plan(本 plan 不含) + +- **Plan 2**:插件推送改造(配置页 + IndexedDB 队列经 SW 落盘 + chrome.alarms worker + optional_host_permissions + apiMsgToChat 字段扩展) +- **Plan 3**:管理后台前端(Vue3 + Element Plus:登录/列表/详情虚拟滚动聊天/统计/导出;CORS 收窄) +- **Plan 4**:正式部署(域名 + Let's Encrypt + 备份 cron + 健康监控 + 限流切 token 维度) diff --git a/docs/superpowers/plans/2026-07-14-frontend.md b/docs/superpowers/plans/2026-07-14-frontend.md new file mode 100644 index 0000000..ad087d7 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-frontend.md @@ -0,0 +1,61 @@ +# 管理后台前端 实施计划 (Plan 3) + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development 或 superpowers:executing-plans。Steps 用 `- [ ]` 跟踪。 + +**Goal:** Vue3 + Element Plus 管理后台,对接 Plan 1 后端(`/api/*`),实现登录 + 询盘列表(筛选/搜索/分页)+ 详情(客户+行为+聊天虚拟滚动)+ 统计 + 导出。 + +**Architecture:** SPA。axios 调后端 REST API(JWT 鉴权)。Pinia 管 auth 状态。Vue Router(+ 守卫)。构建产物 dist 由 nginx 反代(/api→backend, /→dist)。 + +**Tech Stack:** Vue3 + TypeScript + Element Plus + Vite + Pinia + Vue Router + axios + ECharts + vue-virtual-scroller。 + +**关联:** spec §9;后端 Plan 1(`d:/alibb-server/backend`,compose 跑在 `localhost:8080`)。 + +**位置:** `d:/alibb-server/frontend/`(新建,git init)。 + +--- + +## 文件结构 + +``` +d:/alibb-server/frontend/ + src/ + api/{request.ts(client+JWT interceptor), inquiry.ts, stats.ts, auth.ts} + stores/auth.ts # token + 用户名(localStorage) + router/index.ts # 路由 + 守卫 + layouts/MainLayout.vue # 侧边栏 + 头部 + views/{Login.vue, InquiryList.vue, InquiryDetail.vue, Stats.vue} + components/{BehaviorCard.vue, ChatList.vue(虚拟滚动)} + App.vue / main.ts + vite.config.ts # 代理 /api → localhost:8080(dev) + Dockerfile / nginx.conf # 构建 + 反代 + package.json / tsconfig.json +``` + +--- + +## Tasks + +1. **项目初始化**:`npm create vue@latest`(Vue3+TS+Router+Pinia),装 Element Plus / axios / ECharts / vue-virtual-scroller。vite dev 代理 `/api→:8080`。git init。 +2. **API client + auth + 登录**:`api/request.ts`(axios,baseURL `/api`,请求拦截加 Bearer,401 跳登录);`stores/auth.ts`(token 存 localStorage);`views/Login.vue`(用户名+密码 → `/api/auth/login` → 存 token → 跳 `/inquiries`)。 +3. **主布局 + 路由守卫**:`MainLayout.vue`(左侧菜单:询盘/统计;头部:用户名/登出);router 守卫(无 token → `/login`)。 +4. **询盘列表** `InquiryList.vue`:Element Plus `el-table` + 顶部筛选(国家/业务员/状态/时间范围 el-date-picker)+ 搜索框 + el-pagination + 导出按钮(调 `/api/export` 下载 CSV)。列:询价单号/买家/国家/业务员/状态/创建时间/消息数。点行跳详情。 +5. **询盘详情** `InquiryDetail.vue`:`el-tabs` 或左右分栏:客户信息卡 + 行为数据卡(产品浏览/有效询盘/RFQ/登录天数/垃圾询盘/最常行业)+ 聊天(`ChatList.vue` vue-virtual-scroller,游标分页加载 `/api/inquiries/:id/messages`,左右气泡按 direction)。 +6. **统计** `Stats.vue`:数字卡片(总数/本月新增)+ ECharts(国家 Top10 饼图 / 业务员柱状图 / 月趋势折线)。调 `/api/stats/overview`。 +7. **导出**:列表页「导出 CSV」按钮 → `window.open('/api/export?...当前筛选', headers JWT)` 或 axios blob 下载。 +8. **构建 + nginx**:`npm run build` → dist;`nginx.conf`(/api → backend:8080,/ → dist);Dockerfile。可选:加到 backend 的 docker-compose(nginx 服务)。 + +--- + +## 完成标准(DoD) + +- [ ] 登录页 → 输入 admin/admin-password-1234 → 进列表 +- [ ] 列表:筛选(国家/业务员/状态/时间)+ 搜索 + 分页工作 +- [ ] 详情:客户 + 行为 + 聊天(虚拟滚动,游标分页)显示 +- [ ] 统计:卡片 + 3 图表 +- [ ] 导出:按筛选下载 CSV +- [ ] `npm run build` 成功;nginx 反代后前后端通 + +## 后续 + +- 真实 HTTPS 域名(Plan 4) +- 聊天全文搜索、客户合并视图、跟进管理 diff --git a/docs/superpowers/plans/2026-07-14-plugin-push.md b/docs/superpowers/plans/2026-07-14-plugin-push.md new file mode 100644 index 0000000..81308b2 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-plugin-push.md @@ -0,0 +1,395 @@ +# 插件推送改造 实施计划 (Plan 2) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 在现有 `d:\alibb\` Chrome 插件(已实现抓取 + API 直拉聊天)上**新增推送模块**:抓完一个询盘自动入队,由 service worker 经 `chrome.alarms` 消费、POST 到 Plan 1 后端 `/api/ingest`。带 IndexedDB 队列(防丢)+ 指数退避重试 + 配置页 + 运行时域名授权。**保留**现有本地导出能力(双写)。 + +**Architecture:** +- `background.js`(SW)改 **ES module**(`manifest` `type:module`),`import` 推送逻辑 `push/*.js` +- 推送逻辑分三个 ES module(可被 node + vitest 测试):`push/queue.js`(IndexedDB 队列)、`push/payload.js`(组装后端 payload)、`push/worker.js`(消费 + POST + 重试) +- `content.js` 保持经典脚本,抓完一条 → `chrome.runtime.sendMessage({type:'enqueue', record, messages})` 发原始数据给 SW;SW 组装 payload + 入队 +- IndexedDB 由 **SW 落盘**(content script 的 IDB 属页面源,v3 修订),`chrome.storage.local` 只存 `queue_index` + 元数据 +- `popup` 加「后端设置」(URL/token/启用)+ 推送状态显示;`chrome.permissions.request` 运行时授权后端域名 + +**Tech Stack:** 现有 JS 插件 + vitest 2 + fake-indexeddb(node 测队列)+ Chrome MV3 API(alarms/indexedDB/permissions/storage)。 + +**关联:** spec `docs/superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md` §8;后端 Plan 1 已就绪(`d:\alibb-server\backend`,`/api/ingest`)。 + +**前提:** `d:\alibb` 当前**非 git 仓库**,Task 1 会 `git init`。Plan 1 后端能 `docker compose up` 跑起来供端到端验证。 + +--- + +## 文件结构 + +``` +d:\alibb\ # 插件根(本 plan 改这里) + manifest.json # 改:type:module + permissions(alarms/unlimitedStorage) + optional_host_permissions + background.js # 改:ES module,import push/*,alarms worker,接收 enqueue;保留 storeDetail/getBgData(本地导出) + content.js # 改:apiMsgToChat 扩展(messageId/sendTime ISO/translation);fetchChatByApi 扩展(meta);processCurrentDetailPage 加 enqueue + popup.html / popup.js # 改:加后端设置区 + 推送状态显示 + push\ + queue.js # 新:IndexedDB 队列操作(addItem/getPending/markStatus/remove/getStats) + payload.js # 新:buildIngestPayload(record, messages) → 适配后端 zod + worker.js # 新:processQueue(消费 + POST + 指数退避) + tests\ + queue.test.js # vitest + fake-indexeddb + payload.test.js # 纯函数测试 + worker.test.js # mock fetch + fake-indexeddb + package.json # 新:vitest + fake-indexeddb + vitest.config.ts +``` + +--- + +## Task 1: node 测试基础 + git init + +**Files:** `package.json`, `vitest.config.ts`, `.gitignore` + +- [ ] **Step 1:** `cd d:/alibb && git init`(若已是 git 仓库则跳过) +- [ ] **Step 2:** `package.json`:`{ "name":"alibaba-inquiry-plugin", "type":"module", "scripts":{"test":"vitest run"} }` +- [ ] **Step 3:** `npm install -D vitest fake-indexeddb` +- [ ] **Step 4:** `vitest.config.ts`:`{ test:{ globals:true, testTimeout:30000, poolOptions:{ forks:{ singleFork:true } }, setupFiles:['fake-indexeddb/auto'] } }` +- [ ] **Step 5:** `.gitignore`:`node_modules/ .env`（注意:**不忽略** push/*.js、content.js 等插件文件） +- [ ] **Step 6:** commit `chore: add node test harness (vitest + fake-indexeddb)` + +--- + +## Task 2: manifest 改造 + +**Files:** `manifest.json` + +- [ ] **Step 1:** permissions 加 `alarms`、`unlimitedStorage` → `["storage","activeTab","alarms","unlimitedStorage"]` +- [ ] **Step 2:** 加 `optional_host_permissions: ["http://*/*","https://*/*"]`(后端域名运行时授权) +- [ ] **Step 3:** background 加 `"type":"module"` → `{ "service_worker":"background.js", "type":"module" }` +- [ ] **Step 4:** commit `feat(manifest): alarms/unlimitedStorage + optional_host_permissions + sw module` + +--- + +## Task 3: push/queue.js(IndexedDB 队列) + +**Files:** `push/queue.js`, `tests/queue.test.js` + +- [ ] **Step 1:** 写 `tests/queue.test.ts`(TDD):addItem → getPending 返回含它;markStatus('sent') → 不在 pending;getStats → {pending,failed,sent} 计数 +- [ ] **Step 2:** 跑测试确认失败 +- [ ] **Step 3:** 写 `push/queue.js`: +```js +const DB_NAME = 'inquiry_push', STORE = 'queue', VERSION = 1; +function openDb() { + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, VERSION); + req.onupgradeneeded = () => req.result.createObjectStore(STORE, { keyPath: 'id' }); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); +} +function tx(db, mode) { return db.transaction(STORE, mode).objectStore(STORE); } +function req2promise(r) { return new Promise((res, rej) => { r.onsuccess = () => res(r.result); r.onerror = () => rej(r.error); }); } + +export async function addItem(item) { + const db = await openDb(); + const rec = { id: crypto.randomUUID(), payload: item.payload, inquiryId: item.inquiryId, + status: 'pending', retries: 0, nextRetry: Date.now(), createdAt: Date.now() }; + await req2promise(tx(db, 'readwrite').add(rec)); + db.close(); + return rec.id; +} +export async function getPending(limit = 5) { + const db = await openDb(); + const all = await req2promise(tx(db, 'readonly').getAll()); + db.close(); + const now = Date.now(); + return all.filter(r => r.status === 'pending' && r.nextRetry <= now) + .sort((a, b) => a.nextRetry - b.nextRetry) + .slice(0, limit); +} +export async function markStatus(id, status, opts = {}) { + const db = await openDb(); + const store = tx(db, 'readwrite'); + const r = await req2promise(store.get(id)); + if (!r) { db.close(); return; } + Object.assign(r, { status, ...opts }); + await req2promise(store.put(r)); + db.close(); +} +export async function remove(id) { + const db = await openDb(); + await req2promise(tx(db, 'readwrite').delete(id)); + db.close(); +} +export async function getStats() { + const db = await openDb(); + const all = await req2promise(tx(db, 'readonly').getAll()); + db.close(); + return all.reduce((a, r) => { a[r.status] = (a[r.status] || 0) + 1; return a; }, {}); +} +``` +> `crypto.randomUUID()` 在 SW 和 node 18+ 都可用。fake-indexeddb 提供 indexedDB 全局。 +- [ ] **Step 4:** 测试通过 + commit `feat(push): IndexedDB queue (add/getPending/markStatus/remove/stats)` + +--- + +## Task 4: push/payload.js(组装后端 payload) + +**Files:** `push/payload.js`, `tests/payload.test.js` + +- [ ] **Step 1:** 测试:`buildIngestPayload(record, messages)` 输出匹配后端 zod(inquiryId/createTime ISO/updateTime ISO/customer{customerId,country,...}/messages[{messageId,sendTime ISO,direction,content,...}]);空字符串 behavior 归 null;messages 为空时抛错(后端 zod min(1)) +- [ ] **Step 2:** 写 `push/payload.js`: +```js +const s = v => (v == null || v === '') ? null : v; +const iso = ms => new Date(ms).toISOString(); + +export function buildIngestPayload(record, messages) { + if (!messages || messages.length === 0) throw new Error('messages required'); + const iq = record || {}; + return { + inquiry: { + inquiryId: String(iq.inquiryId), + manager: s(iq.manager) || undefined, + // 优先用 meta 的 gmtCreate/gmtModified(record.createTimeFromApi),回退列表 createTime + createTime: iq.createTimeFromApi || iq.createTime || iso(Date.now()), + updateTime: iq.updateTimeFromApi || iq.updateTime || iq.createTimeFromApi || iso(Date.now()), + status: s(iq.status) || undefined, + msgPreview: s(iq.msgPreview) || undefined, + isFlagged: !!iq.isFlagged, + }, + customer: iq.customerId ? { + customerId: String(iq.customerId), + name: s(iq.detailBuyerName) || s(iq.buyerName), + country: s(iq.detailCountry) || s(iq.countryCode) ? (s(iq.detailCountry) || s(iq.countryCode)) : null, + email: s(iq.email), company: s(iq.company), + registerTime: s(iq.registerTime) || null, + avatarUrl: s(iq.detailAvatarUrl) || s(iq.avatarUrl) || undefined, + buyerTags: s(iq.buyerTags) || undefined, buyerFeatures: s(iq.buyerFeatures) || undefined, + behavior: iq.behavior ? { + productViews: n(iq.behavior.productViews), validInquiries: n(iq.behavior.validInquiries), + validRfq: n(iq.behavior.validRfq), loginDays: n(iq.behavior.loginDays), + spamInquiries: n(iq.behavior.spamInquiries), topIndustry: s(iq.behavior.topIndustry), + } : undefined, + } : null, + messages: messages.map(m => ({ + messageId: String(m.messageId), sendTime: m.sendTime, + sender: s(m.sender) || undefined, direction: m.direction, + content: m.original ?? m.content ?? '', translation: s(m.translation) || '', + })), + }; +} +function n(v) { const x = Number(v); return Number.isFinite(x) ? x : null; } +``` +> 注:`messages` 已是 apiMsgToChat 转换后的对象(messageId/sendTime ISO/sender/original/direction/translation)。content 映射用 original(apiMsgToChat 产出 original)。 +- [ ] **Step 3:** 测试通过(对照 Plan 1 后端 PayloadSchema 语义)+ commit `feat(push): buildIngestPayload adapter` + +--- + +## Task 5: push/worker.js(消费 + POST + 重试) + +**Files:** `push/worker.js`, `tests/worker.test.js` + +- [ ] **Step 1:** 测试:mock global.fetch(成功 → markStatus sent 风格的 remove);失败 → retries++ nextRetry 退避;超过 8 次 → markStatus failed。注入 `{addItem,getPending,markStatus,remove}` mock 或用真实 fake-indexeddb + fetch mock +- [ ] **Step 2:** 写 `push/worker.js`: +```js +import { getPending, markStatus, remove } from './queue.js'; +import { buildIngestPayload } from './payload.js'; + +const MAX_RETRIES = 8; +let _config = null; // {serverUrl, token, enabled} + +export function configure(cfg) { _config = cfg; } + +export async function processQueue() { + if (!_config || !_config.enabled || !_config.serverUrl || !_config.token) return { processed: 0 }; + const items = await getPending(3); + let processed = 0; + for (const it of items) { + try { + const payload = buildIngestPayload(it.payload.record, it.payload.messages); + const res = await fetch(`${_config.serverUrl}/api/ingest`, { + method: 'POST', + headers: { 'Authorization': `Bearer ${_config.token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + if (res.ok) { await remove(it.id); processed++; } + else { await bumpRetry(it, `HTTP ${res.status}`); } + } catch (e) { + await bumpRetry(it, String(e && e.message || e)); + } + } + return { processed }; +} + +async function bumpRetry(it, err) { + const retries = it.retries + 1; + if (retries >= MAX_RETRIES) { + await markStatus(it.id, 'failed', { retries, lastError: err }); + } else { + const delay = Math.min(Math.pow(2, retries) * 1000, 3600 * 1000); + await markStatus(it.id, 'pending', { retries, nextRetry: Date.now() + delay, lastError: err }); + } +} +``` +- [ ] **Step 3:** 测试通过 + commit `feat(push): worker (consume + POST + backoff)` + +--- + +## Task 6: background.js(SW,alarms worker + 接收 enqueue) + +**Files:** `background.js` + +- [ ] **Step 1:** 顶部改 ES module import: +```js +import { addItem, getStats } from './push/queue.js'; +import { processQueue, configure } from './push/worker.js'; +``` +- [ ] **Step 2:** 保留现有 storeDetail/getBgData/clearBgData(本地导出)**不动** +- [ ] **Step 3:** 加配置加载 + alarms 注册 + enqueue 处理: +```js +const CONFIG_KEY = 'push_config'; +const ALARM = 'push-tick'; + +async function loadConfig() { + const d = await chrome.storage.local.get(CONFIG_KEY); + return d[CONFIG_KEY] || { enabled: false, serverUrl: '', token: '' }; +} + +// 启动 + 配置变更时刷新 worker 配置 + alarm +async function refresh() { + const cfg = await loadConfig(); + configure(cfg); + if (cfg.enabled) { + await chrome.alarms.create(ALARM, { periodInMinutes: 1 }); + processQueue(); // 立即跑一次 + } else { + await chrome.alarms.clear(ALARM); + } +} + +chrome.runtime.onInstalled.addListener(refresh); +chrome.runtime.onStartup.addListener(refresh); +chrome.alarms.onAlarm.addListener(a => { if (a.name === ALARM) processQueue(); }); + +chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { + if (msg.type === 'enqueue') { + (async () => { + const cfg = await loadConfig(); + if (!cfg.enabled) { sendResponse({ ok: false, reason: 'disabled' }); return; } + const id = await addItem({ payload: { record: msg.record, messages: msg.messages }, inquiryId: msg.record?.inquiryId }); + sendResponse({ ok: true, id }); + processQueue(); // 立即尝试 + })(); + return true; + } + if (msg.type === 'getPushStats') { + (async () => sendResponse(await getStats()))(); + return true; + } + if (msg.type === 'setPushConfig') { + (async () => { + await chrome.storage.local.set({ [CONFIG_KEY]: msg.config }); + await refresh(); + sendResponse({ ok: true }); + })(); + return true; + } +}); +``` +- [ ] **Step 4:** commit `feat(bg): push worker + alarms + enqueue handler (keep local export)` + +--- + +## Task 7: content.js — apiMsgToChat 扩展 + +**Files:** `content.js`(apiMsgToChat ~434) + +- [ ] **Step 1:** apiMsgToChat 改产出 `{ messageId, sendTime(ISO), sender, original, translation, direction }`: + - `messageId: String(m.messageId)` + - `sendTime: new Date(m.sendTime).toISOString()`(替代原来的本地时区 "YYYY-MM-DD HH:MM:SS") + - `translation`:从 `m.extInfo?.icbuData?.extend` 解析(extend 是 JSON 字符串,内含 extParams.inputTranslateInfo.translate);解析失败留 `''` + - sender/original/direction 保持 +- [ ] **Step 2:** translation 解析加双重 JSON.parse 兜底(extend 可能是字符串里的字符串),失败不抛(留 '') +- [ ] **Step 3:** 手动/Playwright 验证(无 node 测试,因 content.js 非 module):用 Playwright 注入新版 apiMsgToChat + 真实 m 样本,断言 messageId/sendTime/translation 正确 +- [ ] **Step 4:** commit `feat(content): apiMsgToChat emits messageId/sendTime(ISO)/translation` + +--- + +## Task 8: content.js — fetchChatByApi 扩展(提取 gmtCreate/country) + +**Files:** `content.js`(fetchChatByApi ~451) + +- [ ] **Step 1:** fetchChatByApi 在调 querySummary 后,额外提取 `d.gmtCreate`/`d.gmtModified`(毫秒→ISO)存入返回 meta;`d.contact?.countryCode` 或 `d.cid` 派生也行(spec 用 contact.countryCode)。返回 `{ messages, meta: { gmtCreate, gmtModified } }` +- [ ] **Step 2:** 调用方(processCurrentDetailPage / scrapeDetailData)把 meta 挂到 detail:`detail.createTimeFromApi = meta.gmtCreate; detail.updateTimeFromApi = meta.gmtModified` +- [ ] **Step 3:** commit `feat(content): fetchChatByApi extracts gmtCreate/gmtModified for payload` + +--- + +## Task 9: content.js — processCurrentDetailPage 加 enqueue + +**Files:** `content.js`(processCurrentDetailPage ~347,storeDetail 之后) + +- [ ] **Step 1:** 在 storeDetail(本地导出)**之后**加 enqueue(推送): +```js +// 推送到后端(Plan 2):发 record + messages 给 SW 入队 +try { + chrome.runtime.sendMessage({ + type: 'enqueue', + record: merged, // {item, detail} 合并 + messages: detail.chatMessages || [], // apiMsgToChat 产出数组 + }, () => {}); // fire-and-forget,失败不影响本地导出 +} catch (e) {} +``` +> `detail.chatMessages` 需在 scrapeDetailData 产出(现有 chatText 是序列化串,加一个 `chatMessages` 数组字段)。确保 merged 含 `createTimeFromApi/updateTimeFromApi`(Task 8)。 +- [ ] **Step 2:** commit `feat(content): enqueue record+messages to SW after detail scrape` + +--- + +## Task 10: popup 配置页 + permissions.request + +**Files:** `popup.html`, `popup.js` + +- [ ] **Step 1:** popup.html 加「后端设置」区:服务器 URL 输入、Token 输入、启用推送 checkbox、「授权域名」按钮、「保存」按钮 +- [ ] **Step 2:** popup.js:加载时 `chrome.runtime.sendMessage({type:'getPushConfig'})` 回填(或直接 `chrome.storage.local.get`);保存时 setPushConfig +- [ ] **Step 3:** 「授权域名」按钮 → `chrome.permissions.request({origins:[serverUrl + '/*']})`(用户手势触发);成功后提示 +- [ ] **Step 4:** commit `feat(popup): backend config UI + runtime origin permission` + +--- + +## Task 11: popup 推送状态显示 + +**Files:** `popup.html`, `popup.js` + +- [ ] **Step 1:** popup 加状态条:「已推 N / 待推 M / 失败 K」 +- [ ] **Step 2:** popup 打开时 + 每 2s `chrome.runtime.sendMessage({type:'getPushStats'})` 刷新 +- [ ] **Step 3:** 失败 K > 0 时红字 + 可点击查看 lastError(简化:弹 alert 或展开列表) +- [ ] **Step 4:** commit `feat(popup): push stats display (pending/failed/sent)` + +--- + +## Task 12: 端到端验证 + +**Files:** `scripts/e2e-plugin.md`(验证步骤文档) + +- [ ] **Step 1:** 起 Plan 1 后端:`cd d:/alibb-server/backend && docker compose up -d --build`,等 health ok +- [ ] **Step 2:** Chrome `chrome://extensions` 开发者模式 → 加载 `d:\alibb`(本 plan 改后的插件) +- [ ] **Step 3:** 点插件图标 → 后端设置填 `http://localhost:8080` + token `dev-ingest-token-1234` + 启用 → 授权域名 → 保存 +- [ ] **Step 4:** 打开 `message.alibaba.com` → 抓一个询盘(用插件现有抓取流程,或 Playwright 注入触发) +- [ ] **Step 5:** 验证:`curl -H "Authorization: Bearer " http://localhost:8080/api/inquiries` 能看到刚抓的询盘;`docker compose exec mysql mysql -uroot -proot inquiry -e "SELECT inquiry_id, messages_count FROM inquiries"` 数据在 +- [ ] **Step 6:** popup 状态条显示「已推 1」 +- [ ] **Step 7:** 验证幂等:再抓同一询盘 → 后端 messages_count 不变(早返回) +- [ ] **Step 8:** commit `test: e2e plugin→backend validation doc` + +--- + +## 完成标准(DoD) + +- [ ] manifest:`type:module` + alarms/unlimitedStorage + optional_host_permissions +- [ ] `push/queue.js` / `payload.js` / `worker.js` 三个 module + vitest 测试全绿 +- [ ] `background.js`:alarms(1 分钟)消费队列,POST `/api/ingest`,失败指数退避(≤8 次);保留本地 storeDetail/getBgData +- [ ] `content.js`:apiMsgToChat 产 messageId/sendTime(ISO)/translation;fetchChatByApi 提取 gmtCreate/gmtModified;processCurrentDetailPage enqueue +- [ ] popup:后端设置(URL/token/启用)+ permissions.request 授权 + 状态显示(已推/待推/失败) +- [ ] 端到端:抓一条询盘 → Plan 1 后端能看到数据;再抓同一条幂等(messages_count 不变) +- [ ] 断网/后端停 → 数据进队列不丢;后端恢复 → 自动推完 + +## 后续(本 plan 不含) + +- Plan 3:管理后台前端(查看/筛选/导出/统计) +- Plan 4:正式部署(域名 + HTTPS + 备份定时 + 监控) +- 增量推送(插件记录 last sendTime,只推新增消息) +- 失败项重试 UI(popup 查看失败 + 手动重推) diff --git a/docs/superpowers/plans/2026-07-15-deploy-test-env.md b/docs/superpowers/plans/2026-07-15-deploy-test-env.md new file mode 100644 index 0000000..17c79be --- /dev/null +++ b/docs/superpowers/plans/2026-07-15-deploy-test-env.md @@ -0,0 +1,126 @@ +# 线上测试环境部署(1Panel + alibaba 库 + OpenResty 站点) + +> 实际部署方案(2026-07-17)。对应 spec §11.2「测试环境:公网 IP + 端口 + HTTP」。 + +## 目标服务器 + +- 公网 IP:`219.157.255.213`,Ubuntu 24.04,主机 `kt-W580G20` +- 登录用户:`kt`(sudo 免密,不在 docker 组但 `sudo docker` 可用) +- 已装:Docker 28.0.1 + compose v2.33.1、Node v20(前端 build 用 docker node 镜像) +- 镜像加速:`/etc/docker/daemon.json` → `https://docker.1panel.live` +- 已有 1Panel + OpenResty(管理所有站点,占 80/443) + +## SSH 访问(本机 → 服务器) + +- 本机密钥:`C:\Users\30997\.ssh\alibb_deploy`(ed25519,无 passphrase) +- 连接(用 Windows 自带 OpenSSH,Git Bash 的 ssh 因私钥权限 644 被拒): + ```bash + /c/Windows/System32/OpenSSH/ssh.exe -i 'C:\Users\30997\.ssh\alibb_deploy' kt@219.157.255.213 '...' + ``` +- 或本机 `~/.ssh/config` 加别名(Windows 路径 IdentityFile): + ``` + Host alibb + HostName 219.157.255.213 + User kt + IdentityFile C:\Users\30997\.ssh\alibb_deploy + ``` + +## 架构 + +``` +浏览器 ──http:23010──► 1Panel OpenResty 站点(alibb) + ├─ / → 站点 index(前端 dist 静态) + └─ /api/* → backend 容器(:8080) +后端容器(alibb-backend,Express)──► 1Panel MySQL(alibaba 库,:3306) +扩展(抓 alibaba)──POST──► http://219.157.255.213:23010/api/ingest +``` + +- **后端**:`~/alibb-server/backend`,docker compose(backend 服务,build 本地 Dockerfile,映射 127.0.0.1:8080)。连 **1Panel 已有的 MySQL**(容器 `1Panel-mysql-sLiG`,库 `alibaba`,密码 `wecbni5zRRnZrMMh`)。env:JWT_SECRET/INGEST_TOKEN/ADMIN_PASSWORD 强随机(存服务器 `~/alibb-secrets.txt`)。 +- **前端**:`~/alibb-server/frontend`,docker build 出 dist → 复制到 **1Panel 站点目录** `/opt/1panel/apps/openresty/openresty/www/sites/alibb/index`。用 1Panel 的 OpenResty serve(不再单独起 nginx 容器)。 +- **站点**:1Panel 建的「alibb」站点,`listen 23010`,`server_name 219.157.255.213`,root = 站点 index,`/api` 反代到 `backend:8080`(容器名),`try_files` SPA fallback,`client_max_body_size 50m`。站点 conf 在 `/opt/1panel/apps/openresty/openresty/conf/conf.d/alibb.conf`(1Panel 管理,**改后要在面板重载 OpenResty**,别手改被覆盖)。 +- **访问**:http://219.157.255.213:23010(admin / ADMIN_PASSWORD) + +## 部署步骤(从零) + +### 1. SSH 密钥(本机生成 + 上传 kt) +```bash +# 本机 Git Bash 生成 +ssh-keygen -t ed25519 -f ~/.ssh/alibb_deploy -N "" +# 上传到 kt(用密码) +/c/Windows/System32/OpenSSH/scp.exe -i ... ~/.ssh/alibb_deploy.pub kt@219.157.255.213:... # 或 ssh-copy-id +``` + +### 2. 传代码(本机 → 服务器 ~/alibb-server) +```bash +cd /d && tar -czf ~/alibb-server.tgz --exclude='alibb-server/*/node_modules' --exclude='*/dist' --exclude='*/.git' alibb-server +/c/Windows/System32/OpenSSH/scp.exe -i ... 'C:\Users\30997\alibb-server.tgz' kt@219.157.255.213:/tmp/ +# 服务器解压 +... 'tar xzf /tmp/alibb-server.tgz -C ~' +``` + +### 3. 后端 build + up(连 alibaba 库) +```bash +cd ~/alibb-server/backend +# 生成线上密钥(替换 compose dev 值) +JWT_SECRET=$(openssl rand -hex 32); INGEST_TOKEN=$(openssl rand -hex 24); ADMIN_PASSWORD=$(openssl rand -hex 16) +sed -i -e "s/dev-ingest-token-1234/$INGEST_TOKEN/" -e "s/dev-jwt-secret-1234/$JWT_SECRET/" -e "s/admin-password-1234/$ADMIN_PASSWORD/" docker-compose.yml +# compose 改成连 1Panel mysql(alibaba 库)+ backend 端口映射 127.0.0.1:8080 +sudo docker compose build backend +sudo docker compose up -d +``` + +### 4. 前端 build → 复制站点 +```bash +cd ~/alibb-server/frontend +sudo docker run --rm -v "$PWD":/app -w /app node:22-alpine sh -c 'npm ci && npm run build' +SITE=/opt/1panel/apps/openresty/openresty/www/sites/alibb/index +sudo rm -rf "$SITE"/* && sudo cp -r dist/. "$SITE"/ && sudo chmod -R a+rX "$SITE" +``` + +### 5. 站点 conf(1Panel 面板配或 alibb.conf) +1Panel 面板建站点「alibb」(域名/IP `219.157.255.213`,端口 23010,根目录 = 站点 index),再配: +- 伪静态:`try_files $uri $uri/ /index.html;`(SPA 路由刷新) +- 反向代理:`/api/` → `http://backend:8080`(容器名,同 docker 网络;或 `http://127.0.0.1:8080`) +- 或直接编辑 `alibb.conf`(root + `/api` proxy + try_files + client_max_body_size 50m),面板重载 OpenResty。 + +### 6. 安全组放 23010(云控制台) +TCP 23010 入站;服务器防火墙 `sudo ufw allow 23010/tcp`(若开)。 + +### 7. 验证 +```bash +curl http://219.157.255.213:23010/ # 200(index.html) +curl http://219.157.255.213:23010/inquiries # 200(SPA fallback) +curl http://219.157.255.213:23010/api/health # {"ok":true} +``` +浏览器开 http://219.157.255.213:23010 → admin / ADMIN_PASSWORD 登录。 + +### 8. 扩展配置 +popup:serverUrl `http://219.157.255.213:23010` + token(~/alibb-secrets.txt 的 INGEST_TOKEN)→ 授权 → 启用推送 → 抓取。 + +## 增量更新(改代码后) + +- **后端改**:传变更文件 → `cd ~/alibb-server/backend && sudo docker compose build backend && sudo docker compose up -d`(migrate 自动跑)。 +- **前端改**:传变更 → 服务器 `npm run build` → 复制 dist 到站点(Step 4)。 +- **扩展改**:本机改 → chrome://extensions 刷新扩展(用户操作)。 + +## 常用运维 + +| 操作 | 命令 | +|---|---| +| 后端日志 | `sudo docker compose logs -f backend` | +| 重启后端 | `sudo docker compose restart backend` | +| 查库 | `sudo docker exec -e MYSQL_PWD=wecbni5zRRnZrMMh 1Panel-mysql-sLiG mysql -ualibaba alibaba` | +| 清询盘重抓 | `... mysql ... -e "SET FOREIGN_KEY_CHECKS=0; TRUNCATE chat_messages; TRUNCATE inquiries; SET FOREIGN_KEY_CHECKS=1;"`(customers 保留,重抓 upsert 更新) | + +## 已知限制 + +- **折叠面板字段抓不到**:客户资料折叠面板里的 手机/座机/商业类型/销售平台/公司官网,alibaba 检查 `event.isTrusted`(只接受真实鼠标点击),扩展 JS 无法展开。这些字段大部分买家本就是"-"(alibaba 默认隐藏联系方式)。保留抓其他全部(基本信息 + 行为数据 + 店内行为 + 最近询盘产品图)。 +- **风控**:抓取间隔已随机化(randSleep),降低被识别为爬虫的概率;大量抓取建议分批 + 中途暂停。 +- **MySQL 公网暴露**:1Panel 的 mysql 映射 `0.0.0.0:3306`(1Panel 默认,非本次部署),建议后续在 1Panel 收紧为本地。 + +## 仓库 + +- 扩展:`d:/alibb`(master) +- 后端:`d:/alibb-server/backend`(master) +- 前端:`d:/alibb-server/frontend`(main) +- 服务器代码:`~/alibb-server/{backend,frontend}` diff --git a/docs/superpowers/plans/2026-08-25-chat-time-range.md b/docs/superpowers/plans/2026-08-25-chat-time-range.md new file mode 100644 index 0000000..5cfb8f3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-25-chat-time-range.md @@ -0,0 +1,587 @@ +# 聊天时间范围抓取 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 聊天抓取支持时间范围(全部/近1/2/3月/自定义起止),到范围起点即停翻页 —— 时间范围同时是风控减压手段。 + +**Architecture:** spec `docs/superpowers/specs/2026-08-25-chat-time-range-design.md`(7轮审查+真机实测,含附录A实现基准伪代码、附录B实测记录)。改动全部在扩展侧(popup + content.js),后端/推送零改动。判定逻辑以 `scripts/verify-chat-range.mjs` 纯函数为基准实现,content.js/popup.js 照搬(IIFE 架构无法被 vitest import,沿用 verify-risk-detect.mjs 复刻模式)。 + +**Tech Stack:** Chrome MV3 扩展(原生 JS)、vitest(fake-indexeddb)、node 复刻验证脚本。 + +--- + +## ⛔ 风控红线(全程有效,用户明确要求) + +1. **不得改动任何现有安全层参数**:`randSleep(2000,5000)` 翻页间隔、`waitUntilWorkTime`、`postForm` 的 RISK 检测、`RISK_KEYWORDS`/`SKIP_KEYWORDS`。 +2. **真机验证只用小规模**:手动验证(Task 8)一律 `limit=10`,单次验证 ≤ 10 个询盘,两次验证之间间隔 ≥ 2 分钟。 +3. **不新增任何 API 端点调用**:只改现有 `querySummary`/`getOpMessages` 的参数(timeStamp)与本地逻辑。 +4. Task 1-7 全部是本地代码/纯逻辑验证,**零真实请求**。 + +## File Structure + +| 文件 | 职责 | +|---|---| +| `scripts/verify-chat-range.mjs`(新) | 判定逻辑基准实现 + 用例锁定(纯函数,零请求) | +| `popup.html` | 时间范围下拉/日期框/置灰 UI | +| `popup.js` | computeChatRange 计算、校验、start 传参 | +| `content.js` | state 字段、fetchChatByApi 主路径、水位结构、DOM 回退 | +| `docs/.../spec`(已完成) | 依据,不再改 | + +**同步义务**:verify 脚本中的纯函数(`minusMonths/computeChatRange/inChatRange/parseWatermark/shouldWriteWatermark/probeFailKind/toMs`)与 popup.js/content.js 内联版本必须逐字一致 —— 改任一侧必须同步另一侧(文件头注明)。 + +--- + +### Task 1: verify-chat-range.mjs — 判定逻辑基准(TDD) + +**Files:** +- Create: `scripts/verify-chat-range.mjs` + +- [ ] **Step 1: 写用例 + 桩函数(预期红)** + +```js +// 验证聊天时间范围判定逻辑(纯函数基准实现)。 +// content.js/popup.js 为 IIFE/非 module,无法被 vitest import —— 本文件是判定逻辑的 +// 基准实现+用例锁定;实现侧照搬函数体。⚠️ 改任一侧必须同步另一侧。 +let pass = 0, fail = 0; +const ok = (name, cond) => { console.log((cond ? 'PASS' : 'FAIL') + ' | ' + name); cond ? pass++ : fail++; }; + +// ---- 基准实现(实现照搬区)---- +function minusMonths(base, n) { throw new Error('TODO'); } +function computeChatRange(mode, sinceStr, untilStr) { throw new Error('TODO'); } +function inChatRange(sendTime, sinceMs, untilMs) { throw new Error('TODO'); } +function parseWatermark(v) { throw new Error('TODO'); } +function shouldWriteWatermark(oldWm, candId, candSendTime) { throw new Error('TODO'); } +function probeFailKind(json) { throw new Error('TODO'); } + +// ---- 用例 ---- +ok('3-31 − 1月 = 2-28(月末钳制,setMonth 溢出防护)', (() => { + const r = minusMonths(new Date(2026, 2, 31, 15, 30), 1); + return r.getMonth() === 1 && r.getDate() === 28 && r.getHours() === 15; +})()); +ok('8-25 − 1月 = 7-25(普通日期)', (() => { + const r = minusMonths(new Date(2026, 7, 25), 1); + return r.getMonth() === 6 && r.getDate() === 25; +})()); +ok('12-31 − 2月 = 10-31(同年跨月)', (() => { + const r = minusMonths(new Date(2026, 11, 31), 2); + return r.getMonth() === 9 && r.getDate() === 31; +})()); +ok('1-31 − 2月 = 上年 11-30(跨年+钳制)', (() => { + const r = minusMonths(new Date(2026, 0, 31), 2); + return r.getFullYear() === 2025 && r.getMonth() === 10 && r.getDate() === 30; +})()); +ok('闰年:2028-03-31 − 1月 = 2028-02-29', (() => { + const r = minusMonths(new Date(2028, 2, 31), 1); + return r.getMonth() === 1 && r.getDate() === 29; +})()); + +ok('all → 双 null(完全同现状)', computeChatRange('all').since === null && computeChatRange('all').until === null); +ok('1m → since 在 25~32 天前,until 不限', (() => { + const r = computeChatRange('1m'); + const age = Date.now() - Date.parse(r.since); + return r.until === null && age > 25 * 86400e3 && age < 32 * 86400e3; +})()); +ok('custom 本地时区(since=本地0点,非 UTC)', (() => { + const r = computeChatRange('custom', '2026-08-25', '2026-08-25'); + const s = new Date(r.since); + return s.getMonth() === 7 && s.getDate() === 25 && s.getHours() === 0 && s.getMinutes() === 0; +})()); +ok('custom until=止日 23:59:59.999 本地(含全天)', (() => { + const r = computeChatRange('custom', '2026-08-01', '2026-08-25'); + const u = new Date(r.until); + return u.getDate() === 25 && u.getHours() === 23 && u.getMinutes() === 59 && u.getMilliseconds() === 999; +})()); +ok('custom 起>止 → error', !!computeChatRange('custom', '2026-08-10', '2026-08-01').error); +ok('custom 起为空 → error', !!computeChatRange('custom', '', '2026-08-01').error); +ok('custom 止>今天 → error', !!computeChatRange('custom', '2026-01-01', '2099-01-01').error); + +const SINCE = Date.parse('2026-07-25T00:00:00'), UNTIL = Date.parse('2026-08-25T23:59:59'); +ok('since 当天消息收(含边界)', inChatRange(SINCE, SINCE, null)); +ok('since 前一天不收', !inChatRange(SINCE - 86400e3, SINCE, null)); +ok('until 末毫秒收(含边界)', inChatRange(UNTIL, null, UNTIL)); +ok('until 次日不收', !inChatRange(UNTIL + 1, null, UNTIL)); +ok('sendTime=null 防御性保留', inChatRange(null, SINCE, UNTIL)); +ok('范围不限(null)全收', inChatRange(1, null, null)); + +ok('新格式水位解析', (() => { const w = parseWatermark('{"id":"a1","sendTime":1787626659384}'); return w.id === 'a1' && w.sendTime === 1787626659384; })()); +ok('旧格式(纯string)→ legacy,sendTime 未知', (() => { const w = parseWatermark('abc123'); return w.id === 'abc123' && w.sendTime === null; })()); +ok('只前进:候选6-30 不覆盖 7-28', !shouldWriteWatermark({ id: 'o', sendTime: Date.parse('2026-07-28') }, 'n', Date.parse('2026-06-30'))); +ok('只前进:候选8-25 覆盖 7-28', shouldWriteWatermark({ id: 'o', sendTime: Date.parse('2026-07-28') }, 'n', Date.parse('2026-08-25'))); +ok('legacy 允许覆盖一次(升级)', shouldWriteWatermark({ id: 'o', sendTime: null }, 'n', Date.parse('2026-06-30'))); +ok('无旧水位 → 写入', shouldWriteWatermark(null, 'n', 1)); +ok('候选缺 sendTime → 不写', !shouldWriteWatermark({ id: 'o', sendTime: 5 }, 'n', null)); + +ok('probe 正常(200+非空) → null', probeFailKind({ code: '200', data: { list: [{}] } }) === null); +ok('probe code≠200 → fail 降级', probeFailKind({ code: '500' }) === 'fail'); +ok('probe data=null → fail 降级', probeFailKind({ code: '200', data: null }) === 'fail'); +ok('probe 非对象 → fail 降级', probeFailKind('ok') === 'fail'); +ok('probe 空 list → empty 降级(实测形态)', probeFailKind({ code: '200', data: { list: [] } }) === 'empty'); +ok('probe list 缺失 → empty 降级', probeFailKind({ code: '200', data: {} }) === 'empty'); + +// toMs:parse 容错(spec §5:非法 → null 降级不限) +function toMs(v) { const n = v ? Date.parse(v) : NaN; return Number.isFinite(n) ? n : null; } +ok('toMs 非法/空 → null,合法 → 毫秒', toMs('garbage') === null && toMs(null) === null && toMs('2026-08-25T00:00:00Z') === Date.parse('2026-08-25T00:00:00Z')); +// until 游标 = untilMs+1:附录B实测语义=返回 sendTime < cursor,故恰好覆盖 <= untilMs 全部、不漏不重 +ok('until 游标 +1ms 恰好覆盖含止日末毫秒', (() => { + const untilMs = Date.parse('2026-06-30T23:59:59.999'); + const cursor = untilMs + 1; + return (untilMs < cursor) && !(untilMs + 1 < cursor); +})()); + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); +``` + +- [ ] **Step 2: 跑,确认红** + +Run: `node scripts/verify-chat-range.mjs` +Expected: 脚本在**第一个用例即崩溃**(stack trace: `Error: TODO`)—— 桩函数的 throw 发生在用例 IIFE 内,无法逐行 FAIL;崩溃 + exit 1 即为有效红灯 + +- [ ] **Step 3: 填基准实现(替换 6 个桩函数)** + +```js +// 自然月回退:直接 setMonth(-n) 月末溢出(3-31−1月→"2-31"→3-02),先退月初再钳制天数 +function minusMonths(base, n) { + const d = new Date(base.getFullYear(), base.getMonth(), 1, + base.getHours(), base.getMinutes(), base.getSeconds(), base.getMilliseconds()); + d.setMonth(d.getMonth() - n); + const lastDay = new Date(d.getFullYear(), d.getMonth() + 1, 0).getDate(); + d.setDate(Math.min(base.getDate(), lastDay)); + return d; +} +// popup 范围计算:mode=all|1m|2m|3m|custom → {since,until}(ISO|null) 或 {error} +function computeChatRange(mode, sinceStr, untilStr) { + if (mode === 'all') return { since: null, until: null }; + if (mode === '1m' || mode === '2m' || mode === '3m') { + return { since: minusMonths(new Date(), Number(mode[0])).toISOString(), until: null }; + } + if (mode === 'custom') { + if (!sinceStr || !untilStr) return { error: '请填写自定义起止日期' }; + const s = new Date(sinceStr + 'T00:00:00.000'); // ⚠️ 必须拼时间:裸 date-only 按 UTC 解析差 8h + const u = new Date(untilStr + 'T23:59:59.999'); + if (isNaN(s.getTime()) || isNaN(u.getTime())) return { error: '日期格式错误' }; + if (s.getTime() > u.getTime()) return { error: '起始日期不能晚于截止日期' }; + const uDay = new Date(untilStr + 'T00:00:00.000'); + const t0 = new Date(); t0.setHours(0, 0, 0, 0); + if (uDay.getTime() > t0.getTime()) return { error: '截止日期不能晚于今天' }; + return { since: s.toISOString(), until: u.toISOString() }; + } + return { error: '未知时间范围模式' }; +} +// 范围过滤(null sendTime 防御性保留 —— 宁多勿漏) +function inChatRange(sendTime, sinceMs, untilMs) { + if (sendTime == null) return true; + if (sinceMs != null && sendTime < sinceMs) return false; + if (untilMs != null && sendTime > untilMs) return false; + return true; +} +// 水位读取兼容:旧=纯 string(sendTime 未知) / 新={id,sendTime} +function parseWatermark(v) { + if (v == null || v === '') return null; + try { + const o = JSON.parse(v); + if (o && typeof o === 'object' && o.id != null) return { id: String(o.id), sendTime: Number(o.sendTime) || null }; + } catch (e) { /* 旧格式继续 */ } + return { id: String(v), sendTime: null }; +} +// 水位只前进写入判定 +function shouldWriteWatermark(oldWm, candId, candSendTime) { + if (candId == null || candSendTime == null) return false; + if (oldWm == null) return true; + if (oldWm.sendTime == null) return true; // legacy 升级,允许覆盖一次 + return candSendTime > oldWm.sendTime; +} +// probe 轮断言分类:'fail'(非对象/code≠200/data=null) | 'empty'(空list) | null(正常)。 +// non-JSON/HTTP/关键词由 postForm 抛 RISK,不经此函数(真风控照抛,不降级) +function probeFailKind(json) { + if (!json || typeof json !== 'object') return 'fail'; + if (json.code !== undefined && Number(json.code) !== 200) return 'fail'; + if (json.data == null) return 'fail'; + const list = json.data && json.data.list; + if (!Array.isArray(list) || list.length === 0) return 'empty'; + return null; +} +``` + +- [ ] **Step 4: 跑,确认全绿(36 passed)** + +Run: `node scripts/verify-chat-range.mjs` +Expected: `36 passed, 0 failed`,exit 0 + +- [ ] **Step 5: Commit** + +```bash +git add scripts/verify-chat-range.mjs +git commit -m "feat(ext): chat time-range predicate baseline + verification cases" +``` + +--- + +### Task 2: popup UI + 传参 + +**Files:** +- Modify: `popup.html`(options 区,chatLimitInput 行附近) +- Modify: `popup.js`(元素引用、syncChatModeUI、start 计算) + +- [ ] **Step 1: popup.html 加控件** + +在"每询盘聊天上限"label 行后追加(chatLimit label 内加 hint span): + +```html + + + +``` + +- [ ] **Step 2: popup.js 加引用 + 模式切换 + 计算传参** + +元素引用区(文件顶部 getElementById 区)追加: + +```js +const chatTimeRangeSelect = document.getElementById('chatTimeRange'); +const chatCustomDates = document.getElementById('chatCustomDates'); +const chatSinceDate = document.getElementById('chatSinceDate'); +const chatUntilDate = document.getElementById('chatUntilDate'); +const chatLimitHint = document.getElementById('chatLimitHint'); +``` + +(在 push-config 区之前)追加模式切换 + 基准函数照搬(注明同步义务,函数体与 verify-chat-range.mjs **逐字一致**): + +```js +// ==== 聊天时间范围(spec 2026-08-25 §4.1)==== +// minusMonths/computeChatRange 与 scripts/verify-chat-range.mjs 基准实现逐字同步 +function minusMonths(base, n) { /* 同 verify-chat-range.mjs */ } +function computeChatRange(mode, sinceStr, untilStr) { /* 同 verify-chat-range.mjs */ } + +function syncChatModeUI() { + const mode = chatTimeRangeSelect.value; + const custom = mode === 'custom'; + chatCustomDates.style.display = custom ? 'block' : 'none'; + if (custom && !chatUntilDate.value) { + const t = new Date(); + chatUntilDate.value = `${t.getFullYear()}-${String(t.getMonth() + 1).padStart(2, '0')}-${String(t.getDate()).padStart(2, '0')}`; + } + const timeMode = mode !== 'all'; + document.getElementById('chatLimitInput').disabled = timeMode; // D1: 时间替代条数 + chatLimitHint.textContent = timeMode ? '(时间模式下忽略)' : ''; +} +chatTimeRangeSelect.addEventListener('change', syncChatModeUI); +syncChatModeUI(); +``` + +btnStart click 里,`send('start', {...})` **之前**插入校验,并改传参: + +```js +const range = computeChatRange(chatTimeRangeSelect.value, chatSinceDate.value, chatUntilDate.value); +if (range.error) { addLog(range.error, 'error'); statusText.textContent = '就绪'; updateButtons('idle'); return; } +``` + +`send('start', {...})` 的参数改为: + +```js +await send('start', { + mode: modeSelect.value, + dedup: optDedup.checked, + chat: optChat.checked, + limit: parseInt(limitSelect.value, 10), + chatLimit: chatTimeRangeSelect.value === 'all' ? (parseInt(document.getElementById('chatLimitInput').value, 10) || 0) : 0, + chatSince: range.since, + chatUntil: range.until, +}); +``` + +- [ ] **Step 3: 语法自检** + +Run: `node --check popup.js 2>&1 || echo "popup.js 非 module,--check 对 ESM 误报可忽略;改用浏览器加载验证"` +手动:重载扩展 → 打开 popup → 切下拉:custom 显示日期框且止=今天;时间模式下条数框置灰;切回 all 恢复。(零请求) + +- [ ] **Step 4: Commit** + +```bash +git add popup.html popup.js +git commit -m "feat(ext): chat time-range picker in popup (all/1m/2m/3m/custom)" +``` + +--- + +### Task 3: content.js — state 字段 + 派生 + +**Files:** +- Modify: `content.js`(start 分支 state 重建对象;fetchChatByApi/loadAllChatHistory 入口) + +- [ ] **Step 1: start 分支重建对象加两字段** + +`case 'start':` 的 `state = {...}` 对象中,`chatLimit: msg.chatLimit ?? 0,` 行后加: + +```js +chatSince: msg.chatSince ?? null, chatUntil: msg.chatUntil ?? null, // 时间范围(spec §4.2);显式列入重建清单 +``` + +- [ ] **Step 2: 文件顶部(IIFE 内、RISK_KEYWORDS 之前)加范围工具(照搬 verify + 派生)** + +```js +// ==== 聊天时间范围工具(与 scripts/verify-chat-range.mjs 基准逐字同步)==== +function inChatRange(sendTime, sinceMs, untilMs) { /* 同 verify-chat-range.mjs */ } +function parseWatermark(v) { /* 同 verify-chat-range.mjs */ } +function shouldWriteWatermark(oldWm, candId, candSendTime) { /* 同 verify-chat-range.mjs */ } +function probeFailKind(json) { /* 同 verify-chat-range.mjs */ } +function toMs(v) { const n = v ? Date.parse(v) : NaN; return Number.isFinite(n) ? n : null; } // 同 verify-chat-range.mjs 照搬区(parse 失败→null 降级不限,spec §5) +``` + +- [ ] **Step 3: fetchChatByApi 与 loadAllChatHistory 入口各派生一次** + +```js +const sinceMs = toMs(state.chatSince), untilMs = toMs(state.chatUntil); +``` + +- [ ] **Step 4: 手动自检(零请求)** + +重载扩展 → popup"开始抓取"选"全部" → 行为与现状一致(state 字段为 null 短路所有新逻辑);SW console 无报错。 + +- [ ] **Step 5: Commit** + +```bash +git add content.js +git commit -m "feat(ext): chatSince/chatUntil state plumbed through start" +``` + +--- + +### Task 4: content.js — 水位存储升级(只前进) + +**Files:** +- Modify: `content.js`(`getLastTopMsgId`/`setLastTopMsgId` 改造为 `getWatermark`/`trySetWatermark`) + +- [ ] **Step 1: 替换两个函数** + +删除 `getLastTopMsgId`/`setLastTopMsgId`,替换为(键名 `lastTopMsg_` 不变,clear 分支无需改): + +```js +// 增量水位(新格式 {id,sendTime};只前进不倒退,spec §4.5) +async function getWatermark(inquiryId) { + if (!inquiryId) return null; + try { + const r = await chrome.storage.local.get(['lastTopMsg_' + inquiryId]); + return parseWatermark(r && r['lastTopMsg_' + inquiryId]); + } catch (e) { return null; } +} +async function trySetWatermark(inquiryId, candId, candSendTime) { + if (!inquiryId) return; + const oldWm = await getWatermark(inquiryId); + if (!shouldWriteWatermark(oldWm, candId, candSendTime)) return; + try { + await chrome.storage.local.set({ ['lastTopMsg_' + inquiryId]: JSON.stringify({ id: candId, sendTime: candSendTime }) }); + } catch (e) { /* 持久化失败不致命 */ } +} +``` + +注:调用点改造在 Task 5(fetchChatByApi 内),本 Task 仅换函数定义会暂留旧调用 → **Task 4 与 Task 5 合并提交**(见 Task 5 Step 5)。 + +--- + +### Task 5: content.js — fetchChatByApi 主路径(spec 附录 A 落地) + +**Files:** +- Modify: `content.js`(`// 2) 循环 getOpMessages` 起 至 `return { messages: ... }` 止的整段) + +- [ ] **Step 1: 替换翻页整段(对照附录 A 伪代码逐行落地)** + +```js +// 增量水位(新格式;旧 string 兼容读取) +const lastTopWm = await getWatermark(inquiryId); +const lastTopId = lastTopWm ? lastTopWm.id : ''; +if (lastTopId) log(`聊天 API 增量模式: 上次水位 ${lastTopId}(sendTime=${lastTopWm && lastTopWm.sendTime}),遇此停止翻页`); +if (sinceMs != null || untilMs != null) { + log(`聊天范围: ${sinceMs != null ? new Date(sinceMs).toLocaleString() : '最早'} ~ ${untilMs != null ? new Date(untilMs).toLocaleString() : '至今'}`); +} +// 2) 循环 getOpMessages(spec 附录A:跳游标/probe/过滤/顺序/停止/水位) +const jump = untilMs != null && !state.cursorJumpUnsupported; +let cursor = jump ? untilMs + 1 : Date.now(); // +1ms:实测语义=严格小于,恰好覆盖 <= untilMs(附录B) +let probeRound = jump; +const raw = []; +const MAX_ROUNDS = state.chatLimit > 0 ? Math.ceil(state.chatLimit / 20) : 2000; +let candId = null, candSendTime = null; // 候选水位:逐轮取最大 sendTime(spec §4.5) +let stopReason = '轮数上限'; +for (let i = 0; i < MAX_ROUNDS; i++) { + if (state.phase === 'paused') return null; + if (!await waitUntilWorkTime('聊天翻页')) return null; + if (state.phase === 'paused') return null; + const json = await postForm( + `https://onetalk.alibaba.com/message/getOpMessages.htm?ctoken=${encodeURIComponent(ctoken)}&_tb_token_=${encodeURIComponent(tbToken)}`, + { params: JSON.stringify({ scene: '', timeSlide: { timeStamp: cursor, forward: false, pageSize: 20 }, secOwnerAccountId, secTradeId, secTargetAccountId, openRealTimeTranslation: true, targetAliId }), _csrf: xsrf } + ); + if (probeRound) { // 首轮探测:仅降级集走 fallback(真风控照抛,spec §4.3) + probeRound = false; + const kind = probeFailKind(json); + if (kind) { + state.cursorJumpUnsupported = false; // 先不设标志,由降级重试结果决定(实测精化) + log(`until 跳游标首轮 ${kind === 'empty' ? '空 list' : '结构异常'},降级 cursor=now 重试`); + cursor = Date.now(); + await randSleep(2000, 5000); + // 重试首轮非空才实锤"不支持"(真空时重试也空,不设标志,下个询盘继续尝试) + // —— 实现方式:本标志在下一轮循环体内设置(见下) + state._probeDowngraded = true; + continue; + } + } else { + if (!json || typeof json !== 'object') throw new Error('RISK:结构异常(非对象)'); + if (json.code !== undefined && Number(json.code) !== 200) throw new Error('RISK:结构异常(code=' + json.code + ')'); + if (json.data == null) throw new Error('RISK:结构异常(data=null)'); + } + const list = (json.data && json.data.list) || []; + if (!list.length) { + // 降级重试后的空 list = 真空(正常到顶);且若 _probeDowngraded 未实锤则清试探态 + if (state._probeDowngraded) { state._probeDowngraded = false; /* 重试也空 → 真空,不设 unsupported */ } + stopReason = '到顶(空列表)'; break; + } + if (state._probeDowngraded) { // 降级重试首批非空 → 跳游标确实漏数据,实锤不支持 + state._probeDowngraded = false; + state.cursorJumpUnsupported = true; saveState(); + } + for (const m of list) { // 候选水位逐轮 max(未过滤口径,spec §4.5) + if (m.sendTime != null && (candSendTime == null || m.sendTime > candSendTime)) { + candId = String(m.messageId); candSendTime = m.sendTime; + } + } + let hitLastTop = false; + for (const m of list) { // 顺序:水位命中查未过滤 list,再范围过滤(spec §4.3.2) + if (lastTopId && String(m.messageId) === lastTopId) { hitLastTop = true; break; } + if (inChatRange(m.sendTime, sinceMs, untilMs)) raw.push(m); + } + let minT = Infinity; + for (const m of list) if (m.sendTime != null && m.sendTime < minT) minT = m.sendTime; // null 跳过(防游标跳1970) + if (minT !== Infinity) cursor = minT; + if (raw.length % 400 < 20) log(`聊天 API 拉取中: ${raw.length} 条`); + if (hitLastTop) { stopReason = '增量命中'; break; } + if (sinceMs != null && cursor < sinceMs) { stopReason = '时间边界'; break; } + if (list.length < 20) { stopReason = '到顶'; break; } + await randSleep(2000, 5000); +} +log(`聊天 API 完成: ${raw.length} 条(停:${stopReason})`, 'success'); +await trySetWatermark(inquiryId, candId, candSendTime); // 只前进写入(spec §4.5) +``` + +同时删除原 `newestMsgIdThisRound` 相关整段与旧 `setLastTopMsgId` 调用;`lastTopMsgId` 读取段(原 `const lastTopMsgId = await getLastTopMsgId(...)`)由开头 `getWatermark` 段替代。**其后的 `return { messages: raw.map(...), meta };` 原样保留,不在替换范围内。** + +- [ ] **Step 2: 同步义务检查** + +对照 `scripts/verify-chat-range.mjs`:`inChatRange/parseWatermark/shouldWriteWatermark/probeFailKind` 函数体逐字一致(Task 3 已内联)。 + +- [ ] **Step 3: 逻辑回归(零请求)** + +Run: `node scripts/verify-chat-range.mjs && node scripts/verify-risk-detect.mjs && npx vitest run` +Expected: 36 passed / 10 passed / 16 passed,全部 exit 0 + +- [ ] **Step 4: 语法自检(浏览器加载)** + +重载扩展,SW console 无报错;popup"开始抓取(全部, limit=10)"跑一轮 —— 行为与现状一致(all → sinceMs/untilMs null,新分支全短路)。 + +- [ ] **Step 5: Commit(Task 4+5 合并)** + +```bash +git add content.js +git commit -m "feat(ext): time-bounded chat paging - cursor jump + probe fallback + forward-only watermark" +``` + +--- + +### Task 6: content.js — DOM 回退路径 + +**Files:** +- Modify: `content.js`(`loadAllChatHistory`) + +- [ ] **Step 1: 全量分支加时间探测停滚** + +`loadAllChatHistory` 全量分支(无 lastDomTime 时)滚动循环内,`await randSleep(2000, 6000)` 之后、计数逻辑之前加: + +```js +// DOM 时间边界:顶部向下找第一个带时间 span 的 wrapper(时间戳稀疏渲染),早于 since → 停滚(spec §4.4) +if (sinceMs != null) { + let oldestT = null; + for (const mw of document.querySelectorAll('.message-item-wrapper')) { + const te = mw.querySelector('.item-base-info'); + let hit = false; + if (te) for (const sp of te.querySelectorAll('span')) { + const m = sp.innerText.match(/(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})/); + if (m) { oldestT = new Date(m[1] + 'T' + m[2]).getTime(); hit = true; break; } + } + if (hit) break; + } + if (oldestT != null && oldestT < sinceMs) { log('DOM 时间边界,停止滚动'); break; } +} +``` + +- [ ] **Step 2: 提取后过滤(全量/增量两分支)** + +全量分支尾部(水位 maxT 计算保持用**未过滤** `all`): + +```js +const all = extractDomChatMessages(); +// ... maxT 用 all 计算并存 lastDomTime(不变,水位=见到的最新)... +const filtered = all.filter(m => inChatRange(m.sendTime ? Date.parse(m.sendTime) : null, sinceMs, untilMs)); +state._incDomMessages = filtered; +state._incDomFiltered = (sinceMs != null || untilMs != null); // 空数组也具决定性(见下) +``` + +**配套改消费点**(scrapeDetailData 内,原 `incMsgs && incMsgs.length` 处)—— 否则空 `filtered` 会穿透 `incMsgs.length` 保护、回退到**未过滤**的 `extractDomChatMessages()`,把范围外可见消息泄漏出去(违背 §4.4"0 条降级"): + +```js +const incMsgs = state._incDomMessages || null; +const incFiltered = !!state._incDomFiltered; // 空数组 + 过滤标记 → 空也有效(0条降级) +state._incDomMessages = null; state._incDomFiltered = false; +// ...(apiMsgs 判定不变)... +if (!msgs) msgs = (incMsgs && (incMsgs.length || incFiltered)) ? incMsgs : null; +``` + +增量分支: + +```js +const inc = all.filter(m => m.sendTime && m.sendTime > lastDomTime) + .filter(m => inChatRange(Date.parse(m.sendTime), sinceMs, untilMs)); +``` + +- [ ] **Step 3: 回归 + Commit** + +Run: `node scripts/verify-chat-range.mjs && npx vitest run` + +```bash +git add content.js +git commit -m "feat(ext): DOM fallback respects chat time range (probe-stop + filter)" +``` + +--- + +### Task 7: 全量回归(零请求) + +- [ ] **Step 1: 三套验证全绿** + +Run: `node scripts/verify-chat-range.mjs && node scripts/verify-risk-detect.mjs && npx vitest run` +Expected: 36 / 10 / 16 全 passed,exit 0 + +- [ ] **Step 2: 对照 spec 附录 A 逐行走查一遍 fetchChatByApi 实际代码(人工,15 分钟)** + +锚点清单:初始游标+1 / probe 只首轮 / PROBE 降级集不含 non-JSON / 水位命中未过滤 / 过滤 null guard / minT 跳 null / 停止优先级(增量>时间>到顶)/ stopReason 日志 / trySetWatermark 只前进 / randSleep 与 waitUntilWorkTime 原样。 + +--- + +### Task 8: 手动验证(⚠️ 风控纪律:limit=10、间隔≥2分钟、不批量) + +- [ ] **Step 1: 回归基线** — 选"全部"+limit=10,跑一轮,行为与旧版一致(注:all 模式**无**"聊天范围"日志,仅范围模式输出)。 +- [ ] **Step 2: 近1月** — 选"近 1 个月"+limit=10(挑含老客户的窗口),日志出现 `停:时间边界`,单询盘条数明显少于全量。两次运行间隔 ≥2 分钟。 +- [ ] **Step 3: 校验** — 选 custom,起>止 → popup 报错不发;止选明天 → 报错。(零请求) +- [ ] **Step 4: until 跳游标复测(附录B复测)** — custom 起=2个月前、止=1个月前 + limit=10:每个询盘**首轮** SW Network 里 `getOpMessages` 请求体 timeStamp ≈ 止日+1ms(注:无成功标志,后续询盘首轮**仍会**以止日起跳 —— 这是设计行为,不是重复试探 bug);首轮返回正常 → 本询盘后续轮次从 minT 正常翻页、无"降级 cursor=now"日志;若出现降级且降级重试首批非空 → 后续询盘直接 now 起跳(标志生效)。 +- [ ] **Step 5: 增量** — 同询盘重复 Step 2,第二次 `停:增量命中` 且推送无重复(后端 count 不变)。 +- [ ] **Step 6: 验证结果记录回 spec 附录 B 复测栏(如行为有出入,STOP 并回到 systematic-debugging)** +- [ ] **Step 7: 范围存活验证** — 范围模式任务中途暂停→恢复、扩展重载后:抓取沿用同一范围(state 持久化链路),日志仍显示同一"聊天范围"。 +- [ ] **Step 8: 水位升级确认** — 首轮范围抓取后旧水位(string)升级为 JSON,下轮增量正常命中(停:增量命中)。 +- [ ] **Step 9: 上线前开关** — 全部验证完成后把 `WORK_TIME_ENABLED` 改回 `true`(生产作息限制;当前为用户测试态 false)。 diff --git a/docs/superpowers/plans/2026-08-25-server-reinstall-backup.md b/docs/superpowers/plans/2026-08-25-server-reinstall-backup.md new file mode 100644 index 0000000..8bdc532 --- /dev/null +++ b/docs/superpowers/plans/2026-08-25-server-reinstall-backup.md @@ -0,0 +1,103 @@ +# 服务器重装备份方案(alibb 项目) + +> 2026-08-25。服务器 `219.157.255.213`(kt-W580G20)需要重装系统。 +> **范围:只备 alibb**(后端 + 前端 + alibaba 库 + 站点配置);服务器上其他业务(1Panel 全家桶、邮件、小红书等)与本备份无关。 +> 备份落盘:本机 `D:\alibb-server\backups\alibb-backup-20260825\`(D 盘重装不受影响)。 + +## 一、备份内容(已完成 ✅) + +服务器端打包于 `~/alibb-backup-20260825`,已 scp 拉回本机 `D:\alibb-server\backups\`,SHA256 全部校验通过。 + +| 文件 | 大小 | 内容 | +|---|---|---| +| `alibaba-db.sql` | 25M | alibaba 库全量 dump(`--single-transaction --no-tablespaces`):chat_messages 61872 行、customers 14794 行、inquiries 13570 行、users、_prisma_migrations | +| `alibb-server-code.tgz` | 998K | `~/alibb-server` 代码(backend+frontend),**docker-compose.yml 内含生产密钥**(JWT_SECRET/INGEST_TOKEN/ADMIN_PASSWORD) | +| `alibb-secrets.txt` | 296B | 生产密钥明文 | +| `alibb-site.conf` | 750B | OpenResty 站点 conf:`listen 23010`、`/api` → `http://backend:8080`、SPA fallback、`client_max_body_size 50m` | +| `alibb-frontend-dist.tgz` | 811K | 1Panel 站点目录(前端 dist;可由代码重 build,备作保险) | +| `alibb-backend-inspect.json` | 11K | 容器完整配置快照(env/网络/挂载) | +| `RESTORE-NOTES.md` | — | 恢复要点:MySQL 授权重建、网络别名、密钥清单 | + +dump 完整性:文件尾 `-- Dump completed on 2026-08-25 14:14:01`,5 张 CREATE TABLE 齐全。 + +## 二、关键事实(重装前记牢) + +1. **密钥全部沿用备包里的值** → 重装后扩展的 serverUrl + INGEST_TOKEN 不用改,老数据 JWT 也不失效。 +2. **mysqldump 不含用户授权** → 重装后必须手工执行(见 RESTORE-NOTES.md): + ```sql + CREATE USER 'alibaba'@'%' IDENTIFIED BY 'wecbni5zRRnZrMMh'; + GRANT ALL PRIVILEGES ON `alibaba`.* TO 'alibaba'@'%' WITH GRANT OPTION; + FLUSH PRIVILEGES; + ``` +3. backend 加入 **external 网络 `1panel-network`**(1Panel 装好后自动存在),容器别名 `alibb-backend`/`backend`——OpenResty 反代按容器名 `backend` 寻址,别名丢了 /api 就 502。 +4. 端口映射是 `127.0.0.1:8090:8080`(宿主 8090,比 7 月部署文档写的 8080 新,以备包为准)。 +5. 1Panel 的 MySQL root 密码重装后会变(装 1Panel 时新设),与业务无关;业务只依赖上面的 `alibaba` 账号。 + +## 三、恢复步骤(新系统从零) + +### 1. 基础环境 +```bash +# Ubuntu 24.04 + Docker(官方脚本)+ 1Panel(官方安装脚本,装时设新的面板/MySQL root 密码) +# 1Panel 应用商店装:OpenResty、MySQL 8.x +# 云安全组放行:23010/tcp(以及 1Panel 面板端口) +``` + +### 2. 恢复代码 +```bash +# 本机推备包上去(Windows OpenSSH) +/c/Windows/System32/OpenSSH/scp.exe -i 'C:\Users\30997\.ssh\alibb_deploy' -r 'D:\alibb-server\backups\alibb-backup-20260825' kt@219.157.255.213:~/ +ssh 上去: +tar xzf ~/alibb-backup-20260825/alibb-server-code.tgz -C ~ # → ~/alibb-server,compose 已是生产值,勿再跑 sed +``` + +### 3. 恢复数据库 +```bash +# 建库 + 授权 + 导入(注意库名 alibaba 与用户名 alibaba 同名) +sudo docker exec -i 1Panel-mysql-<新后缀> mysql -uroot -p'<新root密码>' <<'SQL' +CREATE DATABASE alibaba CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +CREATE USER 'alibaba'@'%' IDENTIFIED BY 'wecbni5zRRnZrMMh'; +GRANT ALL PRIVILEGES ON `alibaba`.* TO 'alibaba'@'%' WITH GRANT OPTION; +FLUSH PRIVILEGES; +SQL +sudo docker exec -i 1Panel-mysql-<新后缀> mysql -uroot -p'<新root密码>' alibaba < ~/alibb-backup-20260825/alibaba-db.sql +``` + +### 4. 起后端 +```bash +cd ~/alibb-server/backend +sudo docker compose build backend && sudo docker compose up -d # migrate 自动跑(应显示全部 already applied) +``` +注意:compose 里 networks 声明 `1panel-network` external——1Panel 装好后已存在,无需改。 + +### 5. 站点 + 前端 +- 1Panel 面板建网站「alibb」:域名/IP `219.157.255.213`,端口 `23010`,根目录默认站点 index +- 直接用备份 conf:把 `alibb-site.conf` 内容对到 `/opt/1panel/apps/openresty/openresty/conf/conf.d/alibb.conf`(经面板配伪静态+反代等价),**面板里重载 OpenResty** +- 前端二选一:解包 `alibb-frontend-dist.tgz` 复制进站点目录;或 `~/alibb-server/frontend` 里 docker node build 后复制(同 7 月文档 Step 4) + +### 6. 验证 +```bash +curl http://219.157.255.213:23010/ # 200 +curl http://219.157.255.213:23010/api/health # {"ok":true} +``` +浏览器登录 admin / ADMIN_PASSWORD(alibb-secrets.txt),确认询盘/客户/聊天记录数量与备份时一致(询盘 13570 / 客户 14794 / 消息 61872)。 + +### 7. 扩展侧 +无需改配置(serverUrl `http://219.157.255.213:23010` + 原 token 均不变)。Chrome 里点一次「测试连接」确认,再恢复日常抓取。 + +## 四、重装前 Checklist + +- [x] 服务器备包完成并拉回本机,SHA256 校验通过 +- [x] MySQL 授权语句已记录(RESTORE-NOTES.md) +- [ ] **再拷一份 `D:\alibb-server\backups\` 到外接硬盘或网盘**(D 盘虽不动,双保险,防重装误格全盘) +- [ ] 重装窗口内**停止扩展抓取**(推送会失败;若误抓,数据在本地,恢复后可重推;被风控的会话除外) +- [ ] 本机 `d:\alibb-server\backend` 有未提交改动(schema.prisma / ingest.service.ts / 新 migration)——与本次备份无关,但建议提交或至少保留现状 +- [x] 重装方式无要求:恢复完全依赖本机 D 盘备份,服务器**彻底重置也可接受**(其他业务数据是否保留与 alibb 无关) + +## 五、风险与说明 + +- **备份截止 2026-08-25 14:14**。之后新增的询盘/抓取数据不在 dump 里;重装前若又抓了数据,可重新导一次(命令见下)。 +- 服务器备包副本留在 `kt@219.157.255.213:~/alibb-backup-20260825/`,若重装保留 /home 可顺带幸存(不依赖)。 +- 增量重导命令: + ```bash + ssh … 'sudo docker exec -e MYSQL_PWD=wecbni5zRRnZrMMh 1Panel-mysql-sLiG mysqldump -ualibaba --single-transaction --no-tablespaces --routines --triggers --databases alibaba' > D:\alibb-server\backups\alibaba-db-final.sql + ``` diff --git a/docs/superpowers/plans/2026-08-25-server-restore-redeploy.md b/docs/superpowers/plans/2026-08-25-server-restore-redeploy.md new file mode 100644 index 0000000..4fbed5f --- /dev/null +++ b/docs/superpowers/plans/2026-08-25-server-restore-redeploy.md @@ -0,0 +1,164 @@ +# alibb 服务器重装后完整恢复部署方案(Runbook) + +> 2026-08-25。配合 [2026-08-25-server-reinstall-backup.md](2026-08-25-server-reinstall-backup.md) 使用。 +> 目标:新装系统后,把 alibb 恢复到与重装前**业务完全一致**(数据、密钥、访问方式、扩展零改动)。 +> 备份位置:本机 `D:\alibb-server\backups\`(SHA256 已校验)。 + +## 0. 恢复后与重装前的差异(预期内,不影响业务) + +| 项 | 重装前 | 重装后 | 影响 | +|---|---|---|---| +| MySQL 容器名 | `1Panel-mysql-sLiG` | `1Panel-mysql-<新随机后缀>` | compose 的 DATABASE_URL 要 sed 替换(见 §4.2) | +| 1Panel 面板/MySQL root 密码 | 旧值 | 装机新设 | 无(alibb 只用 `alibaba` 账号) | +| SSH host key / kt 用户 | 有 | 新建 | 本机要清 known_hosts + 重新传公钥(§1) | +| 公网 IP | 219.157.255.213 | 云盘重装一般保留 | **若 IP 变了**,要改 3 处(§8) | +| 数据 | — | 恢复到 2026-08-25 14:14 备份时点 | 之后的新数据需重装前重导(见备份方案 §五) | + +## 1. 前置:本机能连上新服务器 + +```bash +# 1) 清掉旧 host key(重装后会报 REMOTE HOST IDENTIFICATION HAS CHANGED) +/c/Windows/System32/OpenSSH/ssh.exe -i 'C:\Users\30997\.ssh\alibb_deploy' kt@219.157.255.213 'true' 2>&1 | head -3 +# 报错则执行: +/c/Windows/System32/OpenSSH/ssh-keygen.exe -R 219.157.255.213 + +# 2) 新系统建 kt 用户(sudo 免密)+ 传公钥(用 root/云控制台密码登录一次) +ssh root@219.157.255.213 'useradd -m -s /bin/bash kt && echo "kt ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/kt && mkdir -p /home/kt/.ssh' +/c/Windows/System32/OpenSSH/scp.exe 'C:\Users\30997\.ssh\alibb_deploy.pub' root@219.157.255.213:/home/kt/.ssh/authorized_keys +ssh root@219.157.255.213 'chown -R kt:kt /home/kt/.ssh && chmod 700 /home/kt/.ssh && chmod 600 /home/kt/.ssh/authorized_keys' +``` + +## 2. 系统基础(Ubuntu 24.04) + +```bash +# Docker(官方脚本) +curl -fsSL https://get.docker.com | sudo sh +# 镜像加速(照旧) +sudo tee /etc/docker/daemon.json <<'EOF' +{ "registry-mirrors": ["https://docker.1panel.live"] } +EOF +sudo systemctl restart docker +# 1Panel(官方脚本,装时设新面板密码) +curl -sSL https://resource.fit2cloud.com/1panel/package/quick_start.sh -o quick_start.sh && sudo bash quick_start.sh +``` + +## 3. 1Panel 应用安装 + +面板 → 应用商店,安装: + +- **OpenResty**(必须先于建站) +- **MySQL 5.7.44**(与重装前一致,优先选 5.7) + - ⚠️ 若商店已无 5.7 → 装 8.0 也可:dump 为 utf8mb4,兼容;仅 §4.1 建用户时多加一句 `IDENTIFIED WITH mysql_native_password BY '...'`(防老驱动连不上,Prisma/mysql2 两种都支持,加这句最稳) + +装好后记下 **MySQL 容器实际名称**(下称 ``,形如 `1Panel-mysql-XXXX`)和 root 密码。 + +## 4. 恢复数据库 + +### 4.1 传备包 + 建库授权导入 +```bash +# 本机推备包 +/c/Windows/System32/OpenSSH/scp.exe -i 'C:\Users\30997\.ssh\alibb_deploy' -r 'D:\alibb-server\backups' kt@219.157.255.213:~/ +``` +```bash +# 服务器上(替换 和 ) +M=; R= +sudo docker exec -i $M mysql -uroot -p"$R" <<'SQL' +CREATE DATABASE alibaba CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +CREATE USER 'alibaba'@'%' IDENTIFIED BY 'wecbni5zRRnZrMMh'; +GRANT ALL PRIVILEGES ON `alibaba`.* TO 'alibaba'@'%' WITH GRANT OPTION; +FLUSH PRIVILEGES; +SQL +sudo docker exec -i $M mysql -uroot -p"$R" alibaba < ~/backups/alibaba-db.sql +``` +> 注:mysqldump 不含用户授权,以上 CREATE USER/GRANT 来自重装前实机导出(RESTORE-NOTES.md)。 + +### 4.2 验证行数(必须与下表一致,否则停) +```bash +sudo docker exec -i $M mysql -ualibaba -pwecbni5zRRnZrMMh alibaba -e \ +"SELECT (SELECT COUNT(*) FROM inquiries) inquiries,(SELECT COUNT(*) FROM customers) customers,(SELECT COUNT(*) FROM chat_messages) chat_messages;" +``` +期望:**inquiries 13570 / customers 14794 / chat_messages 61872**(若重装前重导过,以最终 dump 为准)。 + +## 5. 恢复后端 + +```bash +# 解包代码(~/alibb-server,compose 已是生产值,严禁再跑 sed 替换 dev 值) +tar xzf ~/backups/alibb-server-code.tgz -C ~ + +# ★ 关键一步:DATABASE_URL 里的旧容器名替换为新容器名 +cd ~/alibb-server/backend +M= +sed -i "s/1Panel-mysql-sLiG/$M/g" docker-compose.yml +grep DATABASE_URL docker-compose.yml # 人工确认已指向新容器名 + +# build + 起(首次 migrate 会显示全部 already applied) +sudo docker compose build backend && sudo docker compose up -d +sudo docker compose logs backend | tail -20 +``` + +- compose 声明 `1panel-network` 为 external——1Panel 装好后已存在,无需改。 +- 密钥(JWT_SECRET/INGEST_TOKEN/ADMIN_PASSWORD/ADMIN_USERNAME)已内嵌在 compose 里,保持不变 → **扩展 token、登录账号全部沿用**。 + +## 6. 站点 + 前端 + +1. 1Panel → 网站 → 创建网站:名称 `alibb`,域名 `219.157.255.213`,端口 `23010`,根目录用默认站点 index 路径。 +2. 站点配置替换为备份内容(面板「配置文件」里改,或直接编辑 conf 后在面板重载 OpenResty): + ```bash + sudo cp ~/backups/alibb-site.conf /opt/1panel/apps/openresty/openresty/conf/conf.d/alibb.conf + # 面板 → OpenResty → 重载 + ``` + > conf 要点:`listen 23010`、root 指向站点 index、`/api` → `http://backend:8080`、`try_files … /index.html`、`client_max_body_size 50m`。若 1Panel 生成的路径/结构与 conf 里写的不一致,以面板生成的 server 块为骨架,把上面四项并进去。 +3. 前端文件(二选一): + ```bash + # 方案 A(快,直接用备包):解包即完整 dist + sudo tar xzf ~/backups/alibb-frontend-dist.tgz -C /opt/1panel/apps/openresty/openresty/www/sites/alibb/ + sudo chmod -R a+rX /opt/1panel/apps/openresty/openresty/www/sites/alibb/index + # 方案 B(干净,代码重 build):同 7 月部署文档 Step 4(docker node:22-alpine npm ci && npm run build 后复制 dist) + ``` +4. 云安全组放行 **TCP 23010** 入方向。 + +## 7. 端到端验证(全部通过才算恢复完成) + +```bash +curl http://219.157.255.213:23010/ # 200,返回 index.html +curl http://219.157.255.213:23010/inquiries # 200(SPA fallback) +curl http://219.157.255.213:23010/api/health # {"ok":true} +``` +- [ ] 浏览器开 http://219.157.255.213:23010 → admin / ADMIN_PASSWORD(compose 里的原值)登录成功 +- [ ] 询盘/客户/聊天页数据齐全,条数与 §4.2 一致 +- [ ] Chrome 扩展 popup:**配置不变**,点「测试连接」通过 → 恢复日常抓取 + +## 8. 若公网 IP 变了(默认不变,仅备用) + +需要同步改 3 处: +1. 扩展 popup 的 serverUrl → `http://<新IP>:23010` +2. 站点 conf 的 `server_name` → 新 IP +3. 云安全组规则跟随新 IP/网卡 + +## 9. 收尾 + +- [ ] 把 `~/backups/alibb-secrets.txt` 复制回 `~/alibb-secrets.txt`(惯例位置),`chmod 600` +- [ ] 服务器上留一份 `~/alibb-backup-20260825/`(把 ~/backups 里文件拷过去),本机 `D:\alibb-server\backups\` 永久保留 +- [ ] 更新 [2026-07-15-deploy-test-env.md](2026-07-15-deploy-test-env.md) 里的 MySQL 容器名/端口等(或注明以本 runbook 为准) + +## 10. 故障速查 + +| 症状 | 原因 → 处理 | +|---|---| +| `/api` 502 | ① backend 没起来:`docker compose logs backend`;② OpenResty 与 backend 不在同网络/别名丢失:确认 compose networks 段 `1panel-network: external: true` 且容器别名含 `backend`;③ OpenResty 容器未加入 `1panel-network` | +| 后端起不来,连接拒绝 | DATABASE_URL 容器名没替换对(§5),`grep DATABASE_URL` 核对 | +| 登录 401 | ADMIN_USERNAME/ADMIN_PASSWORD 与旧值不一致——用备包 compose,别用本地 dev compose 覆盖 | +| 扩展测试连接失败 | INGEST_TOKEN 变了(同上)或 23010 未放行 | +| MySQL 8.0 下用户连不上 | `ALTER USER 'alibaba'@'%' IDENTIFIED WITH mysql_native_password BY 'wecbni5zRRnZrMMh';` | +| migrate 报表不存在 | dump 导入的库不对/没选 alibaba 库,重做 §4 | + +## 11. 实际恢复记录(2026-08-27,已完成 ✅) + +新环境与原方案的差异(下次重装直接按这个来): + +1. **MySQL 8.4.11**(容器 `1Panel-mysql-YL0F`),非 5.7 → dump 直接导入零问题;`alibaba@%` 用户保持默认 caching_sha2_password,Prisma 连接正常,无需切 native 插件 +2. **OpenResty 1.31 是 host 网络模式** → 容器名 DNS 不可用,`/api` 反代**必须** `proxy_pass http://127.0.0.1:8090`(backend 的宿主映射),不能用 `http://backend:8080` +3. **1Panel 新路径 `/opt/1panel/www`**(挂载到容器 `/www`,与旧 conf 完全兼容):站点在 `www/sites/alibb/`,conf 在 `www/conf.d/alibb.conf` +4. 站点目录//conf 为手工创建,**1Panel 面板里不显示该站点**(如需面板管理,后续在面板建同名站再合并 conf) +5. 精确行数(COUNT(*)):inquiries 14817 / customers 15402 / chat_messages 68268(高于备份时 information_schema 估算值,属正常) +6. 验证结果:外网 `http://219.157.255.213:23010` index 200 / SPA 200 / api/health `{"ok":true}`,migrate 无缺项,安全组 23010 已放行 diff --git a/docs/superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md b/docs/superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md new file mode 100644 index 0000000..5da5cb4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-alibaba-inquiry-backend-design.md @@ -0,0 +1,310 @@ +# 阿里询盘管理后台 设计文档 + +- 日期: 2026-07-13(修订 v3:纳入二次评审发现) +- 状态: 修订完成,待用户审查 +- 关联: 现有 Chrome 插件位于 `d:\alibb\` +- 修订说明: v2 修正了 spec review 发现的 4 个致命问题(host_permissions 运行时不可改 / messageId 丢失 / 队列存储布局 / 淘汰丢数据)与 8 个重要问题(询盘时间源、批量写入、alarms 周期、时区、索引、HTTP 推送、behavior 类型、外键可空)。 +- v3 修正二次评审发现:① **早返回逻辑反转**(失败批次永不重试 → inquiries 表新增 `has_failed_batches` 列 + `messages_count` 改存实际插入数);② **IndexedDB 写入上下文**(content script 的 IDB 属页面源,须由 SW 落盘)+ 双存储一致性写入顺序;③ **MySQL 8 默认 max_allowed_packet 是 64MB 非 4MB**;④ 批量行数 1000/500 统一为 **500**;⑤ direction ENUM 补 `'system'`;⑥ §8.6 补 behavior null 强制转换 / country 来源替换 / translation 双重 JSON.parse。 + +## 1. 概述 + +### 1.1 目标 +把现有 Chrome 插件(抓取阿里国际站询盘/客户/聊天)的「本地 CSV 导出」升级为完整链路: + +> 插件自动推送 → 自建后端 → MySQL → 管理后台(查看 / 筛选 / 导出 / 统计) + +### 1.2 使用模式(已确认) +- **主账号统一抓取**:登录阿里主管理员账号(可看到所有业务员的询盘),**一个人操作插件**一次性抓取全部询盘。 +- **后台单管理员**:现阶段就一个管理员账号,统览全部数据。 +- 询盘记录里的 `manager`(负责人)字段天然标识业务员,后台按此字段统计/筛选,**不需要业务员各自登录或各自推送**。 +- **数据时效 = 上次手动抓取的快照**:插件靠用户在阿里页点「开始抓取」触发,**不是实时同步**;后台看到的是最近一次抓取的数据。后续可考虑定时自动抓(需浏览器保持阿里页打开)。 + +### 1.3 MVP 范围 +**纳入**:插件自动推送;后端入库(三实体幂等);管理后台(列表/详情/导出/统计)。 +**不纳入(后续)**:多业务员登录/权限;跟进管理/CRM;客户合并/标签;聊天全文搜索;复杂报表;增量推送;**后台数据编辑/删除(MVP 只读,错误数据需重抓或直接改库)**。 + +## 2. 背景:现有插件能力 + +`d:\alibb\` 的 Chrome MV3 插件已实现并验证:列表/详情/行为数据抓取;聊天 API 直拉(`querySummary.htm` → `getOpMessages.htm` 游标循环,单询盘实测 8000+ 条);chrome.storage 状态持久化。 + +**本方案对抓取主流程增量扩展,但必须改动消息映射**: +- 现有 `apiMsgToChat()` 只产出 `{sender, time, original, translation, direction}`,**丢弃了 messageId 和毫秒时间戳**。 +- 本方案要求扩展为:`{messageId, sendTime(ISO UTC), sender, content, direction, translation}`,并从 `querySummary.htm` 响应顺带提取询盘权威时间 `gmtCreate`/`gmtModified`、买家国家等。 +- 因此 §8 的插件改造**不是纯增量**,包含对 `apiMsgToChat` 与 `fetchChatByApi` 的字段扩展。 + +## 3. 需求 + +### 3.1 功能需求 +| ID | 需求 | +|---|---| +| F1 | 插件每抓完一个询盘(主体 + 全量聊天)→ 自动 POST 到后端 | +| F2 | 后端按 客户/询盘/聊天 三表存储,重复推送幂等无副作用 | +| F3 | 后台列表:分页 + 筛选(国家/业务员/状态/时间范围)+ 关键词搜索(询价单号/买家名/公司)+ 排序 | +| F4 | 后台详情:客户基本信息 + 行为数据卡片 + 完整聊天(虚拟滚动,左右气泡按方向) | +| F5 | 导出:按当前筛选导出 CSV(**仅列表字段,不含聊天**;聊天在详情页单独导出) | +| F6 | 统计:总数、按国家 Top10、按业务员分布、按月趋势(基于询盘 gmtCreate) | + +### 3.2 非功能需求 +| ID | 需求 | +|---|---| +| NF1 可靠 | 推送不丢(队列持久化 + 重试 + 淘汰策略不丢 pending);入库幂等(唯一键 upsert) | +| NF2 简单好维护 | TypeScript;Prisma schema 单一数据定义源;前后端共享类型 | +| NF3 稳定 | 单操作者低并发;Docker 部署;MySQL 每日备份 | +| NF4 安全 | 正式 HTTPS;插件 Bearer token;管理员 JWT(带过期 + 刷新) | + +## 4. 技术栈 + +| 层 | 选型 | 理由 | +|---|---|---| +| 后端 | **Node.js 18 + TypeScript + Express + Prisma** | 和插件同语言;Prisma schema 直观;TS 类型安全 | +| 数据库 | **MySQL 8** | 事务 + 唯一约束保证幂等 | +| 前端 | **Vue3 + TS + Element Plus + Vite + Pinia + Vue Router** | 管理后台组件成熟 | +| 部署 | **Docker Compose + Nginx** | 一键起 | +| 插件 | 现有 MV3 + 推送模块(chrome.alarms + 队列 + optional permissions) | 见 §8 | + +**决策记录:为何 Node 不选 Python**:用户优先级是"简单 + 好维护 + 稳定 + 好实现(主导搭建)",Node 全栈与插件同语言、可复用数据转换逻辑、搭建排错最快;Python 数据生态优势对 MVP 简单统计是过度设计,且引入前后端两语言复杂度。后续复杂分析可加 Python 微服务。 + +## 5. 架构与数据流 + +### 5.1 组件拓扑 +``` +[操作者电脑 · 登录阿里主账号] + Chrome 插件 + ① 抓取(保留) → 扩展消息映射(messageId/sendTime ISO/gmtCreate) + ② 每抓完一条 → 经 SW 写入【待推队列】(payload 存扩展 IndexedDB,queue_index 存 chrome.storage.local;content script 不能直接写扩展 IDB,见 §8.2) + ③ chrome.alarms(1 分钟)唤醒 SW → 消费队列 → POST /api/ingest + ↓ 失败指数退避重试,成功才出队 +[Linux 服务器 · Docker Compose] + Nginx(client_max_body_size 50m)─┬─ /api/* → Backend + └─ /* → Frontend 静态 + MySQL 8(数据卷 + 每日 mysqldump) + ↓ 幂等写入(唯一键: inquiry_id / customer_id / message_id) +[管理员浏览器] 登录(JWT)→ 列表/详情/导出/统计 +``` + +### 5.2 数据流时序(/api/ingest 处理) +1. zod 校验 payload 结构。 +2. **幂等早返回**:若 `inquiry_id` 已存在 **且 `has_failed_batches=false`** 且 `messages_count` + `last_send_time` 均未变 → 直接 200 返回,跳过消息批写(避免无效重写几千条)。`has_failed_batches` 必须参与判断,否则上次失败的批次会被早返回永远跳过(详见 R6)。 +3. 一个事务 upsert customer + inquiry(customer_id 可空时跳过 customer)。 +4. 聊天消息用**原生批量** `INSERT INTO chat_messages (...) VALUES (...),(...)... ON DUPLICATE KEY UPDATE content=VALUES(content), send_time=VALUES(send_time), ...`,**分批**(每批 500 行,见 §10)执行,每批独立事务;失败批次记入失败列表,不回滚整条询盘。累计 `actualInserted` = 各批实际落盘行数之和。 +5. 更新 inquiry:`messages_count = actualInserted`(实际插入数,**非 payload 长度**)、`last_send_time = max(sendTime)`、`has_failed_batches = (失败批次>0)`。用实际插入数才能让"失败重推时 count 不一致 → 不早返回 → 失败批次自然补写"(v2 用 payload 长度是逻辑反转)。 +6. 返回 `{ok, inserted: actualInserted, failedBatches}`。 + +## 6. 数据模型 + +四张表。关系:customers 1→多 inquiries 1→多 chat_messages。 + +### 6.1 customers(客户) +| 字段 | 类型 | 说明 | +|---|---|---| +| id | BIGINT PK auto | | +| customer_id | VARCHAR(64) **UNIQUE** | 阿里 customerId(加密串) | +| name | VARCHAR(128) | | +| country | VARCHAR(64) **INDEX** | 国家(统计/筛选用) | +| email / company | VARCHAR | | +| register_time | DATETIME NULL | UTC,可空 | +| avatar_url | VARCHAR(512) | | +| buyer_tags / buyer_features | VARCHAR(255) | 逗号分隔 | +| behavior_product_views / valid_inquiries / valid_rfq / login_days / spam_inquiries | INT **NULL** | 可空(抓不到时 null,不存 0 以区分"无数据"与"真为 0") | +| behavior_top_industry | VARCHAR(255) | | +| created_at / updated_at | DATETIME | UTC | + +### 6.2 inquiries(询盘) +| 字段 | 类型 | 说明 | +|---|---|---| +| id | BIGINT PK auto | | +| inquiry_id | VARCHAR(32) **UNIQUE** | 询价单号 | +| customer_id | VARCHAR(64) **NULL** INDEX | FK→customers.customer_id,**可空**(抓不到 customerId 时留空) | +| manager | VARCHAR(128) **INDEX** | 业务员名 | +| create_time | DATETIME | UTC,源自 API `gmtCreate` | +| update_time | DATETIME | UTC,源自 API `gmtModified`(缺失回落 gmtCreate) | +| status | VARCHAR(64) | | +| msg_preview | VARCHAR(512) | | +| is_flagged | BOOLEAN | | +| messages_count | INT DEFAULT 0 | 冗余,供幂等早返回 | +| last_send_time | DATETIME NULL | 冗余,供幂等早返回 | +| has_failed_batches | BOOLEAN DEFAULT FALSE | 上次推送是否有失败批次;为真则关闭早返回、强制重试(v3 新增) | +| created_at / updated_at | DATETIME | UTC | + +### 6.3 chat_messages(聊天明细) +| 字段 | 类型 | 说明 | +|---|---|---| +| id | BIGINT PK auto | | +| message_id | VARCHAR(64) **UNIQUE** | 阿里 messageId(幂等键,全局唯一) | +| inquiry_id | VARCHAR(32) **INDEX** FK→inquiries.inquiry_id | | +| sender | VARCHAR(128) | | +| direction | ENUM('incoming','outgoing','system') | 插件 apiMsgToChat 在 messageType 非 send/rec 时兜底产出 'system'(样本暂未出现,保留以防插入失败) | +| send_time | DATETIME | UTC,源自毫秒戳 | +| content | **MEDIUMTEXT** | 原文(已解码实体)。卡片消息体量待实测确认(样本中为 URL ~500 字符);MEDIUMTEXT 上限 16MB 留足余量 | +| translation | TEXT | 翻译,从 `extInfo.icbuData.extend.extParams.inputTranslateInfo.translate` 提取(可空) | +| created_at | DATETIME | UTC | + +> **索引**:`UNIQUE(message_id)`、`INDEX(inquiry_id, send_time, message_id)`(游标分页复合键)。 + +### 6.4 users(管理员账号) +| 字段 | 类型 | +|---|---| +| id PK / username UNIQUE / password_hash (bcrypt) / created_at | + +> **bootstrap**:后端启动时若 users 表空,根据 `ADMIN_USERNAME` / `ADMIN_PASSWORD` 环境变量 seed 一条记录。推送 token 写 `.env`(`INGEST_TOKEN`),不建表。 + +### 6.5 时间来源与时区(修订) +- **询盘 create_time / update_time**:用 `querySummary.htm` 返回的 `gmtCreate` / `gmtModified` 毫秒戳 → `new Date(ms).toISOString()`。**不再用"最早消息兜底"**(语义错:询盘创建 ≠ 首条消息)。列表页那个只有时分的文本时间不入库,仅必要时回退展示。 +- **消息 send_time**:用 API 毫秒戳 → ISO UTC。 +- 所有 DATETIME 存 UTC;前端用 dayjs 转本地时区展示。 + +## 7. API 设计 + +基地址 `/api`,JSON。 + +| 方法 | 路径 | 鉴权 | 说明 | +|---|---|---|---| +| POST | `/api/ingest` | Bearer token | 插件推送。zod 校验。限流 60/min/token。幂等早返回(见 §5.2) | +| POST | `/api/auth/login` | - | `{username,password}` → `{token, expiresIn}` | +| POST | `/api/auth/refresh` | JWT | 刷新 token | +| GET | `/api/inquiries` | JWT | 列表,query: page,size,country,manager,status,dateFrom,dateTo,q(搜 inquiry_id/msg_preview/customer.name),sort | +| GET | `/api/inquiries/:id` | JWT | 详情 + 客户 + 行为 | +| GET | `/api/inquiries/:id/messages` | JWT | **游标分页**:query `afterSendTime` + `afterMessageId`(默认空=从最早)、size;`WHERE inquiry_id=? AND (send_time, message_id) > (?, ?) ORDER BY send_time, message_id LIMIT ?`(复合游标,避免同时间戳消息丢失) | +| GET | `/api/inquiries/:id/messages/export` | JWT | 单询盘聊天导出(stream) | +| GET | `/api/stats/overview` | JWT | 总数/国家 Top10/业务员/月趋势 | +| GET | `/api/export` | JWT | **仅列表字段**(不含聊天),按筛选 stream 导出 | + +### 7.1 `/api/ingest` payload 结构 +```jsonc +{ + "inquiry": { + "inquiryId": "13776617920", + "manager": "cara Hu", + "createTime": "2026-03-13T10:11:50Z", // 来自 gmtCreate, ISO UTC + "updateTime": "2026-07-13T10:11:50Z", // 来自 gmtModified + "status": "洽谈中", "msgPreview": "...", "isFlagged": false + }, + "customer": { // customerId 为空时整个 customer 省略 + "customerId": "259b4e6f...", "name": "Carlos Loayza Marin", + "country": "PE", "email": null, "company": "World Shop", + "registerTime": "2025-06-23T00:00:00Z", + "avatarUrl": "...", "buyerTags": "TM沟通", "buyerFeatures": "...", + "behavior": { "productViews": null, "validInquiries": null, "validRfq": 39 } // 捕不到为 null + }, + "messages": [ + { "messageId": "4198482017852", "sendTime": "2026-07-13T02:11:50Z", + "sender": "cara Hu", "direction": "outgoing", "content": "hello...", + "translation": "你好..." } + ] +} +``` +- 空字符串数字字段在插件侧 coerce 为 null;后端 zod 接受 `number | null`。 + +## 8. 插件改造(在 `d:\alibb\` 基础上) + +### 8.1 配置页 +popup 增「后端设置」:服务器 URL、Token、启用开关。存 chrome.storage.local。 +**首次保存 URL 时**,调用 `chrome.permissions.request({origins:[serverUrl]})`(需在用户点击手势中触发,会弹 Chrome 授权框)——因为 host_permissions 运行时不可改,见 §8.5。 + +### 8.2 待推队列(修订:IndexedDB 存 payload + 不丢 pending) +- **payload 默认存扩展 IndexedDB**(`idb` 库),每条一个 record(key=id);chrome.storage.local 只存轻量 `queue_index`(有序 id 列表)+ 每项元数据 `{id, retries, nextRetry, status}`。避免 chrome.storage.local 在 GB 级的读写性能问题。 +- **写入上下文(关键,v3)**:content script 的 `indexedDB` 是**页面源**(`message.alibaba.com`)的,不是扩展的;只有 service worker / popup / options 才能访问扩展自己的 IndexedDB。因此正确链路是:**content.js 抓完一条 → `chrome.runtime.sendMessage` 把 payload 传给 SW → SW 落盘到扩展 IndexedDB**。绝不能让 content.js 直接写 IDB(会写到页面的库里)。 +- **双存储一致性(v3)**:IndexedDB 与 chrome.storage 之间无跨库事务。写入顺序固定为**先写 IDB payload,再追加 queue_index**(保证 index 永不引用不存在的 payload);删除反之(先标 success → 删 IDB → 移 index)。SW 启动时跑一次 GC,清理 IDB 中无 index 引用的孤儿 payload。 +- manifest 声明 `"unlimited_storage"`(IndexedDB 容量也受其约束)。 +- **不丢未推数据**:推送成功 → 立即从 IndexedDB + queue_index 删除该队列项;`pending/failed` 永不淘汰。触顶(默认 500 条 pending)→ **暂停抓取**(content.js 每抓一条前向 SW 查队列深度)+ popup 红字提示"队列拥堵,请检查后端"。 + +### 8.3 推送 worker(修订:alarms 1 分钟) +- 用 **`chrome.alarms`**(`periodInMinutes: 1`),**不用 setInterval**(MV3 SW 会休眠)。 +- 注:打包/正式版 alarms 最短 1 分钟(开发版可临时 30s)。 +- alarm 触发 → 取最多 N(默认 3)条 pending 且到 nextRetry 的项 → POST → 成功则删除该队列项 / 失败 `retries++`、`nextRetry = now + min(2^retries, 3600)s`,超 8 次标 failed 保留。 +- **吞吐量(v3)**:1 分钟 × 3 条 = 180 条/小时 ≈ 4320 条/天。全量几千条要好几天才推完,这是异步后蹲、与"慢抓"主题一致;**抓完 ≠ 入库**,popup 的"已推/待推"计数要让使用者意识到。瓶颈在 worker 的 3/min,服务器限流 60/min(§7)永远用不到;日后想加速可提高每 tick 的 N。 + +### 8.4 状态展示 +popup 显示「已推 N / 待推 M / 失败 K」,失败可查看/重试。 + +### 8.5 manifest 改动(修订:optional permissions) +```jsonc +{ + "permissions": ["storage", "activeTab", "alarms", "unlimitedStorage"], + "optional_host_permissions": ["http://*/*", "https://*/*"], + "host_permissions": [ + "*://message.alibaba.com/*", "*://alicrm.alibaba.com/*", "*://onetalk.alibaba.com/*" + ] +} +``` +- 后端服务器地址**不写死在 host_permissions**(运行时不可改,且占位符非法会拒载)。 +- 用 `optional_host_permissions: ["http://*/*","https://*/*"]` + 用户在配置页通过 `chrome.permissions.request({origins:[serverUrl]})` **运行时授权**具体后端地址。 +- 替代:若插件私有/不上架,可直接 `host_permissions: ["*://*/*"]` 一劳永逸(但上架审核会问)。 + +### 8.6 抓取侧字段扩展(修订) +- `fetchChatByApi()` 在调 `querySummary.htm` 后,**提取 `gmtCreate`/`gmtModified`/`country` 等字段**并随 payload 返回。**country 改用 `querySummary.contact.countryCode`**(样本中 `"countryCode":"PE"`,比 DOM 的 `detailCountry` 可靠),是**替换**而非新增字段。 +- `apiMsgToChat()` 扩展产出 `{messageId: String(m.messageId), sendTime: new Date(m.sendTime).toISOString(), sender, content, direction, translation}`,其中 translation 从 `m.extInfo.icbuData.extend.extParams.inputTranslateInfo.translate` 解析(失败留空)。**注意双重 JSON.parse(v3)**:`extend` 本身是 JSON 字符串,`extParams.inputTranslateInfo` 又是嵌套 JSON 字符串,需 `JSON.parse(JSON.parse(extend).extParams.inputTranslateInfo).translate` 两层解析,直接对象访问会拿到 undefined。 +- **behavior 数字字段强制转 null(v3)**:`productViews/validInquiries/validRfq/loginDays/spamInquiries` 在 content.js 现为字符串('' 或数字串),payload 序列化时空串与抓不到的值都要 coerce 为 `null`,匹配后端 zod 的 `number | null`。 + +## 9. 后台前端 + +| 路由 | 页面 | 要点 | +|---|---|---| +| `/login` | 登录 | | +| `/inquiries` | 列表 | Element Plus 表格 + 筛选栏(国家/业务员/状态/时间)+ 搜索 + 排序 + 分页 + 导出按钮 | +| `/inquiries/:id` | 详情 | 客户信息+行为卡片 + 完整聊天(虚拟滚动,游标加载) | +| `/stats` | 统计 | 数字卡片 + 国家 Top10 饼图 + 业务员柱状图 + 月趋势折线(ECharts) | + +性能:聊天用虚拟滚动(vue-virtual-scroller);列表分页默认 20/页。 + +## 10. 可靠性与安全 + +| # | 措施 | +|---|---| +| 幂等 | 唯一键 upsert + 早返回(inquiry 存在 **且 has_failed_batches=false** 且 count/lastSendTime 未变则跳过) | +| 队列不丢 | 独立 key 持久化 + 只淘汰 success + 触顶暂停抓取 | +| 批量写入 | 原生 `INSERT ON DUPLICATE KEY` 分批 500 行/批(见 §5.2),独立事务,失败批次隔离 | +| 鉴权 | 插件 Bearer token(env);管理员 JWT(access 短期 + refresh) | +| 密码 | bcrypt | +| CORS | `/api/ingest` 用 token 鉴权,`Access-Control-Allow-Origin: *` 不做 origin 校验;后台 API 限前端域名 | +| XSS | Vue 默认转义,禁用 v-html 渲染用户数据 | +| body limit | Express `limit:50mb`;**nginx `client_max_body_size 50m`** | +| MySQL | `max_allowed_packet` 设为 128MB(**MySQL 8 服务端默认 64MB,非 4MB**;4MB 是老版本/客户端库默认值,易误传);部署前 `SHOW VARIABLES LIKE 'max_allowed_packet'` 实测确认。单批 500 行(含 MEDIUMTEXT 时 500 行 × 极端 64KB ≈ 32MB,留余量) | +| 备份 | 每日 mysqldump → gzip,保留 14 天,异地复制 | + +## 11. 部署 + +### 11.1 docker-compose(结构) +```yaml +services: + mysql: # mysql:8, 卷, healthcheck, max_allowed_packet 调大 + backend: # Node API, depends_on mysql, .env + nginx: # 反代 + 静态 + client_max_body_size 50m; 正式加 HTTPS + cron: # 备份 sidecar(或宿主 crontab),每日 mysqldump +``` + +### 11.2 测试环境(无域名) +- 公网 IP + 端口,后端 HTTP。 +- **扩展 SW 的 fetch 起源于 `chrome-extension://` 上下文,不受网页 mixed-content 规则约束**,只要 optional_host_permissions 授权了该 HTTP URL,推送正常。 +- **私网 IP(192.168/10.x)注意**:Chrome 的 Private Network Access 会要求预检;优先用公网 IP 测试。 + +### 11.3 正式环境(有域名) +- 域名 + Nginx + Let's Encrypt(自动续期)+ 全程 HTTPS。 + +## 12. 项目结构 + +``` +d:\alibb\ # 现有插件(增量改造:推送模块 + 字段扩展) +d:\alibb-server\ + backend\ prisma\schema.prisma + src\{routes,services,middleware,utils} + Dockerfile + frontend\ Vue3 工程 + Dockerfile + docker-compose.yml + .env.example (DATABASE_URL / INGEST_TOKEN / JWT_SECRET / JWT_REFRESH_SECRET / ADMIN_USERNAME / ADMIN_PASSWORD) +``` + +## 13. 分期 + +**MVP**:后端骨架(Express+Prisma+MySQL)+ `/api/ingest`(校验+幂等+批量)+ auth/JWT + 列表/详情/消息/统计/导出 API + 前端四件套 + 插件推送模块 + Docker + 测试部署 + 备份。 +**后续**:增量推送、聊天全文搜索、跟进管理、客户合并/标签、复杂报表、多管理员、存量 CSV 导入。 + +## 14. 风险与待确认 + +- **R1 payload 体量**:单询盘几千条约 2-5MB;body limit 50MB + nginx 50m + max_allowed_packet 64MB。>1万条极端询盘 MVP 后改分批推送。 +- **R2 增量**:MVP 全量重推 + 早返回优化(未变则跳过);日抓量大后做真增量(插件记 last sendTime)。 +- **R3 manager 重名**:字符串统计,同名合并;后续加 employees 表。 +- **R4 测试环境网络**:公网 IP+HTTP 推送无混合内容问题(扩展 fetch 独立上下文);私网 IP 需注意 Private Network Access 预检。已删除"自签 HTTPS"复杂方案。 +- **R5 customerId 缺失**:customer_id 可空,无 customerId 的询盘只存 inquiry+messages,customer 留空。 +- **R6 早返回的边界**:幂等早返回条件为 `has_failed_batches=false` 且 `messages_count + last_send_time` 未变。 + - **失败批次重试(v3 修正)**:早返回已加入 `has_failed_batches`——上次有批次失败时强制不走早返回,确保失败批次被补写。inquiries 表新增 `has_failed_batches` 列;`messages_count` 存实际插入数(payload 长度会让重推 count 恒定一致,反而触发早返回——这是 v2 的逻辑反转)。 + - **内容修订**:同 messageId 内容(如 translation)被上游修订时,早返回仍会跳过更新。MVP 接受(重推主场景是新增消息)。 +- **R7 首次全量回填耗时**:首次把现有几千询盘全抓入库,按"每询盘几千条聊天 × API 拉取 + 推送"估算需 **数小时**,且**浏览器必须一直打开阿里页**不能关。要求插件支持**分批抓取 + 可中断续抓**(基于 queue/inquiryId 进度持久化,中断后重开继续,而非重头来)。日常增量抓取不受此影响。 diff --git a/docs/superpowers/specs/2026-08-25-chat-time-range-design.md b/docs/superpowers/specs/2026-08-25-chat-time-range-design.md new file mode 100644 index 0000000..f63dc8b --- /dev/null +++ b/docs/superpowers/specs/2026-08-25-chat-time-range-design.md @@ -0,0 +1,231 @@ +# 聊天记录按时间范围抓取 设计文档 + +- 日期: 2026-08-25 +- 状态: 待评审 +- 关联: `d:\alibb\` Chrome 扩展(content.js / popup.html / popup.js) +- 前置讨论: 吸收三轮审查 —— ①自评 2 漏洞(§4.5 水位倒退、§4.3 until 跳游标未实测);②subagent 评审 4 建议;③深度复查 6 项(P1×3:空 list 静默 0 条 / fallback 与 RISK 抛出次序冲突 / date-only UTC 解析陷阱;P2×2:未来止日、水位检测顺序;优化:停止原因日志)及复查细化 2 项(§4.5 更新条件简化、§4.3 任务级降级标志) + +## 1. 概述 + +### 1.1 目标 +给聊天记录抓取增加**时间范围**控制:近 1 个月 / 2 个月 / 3 个月 / 自定义起止日期。选定范围后,翻页**到范围起点即停**(不只是抓完再滤),从而大幅减少对 `getOpMessages` 的请求量 —— 时间范围同时是**风控减压手段**。 + +### 1.2 动机 +现有唯一限制是条数(`chatLimit`,换算成翻页轮数上限)。条数与"我要最近 N 个月"的真实意图不匹配;且全量翻页对老客户动辄几百轮请求(每轮 2-5s 慢速),风控暴露面大。 + +## 2. 已确认决策(用户拍板) + +| # | 决策点 | 结论 | +|---|---|---| +| D1 | 时间范围 vs 条数 | **时间替代条数**:选时间模式后条数输入忽略、置灰 | +| D2 | 与增量机制关系 | **增量仍然生效**:增量命中 / 时间边界到,任一先到先停 | +| D3 | UI 形态 | 下拉:全部(默认,同现状)/ 近1月 / 近2月 / 近3月 / 自定义;自定义显示起止日期框 | +| D4 | 抓取路径 | **API 慢速 + 时间边界停**(否决 DOM 滚动为主 / CDP,理由见 §3) | + +## 3. 方案选型(为什么是 API + 时间边界) + +- **DOM 滚动为主被否决**:① DOM 滚动触发的底层请求仍是 `getOpMessages`(页面自己发),并非"不调 API";② 驱动事件是合成 `WheelEvent`,`isTrusted=false` 是**阿里已确认检测的指纹**(现有代码注释记载:折叠面板只接受真实点击);③ 慢 10 倍以上、图片懒加载丢数据、平台期误判截断。 +- **CDP(chrome.debugger)可信事件被否决**:常驻"正在调试此浏览器"提示条、实现复杂、速度仍慢。 +- **后端过滤被否决**:不省请求、白传数据、旧数据管不了。 +- **API 慢速路径实测战绩**:7-15 全量 4576 询盘 + 68058 条消息零风控触发。 + +## 4. 详细设计 + +### 4.1 UI(popup.html / popup.js) + +options 区"每询盘聊天上限"行下方新增一行 **聊天时间范围**: + +```html + + + ~ +``` + +- 选 `custom` → 显示起止框,**止默认今天**; +- 选任何时间模式(非 all)→ `chatLimitInput` 置灰 + 旁注"(时间模式下忽略)";切回 all → 恢复可用。两模式互斥。 +- popup.js 在 start 时计算并传参(**本地时区**): + - `all` → `chatSince: null, chatUntil: null`(完全等同现状,回归安全); + - `1m/2m/3m` → `chatSince = minusMonths(今日0点, N)`(**自然月整天边界**,起始日 00:00:00.000,不保留当前时分秒 —— 否则起始日当天早间消息被切),`chatUntil = 今日 23:59:59.999`("至今"取今天整天,与 custom 止=今天同一路径); + - `custom` → `chatSince = 起日 T00:00:00.000`,`chatUntil = 止日 T23:59:59.999`(含止日全天); + - **校验(popup 报错不发 start)**:起 > 止 / 起或止为空 / **止日 > 今天**(未来止日会造出未来 cursor,属未实测形态); + - ⚠️ **实现陷阱(date-only UTC)**:`new Date('2026-08-25')` 按 **UTC** 解析,中国时区差 8 小时,止日凌晨的消息会被静默滤掉。必须写 `new Date(v + 'T00:00:00')`(带时间部分 → 本地解析)或 `new Date(y, m-1, d)`。verify 脚本有用例锁定。 + +### 4.2 参数传递与状态(content.js) + +- start 消息新增 `chatSince` / `chatUntil`(ISO 字符串或 null),存入 `state` 并随 `saveState()` 持久化 —— 复用现有跨导航/暂停恢复/扩展重载的状态链路,零额外机制。注意:现有 start 分支是**全量重建 state 对象**,需把这两个字段显式加入重建清单(漏写会变 undefined,虽按 null 容错可用,但显式更清晰)。 +- 进入抓取时派生一次毫秒值:`sinceMs = chatSince ? Date.parse(chatSince) : null`(untilMs 同理),后续循环只用毫秒比较。 +- **时间模式下 popup 一律发 `chatLimit: 0`**:使 API 路径(MAX_ROUNDS→2000)与 DOM 回退路径(→300)两条现有公式都自然解析为全量轮数,无需对称修改换算逻辑。 + +### 4.3 API 主路径(fetchChatByApi)改动 + +翻页循环(现有停止条件:增量命中 hitLastTop / list.length < 20 到顶 / MAX_ROUNDS)增加: + +1. **初始游标**:`cursor = (untilMs != null) ? untilMs + 1 : Date.now()` + - +1ms 缓冲:宁可多取一轮,由批内过滤兜底,确保 until 邻近消息不漏(§6.2); + - 自定义过去截止日时直接从截止日开始往回翻,**省掉 now→until 之间的全部请求**。 +2. **收集过滤**(批内跨边界):`raw.push(m)` 仅当 + `m.sendTime == null || ((!sinceMs || m.sendTime >= sinceMs) && (!untilMs || m.sendTime <= untilMs))` + —— sendTime 为空的消息**防御性保留**(与 §5 约定一致;若不显式加此 guard,`undefined >= sinceMs` 为 false 会把无时间消息错误丢弃)。 + **顺序约定:水位命中检测遍历未过滤的原始 list(现有口径),命中即停;未命中的消息再套范围过滤决定 push。** 禁止先 `list.filter(范围)` 再查水位 —— 水位消息落在范围外时行为会漂移。 +3. **翻页停止**:每轮游标更新后,`if (sinceMs && minT < sinceMs) → break`,日志记 `时间边界提前停止(早于 )`。 +4. MAX_ROUNDS 在时间模式下固定 2000(安全阀防死循环;正常由时间/增量/到顶停)。 +5. 日志:范围模式下每询盘开始抓聊天时输出当前范围(`聊天范围: 2026-07-25 ~ 至今` / `~ 2026-08-31`;all 模式无此日志);完成日志统一带**停止原因**:`聊天 API 完成: N 条(停:时间边界/增量命中/到顶/空列表/轮数上限)` —— 手动验证一眼分辨是哪种停。 + +**until 跳游标的保守 fallback(首轮专项)**:**2026-08-25 已实测**(附录 B):过去时间戳 cursor 完全受支持,语义 = 返回 sendTime **严格小于** timeStamp 的 N 条(方向与现网一致);`untilMs+1` 缓冲恰好覆盖含止日全天。fallback 保留为**保险**(实测样本单一:1 账号 1 会话,阿里行为可能变化),并按实测精化: + +- **触发条件**(仅 `untilMs != null` 时的**首轮**,且仅首轮): + 1. `PROBE_FAIL` 形态:非对象 / code≠200 / data=null —— 参数异常; + 2. **返回空 list** —— 实测(附录 B 组)表明这大概率是"该会话在 until 前确无消息"(真空,如新询盘),但单次响应无法与"不支持"区分,仍降级以保正确(空 list 会被现有"正常到顶"分支 [content.js:816](content.js#L816) 静默吞掉,是真静默丢数据形态)。 +- **实现次序(P1-2)**:首轮结构断言单独 try 包住,抛**独立前缀** `PROBE_FAIL`(勿复用 RISK 前缀,避免误吞 `RISK:结构异常(non-JSON)`);降级集 = {PROBE_FAIL, 空 list};postForm 自身抛的 RISK(HTTP 非 200 / 关键词 / non-JSON)在 try 之外,照常冒泡暂停,不得吞掉。 +- **降级动作**:cursor 重置 `Date.now()`,**同循环 continue** 从现在往回翻到 since 停(批内过滤兜 until 上界);重试首批仍空 list = 真空(正常到顶),不递归降级。 +- **任务级标志(final review 再收紧)**:仅当**降级原因为 'fail'(结构异常)且降级重试首批非空**才设 `state.cursorJumpUnsupported = true`。'empty'(空 list)多为"会话在 until 前本无消息"的真空(附录 B 组形态)—— 据此设标志会把跳游标误禁一整个任务(所有询盘退化为 now 全量回翻,风控暴露面回升),故 empty 降级保正确性但不设标志。实现以局部变量 `downgradeKind` 记录原因。 + +### 4.4 DOM 回退路径(loadAllChatHistory,API 失败才走) + +- **全量分支**:滚动循环每轮**轻量探测**——只读**第一个** `.message-item-wrapper` 的时间 span(聊天 DOM 顶部=最旧;不要全量提取,几千条时每轮全提性能差),若其 sendTime 早于 sinceMs → 停止往上滚(再滚加载的全是范围外)。 +- 滚动结束后 `extractDomChatMessages()` 提取,再按 since/until 过滤后才挂到 `state._incDomMessages` / chatMessages。 +- **增量分支**:现有 `sendTime > lastDomTime` 过滤后,**再叠加** since/until 过滤(两条件都满足才收)。 +- **预期降级(非 bug)**:增量分支只取页面已显示的最新 N 条、从不向过去滚动 —— 自定义**过去**截止日时,叠加过滤后可能产出 0 条。这是回退路径的可接受退化(API 路径才是主路径),勿当 bug 上报。 +- `lastDomTime` 水位计算沿用**未过滤**集合的 max(水位反映"DOM 已见到的最新",过滤只影响推送)—— 现有 max 逻辑天然单调前进,无需改动。 + +### 4.5 增量水位只前进修复(自评漏洞 #1) + +**问题**:自定义过去截止日(如 6-01~6-30)抓完后,`lastTopMsgId` 会被覆盖成本轮翻到的最新(≈6-30),把原 7-28 的水位**倒退** → 下次增量要多翻整月已抓过的消息(数据不脏,后端 skipDuplicates,但请求白费、风控暴露面白增)。 + +**修复**:`lastTopMsg_` 存储结构改为 JSON `{ id, sendTime }`: +- 读取兼容旧格式(纯 string 视为 `sendTime` 未知 → 允许覆盖一次,升级到新格式); +- 更新条件简化为:`候选水位.sendTime > 旧水位.sendTime` 才写入(**水位只前进不倒退**)。原实现的 `raw.length > 0` 附加条件**移除** —— 它在旧格式(id 不可比时间)下防误清,新格式的只前进比较本身已安全;保留反而导致"本轮全被 until 滤空、但确实见到更新消息"时水位保守不前,下次多翻; +- 候选水位取**各轮见过的最大 sendTime**(未过滤 list 口径)。注意与现状的差异:现有实现只在首批设一次(`if (newestMsgIdThisRound === null && ...)`,[content.js:818-823](content.js#L818-L823)),fallback 降级重翻后首批(更新侧)无法再上调候选 → 水位保守停在 probe 轮的旧值,下次多翻。改为逐轮取 max 后此问题消除。 + +注:DOM 侧 `lastDomTime` 是毫秒时间戳比较,天然单调,无需此修复。 + +### 4.6 不改动的部分 + +- 同客户聊天去重(`seenChatCustomer`)、`chatSkipped` 标记、后端归代表:不受影响; +- API 返回空数组 `[]` = 成功无新增(不触发 DOM 回退)的现有语义:不变; +- CSV 导出、enqueue 推送、payload.js 映射、后端/前端:**零改动**(过滤在扩展侧完成,messages 照常流转); +- 询盘列表抓取、详情页客户资料/行为数据:**不受时间范围影响**(时间范围只作用于聊天)。 + +## 5. 边界语义与容错 + +| 项 | 约定 | +|---|---| +| 含边界 | `sendTime >= since` 且 `<= until`;自定义含止日全天(23:59:59.999) | +| 近 N 月 | 自然月,非 30 天,**整天边界**:since=起始日 00:00:00.000、until=今日 23:59:59.999(需求变更 2026-08-25:不保留当前时分秒)。⚠️ 不能直接 `setMonth(-N)`:月末日期溢出(3-31 − 1月 → "2-31" 溢出成 3-02,since 晚一个月**漏抓整月**)。须用 `minusMonths`(退到月初再钳制到目标月最后一天);verify 脚本有用例锁定 | +| 时区 | since/until 按用户**本地时区**构造(popup `new Date()`),与 API 毫秒时间戳、DOM 本地时间解析一致 | +| parse 失败 | chatSince/chatUntil 非法 → 视为 null(降级为不限),日志提示,**不中断抓取** | +| 消息无 sendTime | 防御性保留(宁多勿漏);实际 API 消息均带 sendTime | +| 起止校验 | custom 起>止 / 空 / 止日>今天 → popup 报错不发 start | + +## 6. 语义澄清(用户预期管理) + +1. **范围 ≠ 删除**:时间范围只控制"这次抓什么",**不会删除后端已有历史**。库内数据 = 历次抓取的并集(后端按 messageId skipDuplicates)。改选更短范围重抓,旧数据仍在。 +2. **增量水位对"过去区间"天然无效**:水位只优化"追加新消息"场景;自定义过去区间翻不到水位属正常,由 since 边界停。 +3. **抓取正确性承诺**:范围内一条不漏(边界含)、范围外一条不进(批内过滤)、已抓过的不重复翻(水位只前进 + 增量命中停)。 + +## 7. 验证方案 + +### 7.1 自动验证(content.js 为 IIFE,无法被 vitest import → 沿用复刻脚本模式) +新增 `scripts/verify-chat-range.mjs`(与 `verify-risk-detect.mjs` 同模式,文件头注明与 content.js 同步义务),覆盖: +- 自然月计算(跨年:3 月 31 日 - 1 月 = 2 月 28/29?边界行为锁定); +- 边界含/不含(since 当天消息收、since 前一天不收); +- 跨边界批过滤(一页 20 条横跨 since); +- 水位只前进(旧 {id,sendTime=7-28} 不被新 sendTime=6-30 覆盖;旧 string 格式兼容); +- cursor 缓冲(untilMs+1)与 parse 容错(null → 不限); +- date 输入**本地时区**构造(`new Date(v+'T00:00:00')` 非 UTC 解析,锁定 8-25T00:00 本地毫秒); +- fallback 触发判定(空 list / 结构异常 code、data=null 两形态 → 降级;HTTP/关键词/non-JSON 真风控形态不降级); +- 水位更新:候选**逐轮取 max**(覆盖 fallback 降级重翻场景)+ 只前进写入(sendTime 更大才写;raw 为空也可前进)。 + +### 7.2 手动验证清单 +1. 老客户询盘选"近 1 个月" → 日志出现 `时间边界提前停止`,条数远小于全量; +2. 自定义 起>止 → popup 报错不发; +3. 选"全部" → 行为与当前版本完全一致(回归); +4. **custom 含过去截止日 → 首轮盯 SW Network**(附录 B 已实测通过,此项为实现上线后的复测):`getOpMessages` 请求体 timeStamp 是否为 untilMs+1、返回是否正常; +5. 同询盘先"近3月"再"近1月"重抓 → 第二次条数显著少(增量)且无重复推送。 + +## 8. 风险与开放问题 + +| 风险 | 缓解 | +|---|---| +| ~~cursor 传过去时间戳的 API 行为未实测~~ → **已实测(2026-08-25,附录 B)**:受支持,语义=严格小于,+1ms 缓冲正确;残余风险 = 样本单一(1 账号/1 会话) | 保留首轮 fallback 保险(PROBE_FAIL / 空 list → 降级 now)+ 标志仅在"降级重试首批非空"时设置 + 实现后手动清单 #4 复测 | +| 复刻验证脚本与 content.js 漂移 | 脚本头注明同步义务(同 verify-risk-detect 现状,可接受的架构权衡) | +| 时间模式 + 很老的自定义起点 | MAX_ROUNDS=2000 安全阀兜底;用户自选范围,预期内 | + +## 9. 涉及文件 + +| 文件 | 改动 | +|---|---| +| popup.html | 时间范围下拉 + 自定义日期框 + 条数置灰逻辑的 UI | +| popup.js | 计算 since/until(自然月/含全天/校验)、置灰切换、start 传参 | +| content.js | state 字段、fetchChatByApi(游标/过滤/停止/水位)、loadAllChatHistory(探测/过滤)、lastTopMsg 存储结构 | +| scripts/verify-chat-range.mjs | 新增,复刻判定逻辑的自动验证 | + +## 附录 A:§4.3 修订后翻页循环伪代码(实现基准) + +§4.3 的 7 处改动(游标/探测降级/过滤/顺序/停止/日志/水位)在此串成完整循环,实现以此为准: + +```text +// 前置:sinceMs/untilMs 已从 state.chatSince/chatUntil 派生(§4.2);lastTop = 读水位(§4.5) +jump = untilMs != null && !state.cursorJumpUnsupported +cursor = jump ? untilMs + 1 : Date.now() +probeRound = jump // 仅跳游标首轮为探测轮 +candidate = null // 候选水位 {id, sendTime},逐轮取 max(§4.5) +stopReason = '轮数上限' +for i in 0..MAX_ROUNDS-1: + if state.phase == 'paused': return null + await waitUntilWorkTime() + json = postForm(getOpMessages, { timeStamp: cursor, forward: false, pageSize: 20, ... }) + if probeRound: + probeRound = false + try: + assertStructure(json) // 非对象 / code≠200 / data=null → throw PROBE_FAIL(独立前缀,勿复用 RISK 前缀) + if emptyList(json): throw PROBE_FAIL + catch e: + if e 是 PROBE_FAIL: // 仅降级集;postForm 的 RISK(non-JSON/HTTP/关键词)在 try 外照常冒泡 + // ⚠️ 勘误(实现以 §4.3 收紧版为准):此处不立即设 cursorJumpUnsupported; + // 实现记 downgradeKind,仅 kind='fail' 且降级重试首批非空才设(empty 多为真空,勿误禁全任务) + downgradeKind = kind + log('until 跳游标不受支持,降级 cursor=now') + cursor = Date.now() + await randSleep(2000, 5000) + continue // 重试按常规逻辑,不再探测;仍空 list = 正常到顶,不递归降级 + throw e // 真风控(HTTP/关键词/non-JSON)照常抛出暂停 + else: + assertStructure(json) // 常规轮:结构异常直接 RISK + list = json.data?.list || [] + if list 为空: stopReason = '到顶(空列表)'; break + for m of list: // 候选水位:逐轮最大 sendTime,未过滤口径(§4.5) + if m.sendTime != null && (candidate == null || m.sendTime > candidate.sendTime): + candidate = { id: String(m.messageId), sendTime: m.sendTime } + hit = false + for m of list: // 顺序:水位命中查未过滤 list,再范围过滤收集(§4.3.2) + if lastTop && String(m.messageId) == lastTop.id: hit = true; break + if m.sendTime == null || (!sinceMs || m.sendTime >= sinceMs) && (!untilMs || m.sendTime <= untilMs): + raw.push(m) + cursor = min(list 中非 null 的 sendTime) // null 必须跳过(与现状 [content.js:830-831] 一致);Math.min 含 null 得 0 → 游标跳 1970 + if hit: stopReason = '增量命中'; break + if sinceMs && cursor < sinceMs: stopReason = '时间边界'; break + if list.length < 20: stopReason = '到顶'; break + await randSleep(2000, 5000) +log(`聊天 API 完成: ${raw.length} 条(停:${stopReason})`) +if candidate && (旧水位为旧格式string || candidate.sendTime > 旧水位.sendTime): + 写入 candidate // 水位只前进(§4.5) +``` + +## 附录 B:过去时间戳 cursor 实测记录(2026-08-25) + +**环境**:用户登录态,询盘详情页(imInquiryId=13908460211,会话共 25 条消息),页面上下文复刻 content.js 的 postForm(**无签名头**),请求间隔 2.5s,共 4 次请求,未触发任何风控。 + +| 组 | cursor | HTTP/code | 返回 | 结论 | +|---|---|---|---|---| +| A 对照 | `now` | 200/"200" | 20 条(最新 20 条,oldest=8-24T13:06:27.788Z) | 现网行为正常 | +| B 过去30天 | `now−30d` | 200/"200" | **空 list,完全静默** | 该会话是新询盘,30 天前确无消息 → 空是**正确语义的空**;同时证明空 list 形态真实存在且与"不支持"无法从单次响应区分(P1-1 降级必要性成立) | +| C 边界 | =A 批最旧消息 sendTime | 200/"200" | 5 条,全部**严格小于** cursor(不含 cursor 那条),与 A 批无缝衔接 | ① 语义 = `sendTime < timeStamp`(严格小于,向前翻);② **过去时间戳受支持、方向正确**;③ 翻页连续(cursor=minT 下批不重不漏);④ `untilMs+1` 恰好覆盖 `<= untilMs` 全部消息 | + +**衍生结论**:① until 跳游标可用(省 now→until 全部请求);② +1ms 缓冲正确且必要;③ 空 list 二义性真实 → fallback 保留,但任务级标志仅在"降级重试首批非空"时设置(§4.3 实测精化);④ 复刻 postForm 无签名头直接可用,佐证 §3 选型。 diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000..adaa138 --- /dev/null +++ b/manifest.json @@ -0,0 +1,29 @@ +{ + "manifest_version": 3, + "name": "Alibaba 询盘数据抓取", + "version": "1.0.0", + "description": "抓取阿里国际站询盘列表、客户资料、行为数据、聊天记录，支持 customerId 去重，导出为 CSV", + "permissions": ["storage", "activeTab", "alarms", "unlimitedStorage", "scripting"], + "host_permissions": [ + "*://message.alibaba.com/*", + "*://alicrm.alibaba.com/*", + "*://onetalk.alibaba.com/*", + "http://localhost:8080/*" + ], + "optional_host_permissions": ["http://*/*", "https://*/*"], + "background": { + "service_worker": "background.js", + "type": "module" + }, + "content_scripts": [ + { + "matches": ["*://message.alibaba.com/*"], + "js": ["content.js"], + "run_at": "document_idle" + } + ], + "action": { + "default_popup": "popup.html", + "default_title": "阿里询盘抓取" + } +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..87dd8cf --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1279 @@ +{ + "name": "alibaba-inquiry-plugin", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "alibaba-inquiry-plugin", + "version": "1.0.0", + "devDependencies": { + "fake-indexeddb": "^6.2.5", + "vitest": "^4.1.10" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vitest/expect": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", + "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", + "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.10", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", + "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", + "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.10", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", + "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "@vitest/utils": "4.1.10", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", + "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", + "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fake-indexeddb": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz", + "integrity": "sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", + "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.139.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/vite": { + "version": "8.1.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz", + "integrity": "sha512-bTT9PsdWO+MQMNG9ZXIP/qM9wGh37DFxTV/sPq9cFpHr3w4jkgef032PkAL9jAqhk3Nz8NQw3O8n6/xFkqO4QQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.16", + "rolldown": "~1.1.4", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", + "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.10", + "@vitest/mocker": "4.1.10", + "@vitest/pretty-format": "4.1.10", + "@vitest/runner": "4.1.10", + "@vitest/snapshot": "4.1.10", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.10", + "@vitest/browser-preview": "4.1.10", + "@vitest/browser-webdriverio": "4.1.10", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/ui": "4.1.10", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..3a10abb --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "alibaba-inquiry-plugin", + "version": "1.0.0", + "type": "module", + "private": true, + "scripts": { + "test": "vitest run" + }, + "devDependencies": { + "fake-indexeddb": "^6.2.5", + "vitest": "^4.1.10" + } +} diff --git a/popup.html b/popup.html new file mode 100644 index 0000000..381f593 --- /dev/null +++ b/popup.html @@ -0,0 +1,104 @@ + + + + + + + +
+

阿里询盘数据抓取

+

message.alibaba.com 询盘列表 + 详情

+
+ + + +
+ 状态 + 就绪 +
+ +
+ +
+ 推送:加载中... +
+ +
+ + + + + + + + +
+ + + + + +
+
+ +
+
后端推送设置
+ + + + + +
+ +
+ + + + diff --git a/popup.js b/popup.js new file mode 100644 index 0000000..058f044 --- /dev/null +++ b/popup.js @@ -0,0 +1,311 @@ +// popup.js — 控制面板逻辑 +const btnStart = document.getElementById('btnStart'); +const btnPause = document.getElementById('btnPause'); +const btnExport = document.getElementById('btnExport'); +const btnClear = document.getElementById('btnClear'); +const statusText = document.getElementById('statusText'); +const progressFill = document.getElementById('progressFill'); +const logBox = document.getElementById('logBox'); +const modeSelect = document.getElementById('modeSelect'); +const optDedup = document.getElementById('optDedup'); +const optChat = document.getElementById('optChat'); +const limitSelect = document.getElementById('limitSelect'); +const chatTimeRangeSelect = document.getElementById('chatTimeRange'); +const chatCustomDates = document.getElementById('chatCustomDates'); +const chatSinceDate = document.getElementById('chatSinceDate'); +const chatUntilDate = document.getElementById('chatUntilDate'); +const chatLimitHint = document.getElementById('chatLimitHint'); + +function addLog(msg, type = '') { + const p = document.createElement('p'); + p.className = type; + p.textContent = `[${new Date().toLocaleTimeString()}] ${msg}`; + logBox.appendChild(p); + logBox.scrollTop = logBox.scrollHeight; +} + +function setProgress(current, total) { + const pct = total > 0 ? Math.round((current / total) * 100) : 0; + progressFill.style.width = pct + '%'; + statusText.textContent = `${current} / ${total} (${pct}%)`; +} + +function updateButtons(phase) { + const running = phase === 'list' || phase === 'detail'; + const paused = phase === 'paused'; + + btnStart.disabled = running || paused; + btnPause.disabled = !running && !paused; + btnPause.textContent = paused ? '恢复' : '暂停'; + btnExport.disabled = running || paused; +} + +// 带重试的 send，content script 注入需要时间 +function send(action, data = {}, retries = 3) { + return new Promise(resolve => { + function attempt(remaining) { + chrome.tabs.query({ active: true, currentWindow: true }, tabs => { + if (!tabs[0]) { addLog('未找到活动标签页', 'error'); resolve(); return; } + + // 检查是否在正确的域名 + if (!tabs[0].url || !tabs[0].url.includes('message.alibaba.com')) { + addLog('请在 message.alibaba.com 页面操作', 'error'); + resolve(); + return; + } + + chrome.tabs.sendMessage(tabs[0].id, { action, ...data }, resp => { + if (chrome.runtime.lastError) { + // content script 未注入(扩展重载后旧页面失效/新标签注入慢)→ 程序化注入兜底。 + // content.js 自带 __alibabaScraperInit 防重复注入,重复执行安全。 + chrome.scripting.executeScript({ target: { tabId: tabs[0].id }, files: ['content.js'] }, () => { + if (chrome.runtime.lastError) { + addLog('自动注入失败: ' + chrome.runtime.lastError.message, 'error'); + resolve(); + return; + } + if (remaining > 0) { + setTimeout(() => attempt(remaining - 1), 800); // 注入完成后再试 + } else { + addLog('连接失败，请刷新页面后重试', 'error'); + resolve(); + } + }); + } else { + resolve(resp); + } + }); + }); + } + attempt(retries); + }); +} + +// 检查当前状态 +send('getState').then(resp => { + if (resp) { + if (resp.running) { + addLog(`有进行中的任务 (列表${resp.listCount}条, 详情${resp.detailCount}条, 队列${resp.queueIndex}/${resp.queueTotal})`, 'info'); + statusText.textContent = `运行中 (${resp.queueIndex}/${resp.queueTotal})`; + updateButtons(resp.phase); + } else if (resp.phase === 'paused') { + addLog('有暂停的任务，点击"恢复"继续', 'info'); + statusText.textContent = '已暂停'; + updateButtons('paused'); + } + } +}); + +btnStart.addEventListener('click', async () => { + const range = computeChatRange(chatTimeRangeSelect.value, chatSinceDate.value, chatUntilDate.value); + if (range.error) { addLog(range.error, 'error'); statusText.textContent = '就绪'; updateButtons('idle'); return; } + // 与 range 同步快照:导航等待 5s 内用户仍可改下拉,不能让部分参数冻结、部分活读(参数漂移) + const chatLimitSnap = chatTimeRangeSelect.value === 'all' ? (parseInt(document.getElementById('chatLimitInput').value, 10) || 0) : 0; + updateButtons('list'); + hideRiskBanner(); // 开始新任务时隐藏风控红条 + addLog('开始抓取...', 'info'); + statusText.textContent = '启动中...'; + chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => { + const tab = tabs && tabs[0]; + // 不在消息页 → 自动跳转，等 content.js 注入后再发 start + if (tab && !(tab.url && tab.url.includes('message.alibaba.com'))) { + addLog('当前不在消息页，自动跳转...', 'info'); + statusText.textContent = '跳转消息页...'; + await chrome.tabs.update(tab.id, { url: 'https://message.alibaba.com/message/default.htm#feedback/all' }); + await new Promise(r => setTimeout(r, 5000)); // 等 document_idle + content 注入 + addLog('消息页就绪，开始抓取', 'info'); + } + await send('start', { + mode: modeSelect.value, + dedup: optDedup.checked, + chat: optChat.checked, + limit: parseInt(limitSelect.value, 10), + chatLimit: chatLimitSnap, + chatSince: range.since, + chatUntil: range.until, + }); + }); +}); + +btnPause.addEventListener('click', async () => { + const phase = btnPause.textContent === '恢复' ? 'resume' : 'pause'; + await send(phase); + if (phase === 'pause') { + updateButtons('paused'); + statusText.textContent = '已暂停'; + btnPause.textContent = '恢复'; + addLog('已暂停（支持跨页面恢复）', 'info'); + } else { + updateButtons('list'); + hideRiskBanner(); // 恢复时隐藏风控红条(用户已处理验证码) + statusText.textContent = '恢复中...'; + btnPause.textContent = '暂停'; + addLog('恢复抓取...', 'info'); + } +}); + +btnExport.addEventListener('click', async () => { + addLog('生成 CSV...', 'info'); + const resp = await send('export'); + if (resp && resp.success) { + addLog(`导出 ${resp.count} 条数据`, 'success'); + const blob = new Blob(['\uFEFF' + resp.csv], { type: 'text/csv;charset=utf-8' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `alibaba_inquiry_${new Date().toISOString().slice(0, 10)}.csv`; + a.click(); + URL.revokeObjectURL(url); + } +}); + +btnClear.addEventListener('click', async () => { + await send('clear'); + hideRiskBanner(); // 清空时隐藏风控红条 + logBox.innerHTML = ''; + statusText.textContent = '就绪'; + progressFill.style.width = '0%'; + btnPause.textContent = '暂停'; + updateButtons('idle'); + addLog('数据已清空', 'info'); +}); + +// 监听 content script 推送 +chrome.runtime.onMessage.addListener(msg => { + if (msg.type === 'log') addLog(msg.text, msg.level || ''); + if (msg.type === 'progress') setProgress(msg.current, msg.total); + if (msg.type === 'done') { updateButtons('done'); btnExport.disabled = false; statusText.textContent = '完成'; } + if (msg.type === 'error') { updateButtons('idle'); statusText.textContent = '出错'; } + // 安全层 #6:风控报警 —— 顶部红条 + 日志,按钮切到 paused 态 + if (msg.type === 'risk') { + const banner = document.getElementById('riskBanner'); + if (banner) banner.style.display = 'block'; + addLog('⚠️ 疑似触发风控: ' + (msg.msg || ''), 'error'); + updateButtons('paused'); + statusText.textContent = '风控暂停'; + } +}); + +// 隐藏风控红条(开始新任务/恢复时调用) +function hideRiskBanner() { + const banner = document.getElementById('riskBanner'); + if (banner) banner.style.display = 'none'; +} + +// ==== 聊天时间范围(spec 2026-08-25 §4.1)==== +// minusMonths/computeChatRange 与 scripts/verify-chat-range.mjs 基准实现逐字同步 +// 自然月回退:直接 setMonth(-n) 月末溢出(3-31−1月→"2-31"→3-02),先退月初再钳制天数 +function minusMonths(base, n) { + const d = new Date(base.getFullYear(), base.getMonth(), 1, + base.getHours(), base.getMinutes(), base.getSeconds(), base.getMilliseconds()); + d.setMonth(d.getMonth() - n); + const lastDay = new Date(d.getFullYear(), d.getMonth() + 1, 0).getDate(); + d.setDate(Math.min(base.getDate(), lastDay)); + return d; +} +// popup 范围计算:mode=all|1m|2m|3m|custom → {since,until}(ISO|null) 或 {error} +function computeChatRange(mode, sinceStr, untilStr) { + if (mode === 'all') return { since: null, until: null }; + if (mode === '1m' || mode === '2m' || mode === '3m') { + // 整天边界:since=起始日 00:00:00(完整自然月天数),until=今日 23:59:59.999("至今"取今天整天,与 custom 止=今天一致) + const dayStart = new Date(); dayStart.setHours(0, 0, 0, 0); + const dayEnd = new Date(); dayEnd.setHours(23, 59, 59, 999); + return { since: minusMonths(dayStart, Number(mode[0])).toISOString(), until: dayEnd.toISOString() }; + } + if (mode === 'custom') { + if (!sinceStr || !untilStr) return { error: '请填写自定义起止日期' }; + const s = new Date(sinceStr + 'T00:00:00.000'); // ⚠️ 必须拼时间:裸 date-only 按 UTC 解析差 8h + const u = new Date(untilStr + 'T23:59:59.999'); + if (isNaN(s.getTime()) || isNaN(u.getTime())) return { error: '日期格式错误' }; + if (s.getTime() > u.getTime()) return { error: '起始日期不能晚于截止日期' }; + const uDay = new Date(untilStr + 'T00:00:00.000'); + const t0 = new Date(); t0.setHours(0, 0, 0, 0); + if (uDay.getTime() > t0.getTime()) return { error: '截止日期不能晚于今天' }; + return { since: s.toISOString(), until: u.toISOString() }; + } + return { error: '未知时间范围模式' }; +} + +function syncChatModeUI() { + const mode = chatTimeRangeSelect.value; + const custom = mode === 'custom'; + chatCustomDates.style.display = custom ? 'block' : 'none'; + if (custom && !chatUntilDate.value) { + const t = new Date(); + chatUntilDate.value = `${t.getFullYear()}-${String(t.getMonth() + 1).padStart(2, '0')}-${String(t.getDate()).padStart(2, '0')}`; + } + const timeMode = mode !== 'all'; + document.getElementById('chatLimitInput').disabled = timeMode; // D1: 时间替代条数 + chatLimitHint.textContent = timeMode ? '(时间模式下忽略)' : ''; +} +chatTimeRangeSelect.addEventListener('change', syncChatModeUI); +syncChatModeUI(); + +// === 后端推送配置 === +const cfgServerUrl = document.getElementById('cfgServerUrl'); +const cfgToken = document.getElementById('cfgToken'); +const cfgEnabled = document.getElementById('cfgEnabled'); + +// 加载现有配置（background 返回 { ok, config }） +chrome.runtime.sendMessage({ type: 'getPushConfig' }, resp => { + if (chrome.runtime.lastError) { addLog('读取推送配置失败: ' + chrome.runtime.lastError.message, 'error'); return; } + const cfg = resp && resp.ok ? resp.config : null; + cfgServerUrl.value = (cfg && cfg.serverUrl) || 'http://219.157.255.213:23010'; + cfgToken.value = (cfg && cfg.token) || '688cbb4b6e880b340b2360559c4ad29d01a39ccbb3c31c4d'; + cfgEnabled.checked = cfg ? !!cfg.enabled : true; // 默认勾「启用推送」(首次未配置时) +}); + +// 授权域名（chrome.permissions.request 必须在用户手势内） +document.getElementById('btnAuthorize').addEventListener('click', () => { + const url = cfgServerUrl.value.trim(); + if (!url) { addLog('请先填写服务器 URL', 'error'); return; } + let origin; + try { const u = new URL(url); origin = `${u.origin}/*`; } + catch { addLog('URL 格式错误', 'error'); return; } + chrome.permissions.request({ origins: [origin] }, granted => { + if (chrome.runtime.lastError) { addLog('授权请求失败: ' + chrome.runtime.lastError.message, 'error'); return; } + if (granted) addLog('已授权 ' + origin, 'success'); + else addLog('授权被拒绝', 'error'); + }); +}); + +// 保存设置 +document.getElementById('btnSaveConfig').addEventListener('click', () => { + const config = { + serverUrl: cfgServerUrl.value.trim().replace(/\/+$/, ''), // 去尾斜杠 + token: cfgToken.value.trim(), + enabled: cfgEnabled.checked, + }; + chrome.runtime.sendMessage({ type: 'setPushConfig', config }, resp => { + if (chrome.runtime.lastError) { addLog('保存失败: ' + chrome.runtime.lastError.message, 'error'); return; } + if (resp && resp.ok) addLog('推送设置已保存' + (config.enabled ? '(已启用)' : ''), 'success'); + else addLog('保存失败', 'error'); + }); +}); + +// === 推送状态显示 === +const pushStatus = document.getElementById('pushStatus'); +function refreshPushStatus() { + chrome.runtime.sendMessage({ type: 'getPushStats' }, resp => { + if (chrome.runtime.lastError || !resp || !resp.ok || !resp.stats) { + pushStatus.style.color = '#999'; + pushStatus.textContent = '推送:未启用'; + return; + } + const stats = resp.stats; + const pending = stats.pending || 0; + const failed = stats.failed || 0; + const sent = stats.sent || 0; + pushStatus.style.color = failed > 0 ? '#f5222d' : '#666'; + pushStatus.textContent = `推送:已推 ${sent} / 待推 ${pending} / 失败 ${failed}`; + }); +} +refreshPushStatus(); +const statusTimer = setInterval(refreshPushStatus, 2000); +window.addEventListener('unload', () => clearInterval(statusTimer)); + +// 失败时点击查看提示 +pushStatus.addEventListener('click', () => { + if (pushStatus.textContent.includes('失败 0')) return; + alert('有推送失败项。可在扩展后台日志查看错误,或检查后端是否在线。'); +}); diff --git a/push/config.js b/push/config.js new file mode 100644 index 0000000..cbcafcb --- /dev/null +++ b/push/config.js @@ -0,0 +1,14 @@ +// 推送配置读写:存 chrome.storage.local。background 与 worker 共用。 +// +// ★ worker 必须**每次自己调 loadConfig()**(无状态),不能把结果缓存到模块级变量 —— +// MV3 Service Worker 休眠重启会清空模块级状态,缓存会导致 worker 永不推送(B 的根因)。 +const CONFIG_KEY = 'push_config'; + +export async function loadConfig() { + const data = await chrome.storage.local.get(CONFIG_KEY); + return data[CONFIG_KEY] || { enabled: false, serverUrl: '', token: '' }; +} + +export async function saveConfig(config) { + await chrome.storage.local.set({ [CONFIG_KEY]: config || {} }); +} diff --git a/push/payload.js b/push/payload.js new file mode 100644 index 0000000..4baa2ad --- /dev/null +++ b/push/payload.js @@ -0,0 +1,109 @@ +// Adapter: maps the page-side "record + messages" shape scraped from +// Alibaba's CRM pages to the backend `/api/ingest` zod PayloadSchema. + +// Trim undefined/empty/null to null for optional string fields. +const s = v => (v == null || v === '') ? null : v; + +// Coerce numeric strings (e.g. "39") to number; otherwise null. +const n = v => { + if (v == null || v === '') return null; + const x = Number(v); + return Number.isFinite(x) ? x : null; +}; + +// 只接受标准 ISO(YYYY-MM-DDTHH:...);非 ISO(列表页 "11:58"/"2026-7-8")跳过 → 兜底当前时间 +// 注:取消聊天时无 gmtCreate,列表时间为非 ISO,这里保证 createTime/updateTime 始终合法 ISO。 +// 准确时间待抓聊天时由 gmtCreate 通过 upsert 覆盖。 +const isIso = v => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/.test(v); +// 解析 alibaba 列表的非 ISO 时间: +// "2026-7-1"(YYYY-M-D,创建时间)→ 当天 00:00(本地) +// "13:58"(HH:mm,今天的更新时间)→ 今天该时分(本地) +const parseListTime = v => { + if (v == null || v === '') return null; + const s = String(v).trim(); + const d = s.match(/^(\d{4})-(\d{1,2})-(\d{1,2})$/); + if (d) return new Date(Number(d[1]), Number(d[2]) - 1, Number(d[3])).toISOString(); + const hm = s.match(/^(\d{1,2}):(\d{2})$/); + if (hm) { + const n = new Date(); + return new Date(n.getFullYear(), n.getMonth(), n.getDate(), Number(hm[1]), Number(hm[2])).toISOString(); + } + return null; +}; +const isoFallback = (...vals) => { + for (const v of vals) { + if (isIso(v)) return v; + const p = parseListTime(v); + if (p) return p; + } + return new Date(Date.now()).toISOString(); +}; + +export function buildIngestPayload(record, messages) { + // 阶段1:允许 messages 为空（先走通列表+客户+行为,聊天后续补充） + const msgs = messages || []; + const iq = record || {}; + + const inquiry = { + inquiryId: String(iq.inquiryId), + manager: s(iq.manager) || undefined, + createTime: isoFallback(iq.createTimeFromApi, iq.createTime), + updateTime: isoFallback( + iq.updateTime, + iq.updateTimeFromApi, + iq.createTimeFromApi, + ), + status: s(iq.status) || undefined, + msgPreview: s(iq.msgPreview) || undefined, + isFlagged: !!iq.isFlagged, + }; + + const customer = iq.customerId + ? { + customerId: String(iq.customerId), + name: s(iq.detailBuyerName) || s(iq.buyerName), + country: s(iq.detailCountry) || s(iq.countryCode), + email: s(iq.email), + phone: s(iq.phone), + landline: s(iq.landline), + businessType: s(iq.businessType), + salesPlatform: s(iq.salesPlatform), + website: s(iq.website), + company: s(iq.company), + registerTime: s(iq.registerTime), + avatarUrl: s(iq.detailAvatarUrl) || s(iq.avatarUrl) || undefined, + buyerTags: s(iq.buyerTags) || undefined, + buyerFeatures: s(iq.buyerFeatures) || undefined, + recentInquiryProducts: Array.isArray(iq.recentInquiryProducts) ? iq.recentInquiryProducts : undefined, + storeInquiryProduct: s(iq.storeInquiryProduct) || undefined, + storeInquiryQty: s(iq.storeInquiryQty) || undefined, + storeVisitProducts: Array.isArray(iq.storeVisitProducts) ? iq.storeVisitProducts : undefined, + behavior: iq.behavior + ? { + productViews: n(iq.behavior.productViews), + validInquiries: n(iq.behavior.validInquiries), + validRfq: n(iq.behavior.validRFQ), // detail 用 validRFQ(大写) + loginDays: n(iq.behavior.loginDays), + spamInquiries: n(iq.behavior.spamInquiries), + blacklistCount: n(iq.behavior.blacklistCount), + topIndustry: s(iq.behavior.topIndustry), + } + : undefined, + } + : null; + + const out = { + inquiry, + customer, + messages: msgs.map(m => ({ + messageId: String(m.messageId), + sendTime: m.sendTime, + sender: s(m.sender) || undefined, + direction: m.direction, + content: m.original ?? m.content ?? '', + translation: s(m.translation) ?? '', + })), + }; + + return out; +} diff --git a/push/queue.js b/push/queue.js new file mode 100644 index 0000000..03f9ca2 --- /dev/null +++ b/push/queue.js @@ -0,0 +1,106 @@ +// IndexedDB-backed push queue for inquiry ingest payloads. +// Stores pending/sent/failed items, supports a "nextRetry" backoff gate so +// the worker only picks up items whose retry window has elapsed. + +const DB_NAME = 'inquiry_push'; +const STORE = 'queue'; +const VERSION = 1; + +function openDb() { + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, VERSION); + req.onupgradeneeded = () => { + const db = req.result; + if (!db.objectStoreNames.contains(STORE)) { + db.createObjectStore(STORE, { keyPath: 'id' }); + } + }; + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); +} + +function store(db, mode) { + return db.transaction(STORE, mode).objectStore(STORE); +} + +function p(r) { + return new Promise((res, rej) => { + r.onsuccess = () => res(r.result); + r.onerror = () => rej(r.error); + }); +} + +function uuid() { + if (globalThis.crypto?.randomUUID) return globalThis.crypto.randomUUID(); + return String(Date.now()) + '-' + Math.random().toString(36).slice(2); +} + +export async function addItem(item) { + const db = await openDb(); + try { + const id = uuid(); + const rec = { + id, + payload: item.payload, + inquiryId: item.inquiryId, + status: 'pending', + retries: 0, + nextRetry: Date.now(), + createdAt: Date.now(), + }; + await p(store(db, 'readwrite').add(rec)); + return id; + } finally { + db.close(); + } +} + +export async function getPending(limit = 5) { + const db = await openDb(); + try { + const all = await p(store(db, 'readonly').getAll()); + const now = Date.now(); + return all + .filter(r => r.status === 'pending' && r.nextRetry <= now) + .sort((a, b) => a.nextRetry - b.nextRetry) + .slice(0, limit); + } finally { + db.close(); + } +} + +export async function markStatus(id, status, opts = {}) { + const db = await openDb(); + try { + const st = store(db, 'readwrite'); + const r = await p(st.get(id)); + if (!r) return; + Object.assign(r, { status, ...opts }); + await p(st.put(r)); + } finally { + db.close(); + } +} + +export async function remove(id) { + const db = await openDb(); + try { + await p(store(db, 'readwrite').delete(id)); + } finally { + db.close(); + } +} + +export async function getStats() { + const db = await openDb(); + try { + const all = await p(store(db, 'readonly').getAll()); + return all.reduce((acc, r) => { + acc[r.status] = (acc[r.status] || 0) + 1; + return acc; + }, {}); + } finally { + db.close(); + } +} diff --git a/push/worker.js b/push/worker.js new file mode 100644 index 0000000..d77651d --- /dev/null +++ b/push/worker.js @@ -0,0 +1,58 @@ +// Push queue worker: drains pending items, POSTs to the ingest endpoint, +// removes on success, backs off exponentially on failure, marks failed +// after MAX_RETRIES attempts. Driven by alarms in background.js. +// +// ★ 无状态:每次 processQueue 自己 loadConfig() 从 chrome.storage 读配置, +// 不缓存到模块级变量 —— MV3 SW 休眠重启会清空模块状态,缓存会导致 worker 永不推送(B)。 + +import { getPending, markStatus } from './queue.js'; +import { buildIngestPayload } from './payload.js'; +import { loadConfig } from './config.js'; + +const MAX_RETRIES = 8; + +export async function processQueue() { + const cfg = await loadConfig(); + if (!cfg || !cfg.enabled || !cfg.serverUrl || !cfg.token) { + return { processed: 0 }; + } + const items = await getPending(3); + let processed = 0; + for (const it of items) { + try { + const payload = buildIngestPayload(it.payload.record, it.payload.messages); + const res = await fetch(`${cfg.serverUrl}/api/ingest`, { + method: 'POST', + headers: { + Authorization: `Bearer ${cfg.token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }); + if (res.ok) { + // 标 sent 保留记录供 getStats 统计;旧实现 remove 会让 popup"已推"永远 0(显示 bug) + await markStatus(it.id, 'sent'); + processed++; + } else { + await bumpRetry(it, `HTTP ${res.status}`); + } + } catch (e) { + await bumpRetry(it, String((e && e.message) || e)); + } + } + return { processed }; +} + +async function bumpRetry(it, err) { + const retries = (it.retries || 0) + 1; + if (retries >= MAX_RETRIES) { + await markStatus(it.id, 'failed', { retries, lastError: err }); + } else { + const delay = Math.min(Math.pow(2, retries) * 1000, 3600 * 1000); + await markStatus(it.id, 'pending', { + retries, + nextRetry: Date.now() + delay, + lastError: err, + }); + } +} diff --git a/scripts/e2e-plugin.md b/scripts/e2e-plugin.md new file mode 100644 index 0000000..3ce44f2 --- /dev/null +++ b/scripts/e2e-plugin.md @@ -0,0 +1,47 @@ +# 插件 → 后端 端到端验证 + +## 自动化验证(push 模块 → 后端链路,已验证 ✅) + +前置:Plan 1 后端在跑(`cd d:/alibb-server/backend && docker compose up -d`,`/api/health` ok)。 + +```bash +cd d:/alibb && node scripts/e2e-push.mjs +# 预期:ingest status 200, inserted 1 +# 然后查: +JWT=$(curl -s -XPOST http://localhost:8080/api/auth/login -H 'Content-Type: application/json' -d '{"username":"admin","password":"admin-password-1234"}' | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.parse(s).token))") +curl -s "http://localhost:8080/api/inquiries/E2E-TEST2" -H "Authorization: Bearer $JWT" +# 预期:看到 inquiry + customer + _count.messages=1 +``` + +这验证 `push/payload.js` 组装的 payload 能被 Plan 1 后端 `/api/ingest` 正确接收入库(询盘+客户+消息)。 + +## 完整插件 e2e(需手动,Chrome 加载扩展) + +1. **起后端**:`cd d:/alibb-server/backend && docker compose up -d --build`,等 `curl http://localhost:8080/api/health` → `{"ok":true}` +2. **加载扩展**:Chrome 打开 `chrome://extensions` → 开启「开发者模式」→「加载已解压的扩展程序」→ 选 `d:/alibb` +3. **配置推送**:点插件图标 →「后端设置」区填: + - 服务器 URL:`http://localhost:8080` + - Token:`dev-ingest-token-1234` + - 勾「启用推送」 + - 点「授权域名」→ Chrome 弹框点「允许」 + - 点「保存设置」→ 看到「推送设置已保存(已启用)」 +4. **抓取**:打开 `https://message.alibaba.com/message/default.htm`(登录阿里主账号)→ 点插件「开始抓取」(detail 模式,测试 10 条) +5. **观察**:popup 顶部状态条「推送:已推 N / 待推 M / 失败 K」随抓取增长 +6. **验证后端有数据**: + ```bash + JWT=$(curl -s -XPOST http://localhost:8080/api/auth/login -H 'Content-Type: application/json' -d '{"username":"admin","password":"admin-password-1234"}' | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>console.log(JSON.parse(s).token))") + curl -s "http://localhost:8080/api/inquiries?size=5" -H "Authorization: Bearer $JWT" | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const d=JSON.parse(s);console.log('total:',d.total);d.items.forEach(i=>console.log(' ',i.inquiryId,i.customer?.country,'msgs:',i.messagesCount))})" + ``` + 预期:看到刚抓的询盘,带 country + messagesCount +7. **验证幂等**:再抓同一询盘 → 后端 `messages_count` 不变(早返回生效) + +## 断网/重试验证(防丢) + +1. 停后端:`docker compose down` +2. 抓一条 → popup 状态「待推 1」(入 IndexedDB 队列,未丢) +3. 起后端:`docker compose up -d` +4. 等 ~1 分钟(chrome.alarms 触发 worker)→ popup「已推 1」(自动补推) + +## 收尾 + +`cd d:/alibb-server/backend && docker compose down`(volume 保留) diff --git a/scripts/e2e-push.mjs b/scripts/e2e-push.mjs new file mode 100644 index 0000000..50c573b --- /dev/null +++ b/scripts/e2e-push.mjs @@ -0,0 +1,26 @@ +// 模拟插件推送:用 push/payload.js 组装 payload,POST 到后端 /api/ingest +// 用法: node scripts/e2e-push.mjs (可选 SERVER/TOKEN 环境变量) +import { buildIngestPayload } from '../push/payload.js'; + +const record = { + inquiryId: 'E2E-TEST2', manager: 'tester', + createTimeFromApi: '2026-07-14T10:00:00Z', updateTimeFromApi: '2026-07-14T10:00:00Z', + status: 'new', customerId: 'c-e2e', detailCountry: 'US', detailBuyerName: 'Test Buyer', + behavior: { validRfq: '5' }, +}; +const messages = [{ + messageId: 'e2e-test-m1', sendTime: '2026-07-14T10:00:00Z', + sender: 'tester', original: 'hello e2e', direction: 'outgoing', +}]; + +const payload = buildIngestPayload(record, messages); +const SERVER = process.env.SERVER || 'http://localhost:8080'; +const TOKEN = process.env.TOKEN || 'dev-ingest-token-1234'; + +const res = await fetch(`${SERVER}/api/ingest`, { + method: 'POST', + headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), +}); +console.log('ingest status:', res.status); +console.log('ingest body:', JSON.stringify(await res.json())); diff --git a/scripts/test-cid-length.mjs b/scripts/test-cid-length.mjs new file mode 100644 index 0000000..812da98 --- /dev/null +++ b/scripts/test-cid-length.mjs @@ -0,0 +1,35 @@ +// 根因验证:后端 customers.customer_id 是 VARCHAR(64)(spec §6.1), +// 而插件对无真 customerId 的询盘用 sec_ + secTradeId 兜底(75 字符)。 +// 本脚本推两条测试数据对比:超长 sec_ vs 短 id,看后端 ingest 的实际反应。 +// 用法: node scripts/test-cid-length.mjs +import { buildIngestPayload } from '../push/payload.js'; + +const SERVER = process.env.SERVER || 'http://219.157.255.213:23010'; +const TOKEN = process.env.TOKEN || '688cbb4b6e880b340b2360559c4ad29d01a39ccbb3c31c4d'; + +async function push(label, customerId) { + const record = { + inquiryId: `TEST-CID-${label}`, + customerId, + detailBuyerName: 'Test Buyer', detailCountry: 'US', company: 'Test Co', + createTimeFromApi: '2026-07-20T00:00:00Z', updateTimeFromApi: '2026-07-20T00:00:00Z', + behavior: { productViews: '1' }, + }; + const payload = buildIngestPayload(record, []); + const res = await fetch(`${SERVER}/api/ingest`, { + method: 'POST', + headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + let body = ''; + try { body = await res.text(); } catch (e) { body = '(no body)'; } + console.log(`[${label}] customerId(len=${customerId.length}): status=${res.status}`); + console.log(` body=${body.slice(0, 400)}`); + console.log(` payload.customer.customerId = ${JSON.stringify(payload.customer && payload.customer.customerId)}`); +} + +const longCid = 'sec_MC1IDX1yqSSuFKld-CG7npDfPBic6Iv1S_ZJecxnw8b7kTejq_GNIDRuGTmXE-sjnZMUpze'; +console.log('=== longCid length:', longCid.length, '==='); +await push('LONG75', longCid); +await push('SHORT5', '12345'); +console.log('\n说明:若 LONG75 失败(4xx)而 SHORT5 成功(200) → 证实后端因长度拒绝 sec_ customerId'); diff --git a/scripts/verify-chat-range.mjs b/scripts/verify-chat-range.mjs new file mode 100644 index 0000000..409262d --- /dev/null +++ b/scripts/verify-chat-range.mjs @@ -0,0 +1,156 @@ +// 验证聊天时间范围判定逻辑(纯函数基准实现)。 +// content.js/popup.js 为 IIFE/非 module,无法被 vitest import —— 本文件是判定逻辑的 +// 基准实现+用例锁定;实现侧照搬函数体。⚠️ 改任一侧必须同步另一侧。 +let pass = 0, fail = 0; +const ok = (name, cond) => { console.log((cond ? 'PASS' : 'FAIL') + ' | ' + name); cond ? pass++ : fail++; }; + +// ---- 基准实现(实现照搬区)---- +// 自然月回退:直接 setMonth(-n) 月末溢出(3-31−1月→"2-31"→3-02),先退月初再钳制天数 +function minusMonths(base, n) { + const d = new Date(base.getFullYear(), base.getMonth(), 1, + base.getHours(), base.getMinutes(), base.getSeconds(), base.getMilliseconds()); + d.setMonth(d.getMonth() - n); + const lastDay = new Date(d.getFullYear(), d.getMonth() + 1, 0).getDate(); + d.setDate(Math.min(base.getDate(), lastDay)); + return d; +} +// popup 范围计算:mode=all|1m|2m|3m|custom → {since,until}(ISO|null) 或 {error} +function computeChatRange(mode, sinceStr, untilStr) { + if (mode === 'all') return { since: null, until: null }; + if (mode === '1m' || mode === '2m' || mode === '3m') { + // 整天边界:since=起始日 00:00:00(完整自然月天数),until=今日 23:59:59.999("至今"取今天整天,与 custom 止=今天一致) + const dayStart = new Date(); dayStart.setHours(0, 0, 0, 0); + const dayEnd = new Date(); dayEnd.setHours(23, 59, 59, 999); + return { since: minusMonths(dayStart, Number(mode[0])).toISOString(), until: dayEnd.toISOString() }; + } + if (mode === 'custom') { + if (!sinceStr || !untilStr) return { error: '请填写自定义起止日期' }; + const s = new Date(sinceStr + 'T00:00:00.000'); // ⚠️ 必须拼时间:裸 date-only 按 UTC 解析差 8h + const u = new Date(untilStr + 'T23:59:59.999'); + if (isNaN(s.getTime()) || isNaN(u.getTime())) return { error: '日期格式错误' }; + if (s.getTime() > u.getTime()) return { error: '起始日期不能晚于截止日期' }; + const uDay = new Date(untilStr + 'T00:00:00.000'); + const t0 = new Date(); t0.setHours(0, 0, 0, 0); + if (uDay.getTime() > t0.getTime()) return { error: '截止日期不能晚于今天' }; + return { since: s.toISOString(), until: u.toISOString() }; + } + return { error: '未知时间范围模式' }; +} +// 范围过滤(null sendTime 防御性保留 —— 宁多勿漏) +function inChatRange(sendTime, sinceMs, untilMs) { + if (sendTime == null) return true; + if (sinceMs != null && sendTime < sinceMs) return false; + if (untilMs != null && sendTime > untilMs) return false; + return true; +} +// 水位读取兼容:旧=纯 string(sendTime 未知) / 新={id,sendTime} +function parseWatermark(v) { + if (v == null || v === '') return null; + try { + const o = JSON.parse(v); + if (o && typeof o === 'object' && o.id != null) return { id: String(o.id), sendTime: Number(o.sendTime) || null }; + } catch (e) { /* 旧格式继续 */ } + return { id: String(v), sendTime: null }; +} +// 水位只前进写入判定 +function shouldWriteWatermark(oldWm, candId, candSendTime) { + if (candId == null || candSendTime == null) return false; + if (oldWm == null) return true; + if (oldWm.sendTime == null) return true; // legacy 升级,允许覆盖一次 + return candSendTime > oldWm.sendTime; +} +// probe 轮断言分类:'fail'(非对象/code≠200/data=null) | 'empty'(空list) | null(正常)。 +// non-JSON/HTTP/关键词由 postForm 抛 RISK,不经此函数(真风控照抛,不降级) +function probeFailKind(json) { + if (!json || typeof json !== 'object') return 'fail'; + if (json.code !== undefined && Number(json.code) !== 200) return 'fail'; + if (json.data == null) return 'fail'; + const list = json.data && json.data.list; + if (!Array.isArray(list) || list.length === 0) return 'empty'; + return null; +} +// ISO → 毫秒,parse 容错(spec §5:非法/空 → null = 降级不限) +function toMs(v) { const n = v ? Date.parse(v) : NaN; return Number.isFinite(n) ? n : null; } + +// ---- 用例 ---- +ok('3-31 − 1月 = 2-28(月末钳制,setMonth 溢出防护)', (() => { + const r = minusMonths(new Date(2026, 2, 31, 15, 30), 1); + return r.getMonth() === 1 && r.getDate() === 28 && r.getHours() === 15; +})()); +ok('8-25 − 1月 = 7-25(普通日期)', (() => { + const r = minusMonths(new Date(2026, 7, 25), 1); + return r.getMonth() === 6 && r.getDate() === 25; +})()); +ok('12-31 − 2月 = 10-31(同年跨月)', (() => { + const r = minusMonths(new Date(2026, 11, 31), 2); + return r.getMonth() === 9 && r.getDate() === 31; +})()); +ok('1-31 − 2月 = 上年 11-30(跨年+钳制)', (() => { + const r = minusMonths(new Date(2026, 0, 31), 2); + return r.getFullYear() === 2025 && r.getMonth() === 10 && r.getDate() === 30; +})()); +ok('闰年:2028-03-31 − 1月 = 2028-02-29', (() => { + const r = minusMonths(new Date(2028, 2, 31), 1); + return r.getMonth() === 1 && r.getDate() === 29; +})()); + +ok('all → 双 null(完全同现状)', computeChatRange('all').since === null && computeChatRange('all').until === null); +ok('1m → since=起始日 00:00:00.000(25~32天前),until=今日 23:59:59.999(整天边界)', (() => { + const r = computeChatRange('1m'); + const s = new Date(r.since), u = new Date(r.until); + const ageDays = (Date.now() - s.getTime()) / 86400e3; + const t0 = new Date(); t0.setHours(0, 0, 0, 0); + return ageDays > 25 && ageDays < 32 + && s.getHours() === 0 && s.getMinutes() === 0 && s.getSeconds() === 0 && s.getMilliseconds() === 0 + && u.getTime() - t0.getTime() === 86399999; // = 今日 0 点 + 全天末毫秒 +})()); +ok('custom 本地时区(since=本地0点,非 UTC)', (() => { + const r = computeChatRange('custom', '2026-08-25', '2026-08-25'); + const s = new Date(r.since); + return s.getMonth() === 7 && s.getDate() === 25 && s.getHours() === 0 && s.getMinutes() === 0; +})()); +ok('custom until=止日 23:59:59.999 本地(含全天)', (() => { + const r = computeChatRange('custom', '2026-08-01', '2026-08-25'); + const u = new Date(r.until); + return u.getDate() === 25 && u.getHours() === 23 && u.getMinutes() === 59 && u.getMilliseconds() === 999; +})()); +ok('custom 起>止 → error', !!computeChatRange('custom', '2026-08-10', '2026-08-01').error); +ok('custom 起为空 → error', !!computeChatRange('custom', '', '2026-08-01').error); +ok('custom 止>今天 → error', !!computeChatRange('custom', '2026-01-01', '2099-01-01').error); +ok('custom 垃圾日期 → error', !!computeChatRange('custom', 'not-a-date', '2026-08-01').error); +ok('未知模式 → error', !!computeChatRange('6m').error); + +const SINCE = Date.parse('2026-07-25T00:00:00'), UNTIL = Date.parse('2026-08-25T23:59:59'); +ok('since 当天消息收(含边界)', inChatRange(SINCE, SINCE, null)); +ok('since 前一天不收', !inChatRange(SINCE - 86400e3, SINCE, null)); +ok('until 末毫秒收(含边界)', inChatRange(UNTIL, null, UNTIL)); +ok('until 次日不收', !inChatRange(UNTIL + 1, null, UNTIL)); +ok('sendTime=null 防御性保留', inChatRange(null, SINCE, UNTIL)); +ok('范围不限(null)全收', inChatRange(1, null, null)); + +ok('新格式水位解析', (() => { const w = parseWatermark('{"id":"a1","sendTime":1787626659384}'); return w.id === 'a1' && w.sendTime === 1787626659384; })()); +ok('旧格式(纯string)→ legacy,sendTime 未知', (() => { const w = parseWatermark('abc123'); return w.id === 'abc123' && w.sendTime === null; })()); +ok('空值(null/undefined/空串)→ null(首次抓取必走)', parseWatermark(null) === null && parseWatermark(undefined) === null && parseWatermark('') === null); +ok('只前进:候选6-30 不覆盖 7-28', !shouldWriteWatermark({ id: 'o', sendTime: Date.parse('2026-07-28') }, 'n', Date.parse('2026-06-30'))); +ok('只前进:候选8-25 覆盖 7-28', shouldWriteWatermark({ id: 'o', sendTime: Date.parse('2026-07-28') }, 'n', Date.parse('2026-08-25'))); +ok('legacy 允许覆盖一次(升级)', shouldWriteWatermark({ id: 'o', sendTime: null }, 'n', Date.parse('2026-06-30'))); +ok('无旧水位 → 写入', shouldWriteWatermark(null, 'n', 1)); +ok('候选缺 sendTime → 不写', !shouldWriteWatermark({ id: 'o', sendTime: 5 }, 'n', null)); + +ok('probe 正常(200+非空) → null', probeFailKind({ code: '200', data: { list: [{}] } }) === null); +ok('probe code≠200 → fail 降级', probeFailKind({ code: '500' }) === 'fail'); +ok('probe data=null → fail 降级', probeFailKind({ code: '200', data: null }) === 'fail'); +ok('probe 非对象 → fail 降级', probeFailKind('ok') === 'fail'); +ok('probe 空 list → empty 降级(实测形态)', probeFailKind({ code: '200', data: { list: [] } }) === 'empty'); +ok('probe list 缺失 → empty 降级', probeFailKind({ code: '200', data: {} }) === 'empty'); + +ok('toMs 非法/空 → null,合法 → 毫秒', toMs('garbage') === null && toMs(null) === null && toMs('2026-08-25T00:00:00Z') === Date.parse('2026-08-25T00:00:00Z')); +// (文档性说明,非行为锁定)until 游标 = untilMs+1:附录B实测语义=返回 sendTime < cursor,恰好覆盖 <= untilMs;真实验证靠 Task 8 #4 SW Network +ok('文档性:until 游标 +1ms 覆盖含止日末毫秒(真实验证靠 Task 8 #4)', (() => { + const untilMs = Date.parse('2026-06-30T23:59:59.999'); + const cursor = untilMs + 1; + return (untilMs < cursor) && !(untilMs + 1 < cursor); +})()); + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/verify-risk-detect.mjs b/scripts/verify-risk-detect.mjs new file mode 100644 index 0000000..6e7959d --- /dev/null +++ b/scripts/verify-risk-detect.mjs @@ -0,0 +1,96 @@ +// 验证 content.js postForm 的风控检测逻辑(纯函数复刻版,须与 content.js 内 postForm 保持一致)。 +// content.js 是 IIFE 闭包 + content_script 不能 ES module import,内部函数无法被 vitest 直接测, +// 故在此复刻逻辑、跑关键 case 佐证正确性。改 content.js 的检测逻辑时务必同步本文件。 +const RISK_KEYWORDS = ['验证码', 'captcha', '滑动验证', '安全验证', 'blocked', '访问频繁', '限制访问', '风控']; +const RISK_TOP_FIELDS = new Set(['msg', 'message', 'errmsg', 'statue', 'errormessage', 'err']); +const hitsRiskKeyword = s => { + const lower = String(s == null ? '' : s).toLowerCase(); + return RISK_KEYWORDS.find(kw => lower.includes(String(kw).toLowerCase())); +}; + +// 复刻 content.js 的账号注销/管控跳过检测(与 hitsSkipKeyword 保持一致) +const SKIP_KEYWORDS = ['对方账号已自行注销', '违反平台规则被管控']; +const hitsSkipKeyword = s => { + const lower = String(s == null ? '' : s).toLowerCase(); + return SKIP_KEYWORDS.find(kw => lower.includes(String(kw).toLowerCase())); +}; + +// 复刻 postForm 的检测分支(去掉 fetch,直接吃响应 text)。返回 {risk} 或 {ok, json}。 +function detect(text) { + let j; + try { j = JSON.parse(text); } + catch (e) { + const kw = hitsRiskKeyword(text); + if (kw) return { risk: 'RISK:关键词(' + kw + ')' }; + return { risk: 'RISK:结构异常(non-JSON)' }; + } + if (j && typeof j === 'object') { + for (const f of Object.keys(j)) { + if (RISK_TOP_FIELDS.has(f.toLowerCase()) && typeof j[f] === 'string') { + const kw = hitsRiskKeyword(j[f]); + if (kw) return { risk: 'RISK:关键词(' + kw + '@' + f + ')' }; + } + } + } + return { ok: true, json: j }; +} + +const cases = [ + { + name: '正常 getOpMessages:正文含 captcha/blocked/验证码 不应误报(核心回归)', + text: JSON.stringify({ code: '200', data: { list: [ + { messageId: '1', content: 'are you blocked? the captcha 验证码 is wrong', sendTime: 1700000000000 }, + ] } }), + expectRisk: false, + }, + { + name: 'HTML 滑块验证页(非JSON,含关键词)→ 应报风控', + text: '请完成滑动验证 security check captcha required', + expectRisk: true, + }, + { + name: 'JSON 顶层 msg="访问频繁" → 应报风控', + text: JSON.stringify({ code: '200', msg: '您访问频繁,请稍后再试' }), + expectRisk: true, + }, + { + name: 'JSON 顶层 errMsg 含 captcha → 应风控(字段名大小写无关)', + text: JSON.stringify({ errMsg: 'captcha verification failed' }), + expectRisk: true, + }, + { + name: '非JSON 无关键词(登录页)→ 结构异常,应暂停', + text: 'please login', + expectRisk: true, + }, + { + name: '正常 querySummary → 不报', + text: JSON.stringify({ code: 200, data: { owner: { accountIdEncrypt: 'x' } } }), + expectRisk: false, + }, +]; + +let pass = 0, fail = 0; +for (const c of cases) { + const r = detect(c.text); + const got = !!r.risk; + const ok = got === c.expectRisk; + console.log((ok ? 'PASS' : 'FAIL') + ' | ' + c.name + (r.risk ? ' -> ' + r.risk : ' -> ok')); + ok ? pass++ : fail++; +} +console.log(`\n--- SKIP 检测(账号注销/管控) ---`); +const skipCases = [ + { name: '账号注销完整提示 → 应命中 SKIP', s: '对方账号已自行注销或者因违反平台规则被管控，当前无法回复。', expect: true }, + { name: '仅“违反平台规则被管控”片段 → 应命中 SKIP', s: '该成员因违反平台规则被管控', expect: true }, + { name: '正常聊天 → 不应命中 SKIP', s: 'hello, please send price', expect: false }, + { name: '含“管控”但非违规语境(管控措施) → 不应误命中', s: '我们公司有严格的管控措施', expect: false }, +]; +for (const c of skipCases) { + const got = !!hitsSkipKeyword(c.s); + const ok = got === c.expect; + console.log((ok ? 'PASS' : 'FAIL') + ' | ' + c.name + ' -> ' + (got ? 'SKIP' : 'ok')); + ok ? pass++ : fail++; +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/tests/config.test.js b/tests/config.test.js new file mode 100644 index 0000000..a44d6d1 --- /dev/null +++ b/tests/config.test.js @@ -0,0 +1,34 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { loadConfig, saveConfig } from '../push/config.js'; + +describe('config', () => { + beforeEach(() => { + // 内存版 chrome.storage.local,验证 load/save 往返(不依赖真实 chrome) + const store = {}; + globalThis.chrome = { + storage: { + local: { + get: vi.fn(async key => { + const k = typeof key === 'string' ? key : key[0]; + return k in store ? { [k]: store[k] } : {}; + }), + set: vi.fn(async obj => { Object.assign(store, obj); }), + }, + }, + }; + }); + + it('loadConfig returns disabled default when unset', async () => { + const cfg = await loadConfig(); + expect(cfg.enabled).toBe(false); + expect(cfg.serverUrl).toBe(''); + }); + + it('saveConfig then loadConfig round-trips', async () => { + await saveConfig({ enabled: true, serverUrl: 'http://x', token: 't' }); + const cfg = await loadConfig(); + expect(cfg.enabled).toBe(true); + expect(cfg.serverUrl).toBe('http://x'); + expect(cfg.token).toBe('t'); + }); +}); diff --git a/tests/payload.test.js b/tests/payload.test.js new file mode 100644 index 0000000..a2d3fee --- /dev/null +++ b/tests/payload.test.js @@ -0,0 +1,97 @@ +import { describe, it, expect } from 'vitest'; +import { buildIngestPayload } from '../push/payload.js'; + +describe('buildIngestPayload', () => { + it('maps record + messages to ingest schema', () => { + const record = { + inquiryId: '13776617920', + manager: 'cara Hu', + createTimeFromApi: '2026-03-13T10:11:50Z', + updateTimeFromApi: '2026-07-13T10:11:50Z', + status: '洽谈中', + msgPreview: 'hi', + isFlagged: false, + customerId: 'cid-1', + detailBuyerName: 'Carlos', + detailCountry: 'PE', + company: 'World Shop', + registerTime: '2025-06-23T00:00:00Z', + behavior: { productViews: '0', validRFQ: '39' }, + }; + const messages = [ + { + messageId: 'm1', + sendTime: '2026-07-13T02:11:50Z', + sender: 'cara Hu', + original: 'hello', + direction: 'outgoing', + }, + ]; + const out = buildIngestPayload(record, messages); + expect(out.inquiry.inquiryId).toBe('13776617920'); + expect(out.inquiry.createTime).toBe('2026-03-13T10:11:50Z'); + expect(out.customer.customerId).toBe('cid-1'); + expect(out.customer.country).toBe('PE'); + expect(out.customer.behavior.productViews).toBe(0); // numeric string -> number + expect(out.customer.behavior.validRfq).toBe(39); + expect(out.messages[0].messageId).toBe('m1'); + expect(out.messages[0].content).toBe('hello'); // original -> content + }); + + it('null customer when no customerId', () => { + const out = buildIngestPayload( + { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + }, + [ + { + messageId: 'm', + sendTime: '2026-01-01T00:00:00Z', + direction: 'outgoing', + content: 'c', + }, + ], + ); + expect(out.customer).toBeNull(); + }); + + it('allows empty messages (chat deferred to later phase)', () => { + const out = buildIngestPayload( + { inquiryId: 'X', createTimeFromApi: '2026-01-01T00:00:00Z', updateTimeFromApi: '2026-01-01T00:00:00Z' }, + [], + ); + expect(out.messages).toEqual([]); + expect(out.inquiry.inquiryId).toBe('X'); + }); + + it('non-ISO createTime (list page "11:58") falls back to current ISO', () => { + const out = buildIngestPayload({ inquiryId: 'X', createTime: '11:58', updateTime: '13:39' }, []); + expect(out.inquiry.createTime).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/); + expect(out.inquiry.updateTime).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/); + }); + + it('empty string -> null for optional fields', () => { + const out = buildIngestPayload( + { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + customerId: 'c', + email: '', + company: '', + }, + [ + { + messageId: 'm', + sendTime: '2026-01-01T00:00:00Z', + direction: 'outgoing', + content: 'c', + }, + ], + ); + expect(out.customer.email).toBeNull(); + expect(out.customer.company).toBeNull(); + }); +}); diff --git a/tests/queue.test.js b/tests/queue.test.js new file mode 100644 index 0000000..0e99572 --- /dev/null +++ b/tests/queue.test.js @@ -0,0 +1,54 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { addItem, getPending, markStatus, remove, getStats } from '../push/queue.js'; + +const DB_NAME = 'inquiry_push'; + +describe('queue', () => { + beforeEach(async () => { + // Each test starts with a clean DB. fake-indexeddb exposes indexedDB + // globally via the setupFiles entry. databases() may be missing on + // some builds, so fall back to the known DB name we use in queue.js. + const dbs = (await indexedDB.databases?.().catch(() => null)) || []; + const names = dbs.length ? dbs.map(d => d.name) : [DB_NAME]; + await Promise.all(names.map(name => new Promise(r => { + if (!name) { r(); return; } + const req = indexedDB.deleteDatabase(name); + req.onsuccess = req.onerror = req.onblocked = () => r(); + }))); + }); + + it('addItem then getPending returns it', async () => { + const id = await addItem({ + payload: { record: { inquiryId: 'X1' }, messages: [{ messageId: 'm1' }] }, + inquiryId: 'X1', + }); + const pending = await getPending(5); + expect(pending.length).toBe(1); + expect(pending[0].id).toBe(id); + expect(pending[0].status).toBe('pending'); + }); + + it('markStatus pending->sent then not in pending', async () => { + const id = await addItem({ + payload: { record: {}, messages: [] }, + inquiryId: 'X', + }); + await markStatus(id, 'sent'); + expect((await getPending(5)).length).toBe(0); + }); + + it('getStats counts by status', async () => { + await addItem({ payload: { record: {}, messages: [] }, inquiryId: 'A' }); + const id2 = await addItem({ payload: { record: {}, messages: [] }, inquiryId: 'B' }); + await markStatus(id2, 'failed', { retries: 8 }); + const stats = await getStats(); + expect(stats.pending).toBe(1); + expect(stats.failed).toBe(1); + }); + + it('remove deletes item', async () => { + const id = await addItem({ payload: { record: {}, messages: [] }, inquiryId: 'A' }); + await remove(id); + expect((await getPending(5)).length).toBe(0); + }); +}); diff --git a/tests/sync.test.js b/tests/sync.test.js new file mode 100644 index 0000000..4feddac --- /dev/null +++ b/tests/sync.test.js @@ -0,0 +1,40 @@ +// 复刻函数漂移防护:content.js / popup.js 的内联函数块必须与 scripts/verify-chat-range.mjs +// 基准照搬区逐字一致。比对前统一 CRLF→LF(content.js 全文件 CRLF,基准/popup 为 LF, +// 行尾风格非漂移信号)并去尾部空白(切块端点的空行/缩进差异非漂移信号); +// 除此之外必须逐字节一致 —— 改基准或实现任一侧后此测试变红 = 你忘了同步另一侧。 +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; + +const read = p => readFileSync(new URL(p, import.meta.url), 'utf8'); +const verify = read('../scripts/verify-chat-range.mjs'); +const content = read('../content.js'); +const popup = read('../popup.js'); + +// CRLF→LF + 去尾部空白(仅此两项豁免,内容仍须逐字一致) +const norm = s => s.replace(/\r\n/g, '\n').replace(/\s+$/, ''); + +// 从 src 中切出 [startMarker, endMarker) 之间的文本(含 start,不含 end) +function sliceBetween(src, startMarker, endMarker) { + const s = src.indexOf(startMarker); + if (s < 0) return null; + const e = endMarker ? src.indexOf(endMarker, s) : -1; + return src.slice(s, e < 0 ? undefined : e); +} + +describe('replica sync drift guard', () => { + it('content.js 5-function block byte-matches verify baseline region', () => { + const base = sliceBetween(verify, '// 范围过滤(null sendTime 防御性保留', '// ---- 用例'); + const inl = sliceBetween(content, '// 范围过滤(null sendTime 防御性保留', '// ============ 安全层:风控关键词'); + expect(base).toBeTruthy(); + expect(inl).toBeTruthy(); + expect(norm(inl)).toBe(norm(base)); + }); + + it('popup.js minusMonths+computeChatRange byte-matches verify baseline region', () => { + const base = sliceBetween(verify, '// 自然月回退:直接 setMonth', '// 范围过滤(null sendTime 防御性保留'); + const inl = sliceBetween(popup, '// 自然月回退:直接 setMonth', 'function syncChatModeUI'); + expect(base).toBeTruthy(); + expect(inl).toBeTruthy(); + expect(norm(inl)).toBe(norm(base)); + }); +}); diff --git a/tests/worker.test.js b/tests/worker.test.js new file mode 100644 index 0000000..e3d100e --- /dev/null +++ b/tests/worker.test.js @@ -0,0 +1,167 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { addItem } from '../push/queue.js'; +import { processQueue } from '../push/worker.js'; + +const DB_NAME = 'inquiry_push'; + +async function dumpAll() { + // Bypass getPending (which filters by status/nextRetry) to inspect raw rows. + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME); + req.onsuccess = () => { + const db = req.result; + const tx = db.transaction('queue', 'readonly').objectStore('queue').getAll(); + tx.onsuccess = () => { db.close(); resolve(tx.result); }; + tx.onerror = () => { db.close(); reject(tx.error); }; + }; + req.onerror = () => reject(req.error); + }); +} + +// worker 无状态:每次 processQueue 自己从 chrome.storage 读配置(模拟已保存的启用配置)。 +// 不再用 configure() 注入 —— 模块级配置会在 SW 休眠重启后丢失(正是 B 的根因)。 +function mockStorage(config) { + globalThis.chrome = { + storage: { local: { get: vi.fn().mockResolvedValue({ push_config: config }) } }, + }; +} + +beforeEach(async () => { + const dbs = (await indexedDB.databases?.().catch(() => null)) || []; + const names = dbs.length ? dbs.map(d => d.name) : [DB_NAME]; + await Promise.all(names.map(name => new Promise(r => { + if (!name) { r(); return; } + const req = indexedDB.deleteDatabase(name); + req.onsuccess = req.onerror = req.onblocked = () => r(); + }))); + mockStorage({ enabled: true, serverUrl: 'http://test', token: 'tok' }); +}); + +describe('worker', () => { + it('pushes when SW resumed without configure() — reads config from storage (regression for B)', async () => { + // SW 休眠重启后模块级配置丢失;worker 必须自己从 storage 读,不能依赖 configure()。 + // beforeEach 已 mock 启用配置,此处故意不调 configure —— 复现并锁定 B 的修复。 + globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200 }); + await addItem({ + payload: { + record: { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + }, + messages: [], + }, + inquiryId: 'X', + }); + const r = await processQueue(); + expect(r.processed).toBe(1); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + }); + + it('POSTs pending, marks sent on 200', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 200 }); + globalThis.fetch = fetchMock; + const id = await addItem({ + payload: { + record: { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + }, + messages: [ + { + messageId: 'm1', + sendTime: '2026-01-01T00:00:00Z', + direction: 'outgoing', + content: 'c', + }, + ], + }, + inquiryId: 'X', + }); + const r = await processQueue(); + expect(r.processed).toBe(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0]).toBe('http://test/api/ingest'); + const all = await dumpAll(); + expect(all.find(x => x.id === id).status).toBe('sent'); + }); + + it('successful push counts as sent in getStats (not silently removed)', async () => { + // 显示 bug 回归:成功推送的记录必须能在 getStats 里看到(sent), + // 否则 popup 永远显示"已推 0"(旧实现成功即 remove → 统计不到)。 + globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200 }); + await addItem({ + payload: { + record: { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + }, + messages: [], + }, + inquiryId: 'X', + }); + await processQueue(); + const { getStats } = await import('../push/queue.js'); + const stats = await getStats(); + expect(stats.sent).toBe(1); + }); + + it('on failure, retries++ and nextRetry in future', async () => { + globalThis.fetch = vi.fn().mockRejectedValue(new Error('network')); + const id = await addItem({ + payload: { + record: { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + }, + messages: [ + { + messageId: 'm', + sendTime: '2026-01-01T00:00:00Z', + direction: 'outgoing', + content: 'c', + }, + ], + }, + inquiryId: 'X', + }); + await processQueue(); + const all = await dumpAll(); + const item = all.find(x => x.id === id); + expect(item.status).toBe('pending'); + expect(item.retries).toBe(1); + expect(item.nextRetry).toBeGreaterThan(Date.now()); + }); + + it('after MAX_RETRIES, mark failed', async () => { + globalThis.fetch = vi.fn().mockRejectedValue(new Error('network')); + const id = await addItem({ + payload: { + record: { + inquiryId: 'X', + createTimeFromApi: '2026-01-01T00:00:00Z', + updateTimeFromApi: '2026-01-01T00:00:00Z', + }, + messages: [ + { + messageId: 'm', + sendTime: '2026-01-01T00:00:00Z', + direction: 'outgoing', + content: 'c', + }, + ], + }, + inquiryId: 'X', + }); + // Pre-seed retries=7 so one more failure pushes it to 8 -> failed. + const { markStatus } = await import('../push/queue.js'); + await markStatus(id, 'pending', { retries: 7, nextRetry: 0 }); + await processQueue(); + const all = await dumpAll(); + const item = all.find(x => x.id === id); + expect(item.status).toBe('failed'); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..8be438d --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + testTimeout: 30000, + // Vitest 4: `test.poolOptions.forks.singleFork` was removed; run all test + // files in a single fork sequentially (maxWorkers=1) to keep fake-indexeddb + // in-memory state consistent across tests. + fileParallelism: false, + setupFiles: ['fake-indexeddb/auto'], + }, +}); diff --git a/使用手册.html b/使用手册.html new file mode 100644 index 0000000..6457595 --- /dev/null +++ b/使用手册.html @@ -0,0 +1,629 @@ + + + + + +阿里询盘数据抓取 · 使用指南 + + + + + + + + +
+
+ + 阿里询盘数据抓取 + v1.0.0 · 使用指南 +
+
+ +
+ + +
+ 图 文 操 作 指 南 +

五步学会用
询盘抓取插件

+

把阿里后台的询盘、客户资料和聊天记录，自动整理成表格、同步到你的数据库。不用懂代码，照着下图做就行。

+
+ 01 安装插件 + 02 配置地址 + 03 开始抓取 + 04 等待完成 + 05 拿到数据 +
+
+ + +
+
+

✓ 开始之前，请准备好

+
    +
  • 用 Chrome 或 Edge 浏览器（360 极速、Brave 等也可以）。
    • +
  • 登录你的阿里主管理员账号——就是能看到所有业务员询盘的那个账号。
    • +
  • 插件文件夹已经放在电脑上（默认 d:\alibb）。
    • +
+
+
+ + +
+
+
1
+
+

安装插件

+ 一次性操作 · 约 1 分钟 +
+
+

把插件装进浏览器，只需做一次。

+ +
+
+
+
+
🔒 chrome://extensions
+
+
+
📂 加载已解压的扩展程序
+
开发者模式
+
+
+
询
+
+ 阿里询盘数据抓取 1.0.0 + 抓取询盘列表、客户资料、行为与聊天记录 +
+
已启用
+
+
+
+
+ +
    +
  1. 在浏览器地址栏输入 chrome://extensions 回车，打开扩展管理页。
    2. +
  2. 打开页面右上角的「开发者模式」开关。
    3. +
  3. 点左上角的「加载已解压的扩展程序」，选择插件文件夹 d:\alibb。
    4. +
  4. 点浏览器右上角的拼图图标 🧩，找到「阿里询盘抓取」，点图钉📌把它固定到工具栏，方便随时打开。
    5. +
+
+ ! + 装好后不要移动或删除 d:\alibb 文件夹，否则插件会失效。日后更新了代码，回这个页面点「↻ 重新加载」即可。 +
+
+ + +
+
+
2
+
+

配置推送地址

+ 一次性操作 · 只需做一次 +
+
+

告诉插件数据要送到哪个后台。地址和口令默认已填好，你只需授权并保存。

+ +
+
+
+
+
插件弹窗 · 后端推送设置
+
+
+
+
+ +
http://219.157.255.213:23010
+
+
+ +
688cbb4b6e880b34…01a39ccbb3c31c4d
+
+
+
🔗 授权域名
+
💾 保存设置
+
+
+
+
+
+ +
    +
  1. 点工具栏的📌插件图标，打开控制面板，往下滚动到「后端推送设置」。
    2. +
  2. 地址和 Token 默认已填好，不用改（除非你的后台换了地址）。
    3. +
  3. 先点「授权域名」——浏览器会弹框问是否允许，点「允许」。
    4. +
  4. 再点「保存设置」，看到绿色「已保存」提示就完成了。
    5. +
+
+ ✓ + 只想导出 Excel 表格、不需要联网同步？那这一步可以直接跳过，去第 3 步开始抓取就行。 +
+
+ + +
+
+
3
+
+

开始抓取

+ 每次使用都从这里开始 +
+
+

选好范围，点一下「开始」就行，剩下全自动。

+ +
+
+
+
+
🔒 message.alibaba.com 询
+
+
+
+
+
询
+
阿里询盘数据抓取message.alibaba.com
+
+
+
就绪等待开始
+
列表+详情
+
前 10 条
+
+ customerId 去重 + 包含聊天记录 +
+
▶ 开始抓取
+
+
+
+
+
+ +
    +
  1. 确认你已经登录阿里，并停在 message.alibaba.com 询盘页面（不在的话，点开始后会自动跳过去）。
    2. +
  2. 抓取模式选「列表+详情」（数据最全）；抓取条数第一次选「前 10 条」先试跑一下。
    3. +
  3. 点橙色的「▶ 开始抓取」按钮，开始！
    4. +
+
+ + +
+
+
4
+
+

等待完成

+ 自动进行 · 你可以走开 +
+
+

看进度条走到 100%。期间要离开就按暂停，下次能接着抓。

+ +
+
+
+
+
🔒 message.alibaba.com 询
+
+
+
+
+
当前 30 / 50 (60%)● 运行中
+
+
+
⏸ 暂停
+
导出 CSV
+
+
推送: 已推 20 / 待推 10 / 失败 0
+
+
+
+
+
+ +
    +
  1. 顶部会显示「当前 30 / 50 (60%)」和橙色进度条，一眼看到进度。
    2. +
  2. 需要离开或休息，就点「⏸ 暂停」。下次打开插件点「恢复」，从断点接着抓，不用重来。
    3. +
  3. 抓取期间不要关闭阿里页面和浏览器，最小化没关系。
    4. +
+
+ + +
+
+
5
+
+

拿到数据

+ 抓完后 · 两种方式都用上了 +
+
+

进度到 100% 后，导出 Excel 表格；同时数据已自动送进后台数据库。

+ +
+
+
+
+
完成 · 你的数据在这里
+
+
+
+
+
+ 导出 CSV 表格 + Excel 直接打开 +
+
→
+
+
🗄️
+ 后台数据库 + 自动同步完成 +
+
+
+
+
+
完成 ✓ 50 / 50全部完成
+
📥 导出 CSV
+
推送: 已推 50 / 待推 0 / 失败 0
+
+
+
+
+
+ +
    +
  1. 点绿色的「📥 导出 CSV」，下载一个 alibaba_inquiry_日期.csv 文件，双击用 Excel 打开就能看，不会乱码。
    2. +
  2. 同时，抓到的数据已经自动传到后台数据库（看「推送」状态条，全部「已推」就对了），不用你操心。
    3. +
+
+ ✓ + 数据量很大时，后台同步可能比抓取慢一些（每小时约 180 条），这是正常的——数据已安全存在本地，会慢慢自动补传完，关电脑也不丢。 +
+
+ + +
+
+

★ 用好它的小窍门

+
    +
  • 第一次先抓 10 条试试，确认没问题再抓全部，避免白忙一场。
    • +
  • 大批量分几次抓，每抓几百条暂停休息几分钟，对账号更安全。
    • +
  • 数据会自动传后台，断网或关电脑都不会丢，下次打开会接着传。
    • +
  • 有些客户的电话、手机可能显示为空——这是阿里页面本身就没填，不是插件漏抓。
    • +
+
+
+ +
+ + + + +