perf(api): 会话/5数字弹窗超时修复——日粒度汇总层 msg_daily_stats
根因(服务器实测):sessions 对 owner 全量 message GROUP BY,重账号 5.2万条 3~5s;staff-chat-stats 全时段窗 6.9s——128MB buffer pool 下被推过前端 15s 超时(ECONNABORTED)。EXPLAIN ANALYZE 证实最优 covering 索引也要 18.9s,瓶颈是磁盘不是查询形状。 修复(预聚合,云客同款思路): - msg_daily_stats 表:每「渠道×账号×会话×上海自然日」一行(收发计数/ 当日最后时间与方向);迁移 a7d3e9f1c2b4(幂等建表) - 写路径:repos.upsert_messages 落库自动重算本批触及的会话×天(所有 抓取路径汇到该函数,一处挂全;失败不阻塞写入) - 读路径:sessions/staff-chat-stats 表有行时读汇总(namedtuple 同形, 端点主体零改动),空表守卫回退旧聚合(部署回填前不空白) - 自愈:CLI daily-stats 全量重建(幂等);full_sync_all.sh 夜间追加第 11 步——顺带修掉尾部 sqlite3 直连 smoke.db 的硬编码(服务器 MySQL 上会炸) - 口径统一:day 一律 Asia/Shanghai Python 计算(旧 from_unixtime 跟随 DB 会话时区与今日数割裂);统计窗口对齐自然日(文档化语义变化) 测试:12 新增(汇总层 8 + 端点判别/对拍 4,判别手法=回填后删光 message 仍出完整数据),全量 512 绿。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -45,30 +45,54 @@ echo "$(date) --- 9c/10 Moments (员工朋友圈) ---"
|
||||
$PY scripts/scrape_via_page.py --entity moments
|
||||
echo "$(date) --- 10/10 Media download (audio+images) ---"
|
||||
$PY scripts/scrape_via_page.py --entity media
|
||||
# 11:会话日汇总全量重建(夜间自愈:绕过 upsert 钩子的写入由它纠正;
|
||||
# sessions/staff-chat-stats 读路径依赖此表,2026-09-03)
|
||||
echo "$(date) --- 11/11 Daily-stats rebuild ---"
|
||||
$PY -m app.sync.cli daily-stats
|
||||
|
||||
echo "=============================================="
|
||||
echo "$(date) === FULL RE-SYNC COMPLETE ==="
|
||||
echo "=============================================="
|
||||
|
||||
$PY -c "
|
||||
import sqlite3
|
||||
con = sqlite3.connect('smoke.db')
|
||||
c = con.cursor()
|
||||
print()
|
||||
print('=== FINAL DATA SUMMARY ===')
|
||||
for label, sql in [
|
||||
('wechat accounts', \"SELECT COUNT(*) FROM wechat_account\"),
|
||||
('whatsapp accounts', \"SELECT COUNT(*) FROM whatsapp_account\"),
|
||||
('wechat customers', \"SELECT COUNT(*) FROM customer WHERE channel='wechat'\"),
|
||||
('whatsapp customers', \"SELECT COUNT(*) FROM customer WHERE channel='whatsapp'\"),
|
||||
('wechat groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='wechat'\"),
|
||||
('whatsapp groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='whatsapp'\"),
|
||||
('wechat messages', \"SELECT COUNT(*) FROM message WHERE channel='wechat'\"),
|
||||
('whatsapp messages', \"SELECT COUNT(*) FROM message WHERE channel='whatsapp'\"),
|
||||
('voice calls', \"SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')\"),
|
||||
('voice clips', 'SELECT COUNT(*) FROM voice_clip'),
|
||||
('moments posts', 'SELECT COUNT(*) FROM moments_post'),
|
||||
('media files', 'SELECT COUNT(*) FROM media'),
|
||||
]:
|
||||
print(f' {label:22s} {c.execute(sql).fetchone()[0]:>6}')
|
||||
"
|
||||
# 终局数据快照:走 app 引擎(DB_URL 可移植,SQLite/MySQL 通吃;原 sqlite3
|
||||
# 直连 smoke.db 的写法在服务器 MySQL 上会炸,2026-09-03 修)
|
||||
$PY - <<'PYEOF'
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
TABLES = [
|
||||
("wechat accounts", "SELECT COUNT(*) FROM wechat_account"),
|
||||
("whatsapp accounts", "SELECT COUNT(*) FROM whatsapp_account"),
|
||||
("wechat customers", "SELECT COUNT(*) FROM customer WHERE channel='wechat'"),
|
||||
("whatsapp customers", "SELECT COUNT(*) FROM customer WHERE channel='whatsapp'"),
|
||||
("wechat groups", "SELECT COUNT(*) FROM chat_group WHERE channel='wechat'"),
|
||||
("whatsapp groups", "SELECT COUNT(*) FROM chat_group WHERE channel='whatsapp'"),
|
||||
("wechat messages", "SELECT COUNT(*) FROM message WHERE channel='wechat'"),
|
||||
("whatsapp messages", "SELECT COUNT(*) FROM message WHERE channel='whatsapp'"),
|
||||
("daily stats rows", "SELECT COUNT(*) FROM msg_daily_stats"),
|
||||
("voice calls", "SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')"),
|
||||
("voice clips", "SELECT COUNT(*) FROM voice_clip"),
|
||||
("moments posts", "SELECT COUNT(*) FROM moments_post"),
|
||||
("media files", "SELECT COUNT(*) FROM media"),
|
||||
]
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
engine = create_async_engine(get_settings().db_url)
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
print()
|
||||
print("=== FINAL DATA SUMMARY ===")
|
||||
for label, sql in TABLES:
|
||||
n = (await conn.execute(text(sql))).scalar_one()
|
||||
print(f" {label:22s} {n:>6}")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
PYEOF
|
||||
|
||||
Reference in New Issue
Block a user