perf(api): 会话/5数字弹窗超时修复——日粒度汇总层 msg_daily_stats
根因(服务器实测):sessions 对 owner 全量 message GROUP BY,重账号 5.2万条 3~5s;staff-chat-stats 全时段窗 6.9s——128MB buffer pool 下被推过前端 15s 超时(ECONNABORTED)。EXPLAIN ANALYZE 证实最优 covering 索引也要 18.9s,瓶颈是磁盘不是查询形状。 修复(预聚合,云客同款思路): - msg_daily_stats 表:每「渠道×账号×会话×上海自然日」一行(收发计数/ 当日最后时间与方向);迁移 a7d3e9f1c2b4(幂等建表) - 写路径:repos.upsert_messages 落库自动重算本批触及的会话×天(所有 抓取路径汇到该函数,一处挂全;失败不阻塞写入) - 读路径:sessions/staff-chat-stats 表有行时读汇总(namedtuple 同形, 端点主体零改动),空表守卫回退旧聚合(部署回填前不空白) - 自愈:CLI daily-stats 全量重建(幂等);full_sync_all.sh 夜间追加第 11 步——顺带修掉尾部 sqlite3 直连 smoke.db 的硬编码(服务器 MySQL 上会炸) - 口径统一:day 一律 Asia/Shanghai Python 计算(旧 from_unixtime 跟随 DB 会话时区与今日数割裂);统计窗口对齐自然日(文档化语义变化) 测试:12 新增(汇总层 8 + 端点判别/对拍 4,判别手法=回填后删光 message 仍出完整数据),全量 512 绿。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -0,0 +1,113 @@
|
||||
"""sessions / staff-chat-stats 的汇总表读路径判别测试(2026-09-03)。
|
||||
|
||||
两个端点在 msg_daily_stats 有行时必须改读汇总表(毫秒级),空表时回退旧
|
||||
实时聚合(部署回填前的守卫)。判别手法:**回填汇总后把 message 表删光**——
|
||||
旧实现会返回空,只有真正读汇总表的实现还能吐出完整数据。
|
||||
|
||||
另配对拍测试:同一批种子,「无汇总(旧路径)」与「有汇总(新路径)」的
|
||||
响应必须逐字段一致。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ChatGroup, Customer, Message
|
||||
from app.storage.summary import recompute_all
|
||||
|
||||
OWNER = "wx_owner_sum"
|
||||
CH = "wechat"
|
||||
_NOW = int(time.time() * 1000)
|
||||
_MS_DAY = 86_400_000
|
||||
|
||||
|
||||
async def _seed(session: AsyncSession) -> None:
|
||||
session.add_all([
|
||||
# t1:今天 1 发 1 收,2 天前还有一条
|
||||
Message(msg_id="s1", channel=CH, owner_account_id=OWNER, talker_id="t1", room_id=None, is_send=True, timestamp=_NOW, type="text", type_raw=1, text="x"),
|
||||
Message(msg_id="s2", channel=CH, owner_account_id=OWNER, talker_id="t1", room_id=None, is_send=False, timestamp=_NOW - 1000, type="text", type_raw=1, text="y"),
|
||||
Message(msg_id="s3", channel=CH, owner_account_id=OWNER, talker_id="t1", room_id=None, is_send=False, timestamp=_NOW - 2 * _MS_DAY, type="text", type_raw=1, text="old"),
|
||||
# t2:只有 5 天前的消息(今日 0)
|
||||
Message(msg_id="s4", channel=CH, owner_account_id=OWNER, talker_id="t2", room_id=None, is_send=True, timestamp=_NOW - 5 * _MS_DAY, type="text", type_raw=1, text="o2"),
|
||||
# 群:今天 2 条
|
||||
Message(msg_id="s5", channel=CH, owner_account_id=OWNER, talker_id="m1", room_id="room1@chatroom", is_send=True, timestamp=_NOW, type="text", type_raw=1, text="g1"),
|
||||
Message(msg_id="s6", channel=CH, owner_account_id=OWNER, talker_id="m2", room_id="room1@chatroom", is_send=False, timestamp=_NOW - 2000, type="text", type_raw=1, text="g2"),
|
||||
Customer(friend_id="t1", channel=CH, owner_account_id=OWNER, friend_nickname="Alice", friend_remark=None),
|
||||
Customer(friend_id="t2", channel=CH, owner_account_id=OWNER, friend_nickname="Bob", friend_remark=None),
|
||||
ChatGroup(room_id="room1@chatroom", channel=CH, owner_account_id=OWNER, room_name="群一号"),
|
||||
])
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _seed_with_stats(session: AsyncSession) -> None:
|
||||
await _seed(session)
|
||||
await recompute_all(session)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _stats_then_wipe(session: AsyncSession) -> None:
|
||||
await _seed_with_stats(session)
|
||||
await session.execute(sa.delete(Message))
|
||||
await session.commit()
|
||||
|
||||
|
||||
# ── sessions ─────────────────────────────────────────────────────────────── #
|
||||
|
||||
|
||||
async def test_sessions_served_from_summary_when_populated(make_client):
|
||||
"""回填后删光 message:响应仍完整 → 数据来自汇总表(旧实现此刻返回空)。"""
|
||||
wiped = await make_client(_stats_then_wipe)
|
||||
r = await wiped.get("/api/sessions", params={"owner_account_id": OWNER, "channel": CH})
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
by = {(s["type"], s.get("talker_id") or s.get("room_id")): s for s in body["sessions"]}
|
||||
assert set(by) == {("friend", "t1"), ("friend", "t2"), ("group", "room1@chatroom")}
|
||||
assert by[("friend", "t1")]["today_count"] == 2
|
||||
assert by[("friend", "t1")]["last_chat_time"] == _NOW
|
||||
assert by[("friend", "t2")]["today_count"] == 0
|
||||
assert by[("group", "room1@chatroom")]["today_count"] == 2
|
||||
assert by[("friend", "t1")]["name"] == "Alice"
|
||||
|
||||
|
||||
async def test_sessions_parity_and_legacy_guard(make_client):
|
||||
"""对拍:无汇总(守卫回退旧路径)响应 == 有汇总(新路径)响应,且逐字段一致。"""
|
||||
legacy = await make_client(_seed)
|
||||
modern = await make_client(_seed_with_stats)
|
||||
params = {"owner_account_id": OWNER, "channel": CH}
|
||||
r1 = await legacy.get("/api/sessions", params=params)
|
||||
r2 = await modern.get("/api/sessions", params=params)
|
||||
assert r1.status_code == r2.status_code == 200
|
||||
assert r1.json() == r2.json()
|
||||
|
||||
|
||||
# ── staff-chat-stats ─────────────────────────────────────────────────────── #
|
||||
|
||||
|
||||
async def test_staff_chat_stats_served_from_summary(make_client):
|
||||
"""全时段窗口(旧实现实测 6.9s 的场景):回填后删光 message 仍出全量数字。"""
|
||||
client = await make_client(_stats_then_wipe)
|
||||
r = await client.get(
|
||||
"/api/staff-chat-stats",
|
||||
params={"owner_account_id": OWNER, "start": 0, "end": _NOW + _MS_DAY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
by = {c["talker_id"]: c for c in body["customers"]}
|
||||
assert set(by) == {"t1", "t2"} # 私聊口径,群不计
|
||||
assert by["t1"]["send_count"] == 1 and by["t1"]["receive_count"] == 2
|
||||
assert by["t1"]["last_time"] == _NOW
|
||||
assert by["t2"]["send_count"] == 1 and by["t2"]["receive_count"] == 0
|
||||
|
||||
|
||||
async def test_staff_chat_stats_parity(make_client):
|
||||
"""宽窗口:旧路径与汇总路径响应一致。"""
|
||||
legacy = await make_client(_seed)
|
||||
modern = await make_client(_seed_with_stats)
|
||||
params = {"owner_account_id": OWNER, "start": 0, "end": _NOW + _MS_DAY}
|
||||
r1 = await legacy.get("/api/staff-chat-stats", params=params)
|
||||
r2 = await modern.get("/api/staff-chat-stats", params=params)
|
||||
assert r1.status_code == r2.status_code == 200
|
||||
assert r1.json() == r2.json()
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Tests for the msg_daily_stats 汇总层(日粒度会话统计)。
|
||||
|
||||
三个被钉死的行为:
|
||||
|
||||
1. **日桶口径**:day 一律 Asia/Shanghai 自然日(Python 计算,不用 SQL 日期
|
||||
函数——旧实现 from_unixtime 跟随 MySQL 会话时区,UTC 容器下和今日数
|
||||
的上海口径不一致,这是顺带修复的旧病)。
|
||||
2. **写路径钩子**::func:`app.storage.repos.upsert_messages` 落库后自动
|
||||
重算本批触及的「会话×天」行——页面抓取器/CLI 手动同步全部路径共用此
|
||||
入口,一处挂全。幂等:同一批重放不产生重复/漂移。
|
||||
3. **全量重算**::func:`app.storage.summary.recompute_all` 从 message 表
|
||||
重建全部日行(夜间自愈/部署回填用),幂等。
|
||||
|
||||
fixtures 用内存 SQLite(测试对 message 的所有写入都走 upsert_messages,
|
||||
从而同时覆盖钩子行为)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.db.base import Base
|
||||
from app.db.models import Message, MsgDailyStats
|
||||
from app.storage.repos import upsert_messages
|
||||
from app.storage.summary import day_key, recompute_all
|
||||
|
||||
_SH = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
||||
def _sh(y, m, d, h, mi=0, s=0) -> int:
|
||||
return int(
|
||||
datetime(y, m, d, h, mi, s, tzinfo=_SH).timestamp() * 1000
|
||||
)
|
||||
|
||||
|
||||
def _utc(y, m, d, h, mi=0, s=0) -> int:
|
||||
return int(
|
||||
datetime(y, m, d, h, mi, s, tzinfo=timezone.utc).timestamp() * 1000
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def session():
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite:///:memory:",
|
||||
poolclass=StaticPool,
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
maker = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with maker() as s:
|
||||
yield s
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ── day_key:上海自然日 ──────────────────────────────────────────────────── #
|
||||
|
||||
|
||||
def test_day_key_shanghai_calendar_day():
|
||||
# 上海 2026-09-03 09:30 → 20260903
|
||||
assert day_key(_sh(2026, 9, 3, 9, 30)) == 20260903
|
||||
|
||||
|
||||
def test_day_key_uses_shanghai_not_utc():
|
||||
# UTC 2026-09-02 16:30 = 上海 2026-09-03 00:30 → 必须归入 0903(旧
|
||||
# from_unixtime 在 UTC 容器里会归入 0902——口径修复点)
|
||||
assert day_key(_utc(2026, 9, 2, 16, 30)) == 20260903
|
||||
|
||||
|
||||
def test_day_key_midnight_boundary():
|
||||
# 上海零点整属于新的一天;前一天 23:59:59 属于前一天
|
||||
assert day_key(_sh(2026, 9, 3, 0, 0, 0)) == 20260903
|
||||
assert day_key(_sh(2026, 9, 2, 23, 59, 59)) == 20260902
|
||||
|
||||
|
||||
# ── 写路径钩子:upsert_messages 自动维护日行 ────────────────────────────── #
|
||||
|
||||
|
||||
def _msg(msg_id, ts, *, owner="wx_a", talker="t1", room=None, is_send=True):
|
||||
return dict(
|
||||
msg_id=msg_id,
|
||||
channel="wechat",
|
||||
owner_account_id=owner,
|
||||
talker_id=talker if room is None else "room_member",
|
||||
room_id=room,
|
||||
is_send=is_send,
|
||||
timestamp=ts,
|
||||
type="text",
|
||||
type_raw=1,
|
||||
text="x",
|
||||
)
|
||||
|
||||
|
||||
async def _stats_rows(session):
|
||||
rows = (await session.execute(select(MsgDailyStats))).scalars().all()
|
||||
return {
|
||||
(r.peer_kind, r.peer_id, r.day): (r.send_cnt, r.recv_cnt, r.last_ts, r.last_is_send)
|
||||
for r in rows
|
||||
}
|
||||
|
||||
|
||||
async def test_upsert_messages_maintains_daily_rows(session):
|
||||
await upsert_messages(
|
||||
session,
|
||||
[
|
||||
_msg("m1", _sh(2026, 9, 3, 10, 0), is_send=True),
|
||||
_msg("m2", _sh(2026, 9, 3, 11, 0), is_send=False),
|
||||
_msg("m3", _sh(2026, 9, 3, 11, 30), is_send=True),
|
||||
],
|
||||
)
|
||||
await session.commit()
|
||||
rows = await _stats_rows(session)
|
||||
assert set(rows) == {("friend", "t1", 20260903)}
|
||||
assert rows[("friend", "t1", 20260903)] == (2, 1, _sh(2026, 9, 3, 11, 30), True)
|
||||
|
||||
|
||||
async def test_upsert_messages_splits_days_and_kinds(session):
|
||||
await upsert_messages(
|
||||
session,
|
||||
[
|
||||
_msg("d1", _sh(2026, 9, 2, 23, 59), talker="t1"),
|
||||
_msg("d2", _sh(2026, 9, 3, 0, 1), talker="t1"),
|
||||
_msg("g1", _sh(2026, 9, 3, 9, 0), room="room1@chatroom", is_send=None),
|
||||
],
|
||||
)
|
||||
await session.commit()
|
||||
rows = await _stats_rows(session)
|
||||
# 跨上海零点的两条消息分属两天;群消息单独一行(room 维度)
|
||||
assert set(rows) == {
|
||||
("friend", "t1", 20260902),
|
||||
("friend", "t1", 20260903),
|
||||
("room", "room1@chatroom", 20260903),
|
||||
}
|
||||
# is_send=NULL 的消息:两个方向都不计(对齐旧聚合 case 语义),但占用行
|
||||
assert rows[("room", "room1@chatroom", 20260903)] == (0, 0, _sh(2026, 9, 3, 9, 0), None)
|
||||
|
||||
|
||||
async def test_upsert_messages_idempotent(session):
|
||||
batch = [
|
||||
_msg("m1", _sh(2026, 9, 3, 10, 0)),
|
||||
_msg("m2", _sh(2026, 9, 3, 11, 0), is_send=False),
|
||||
]
|
||||
await upsert_messages(session, batch)
|
||||
await session.commit()
|
||||
await upsert_messages(session, batch) # 同批重放(跨页重复是常态)
|
||||
await session.commit()
|
||||
rows = await _stats_rows(session)
|
||||
assert len(rows) == 1
|
||||
assert rows[("friend", "t1", 20260903)] == (1, 1, _sh(2026, 9, 3, 11, 0), False)
|
||||
|
||||
|
||||
async def test_upsert_messages_null_timestamp_skipped(session):
|
||||
await upsert_messages(session, [_msg("n1", None)])
|
||||
await session.commit()
|
||||
assert await _stats_rows(session) == {}
|
||||
|
||||
|
||||
# ── 全量重算(夜间自愈/部署回填)────────────────────────────────────────── #
|
||||
|
||||
|
||||
async def test_recompute_all_rebuilds_and_is_idempotent(session):
|
||||
# 先写一批消息(顺带产生日行),再篡改一条日行模拟漂移,全量重算应修复
|
||||
await upsert_messages(
|
||||
session,
|
||||
[_msg("m1", _sh(2026, 9, 3, 10, 0)), _msg("m2", _sh(2026, 9, 3, 11, 0), is_send=False)],
|
||||
)
|
||||
await session.commit()
|
||||
bad = (await session.execute(select(MsgDailyStats))).scalars().one()
|
||||
bad.send_cnt = 999 # 漂移
|
||||
await session.commit()
|
||||
|
||||
n = await recompute_all(session)
|
||||
assert n >= 1
|
||||
rows = await _stats_rows(session)
|
||||
assert rows[("friend", "t1", 20260903)] == (1, 1, _sh(2026, 9, 3, 11, 0), False)
|
||||
|
||||
await recompute_all(session) # 幂等(重建后是全新 ORM 对象,比字段值不比对象)
|
||||
assert await _stats_rows(session) == rows
|
||||
Reference in New Issue
Block a user