perf(api): 会话/5数字弹窗超时修复——日粒度汇总层 msg_daily_stats
根因(服务器实测):sessions 对 owner 全量 message GROUP BY,重账号 5.2万条 3~5s;staff-chat-stats 全时段窗 6.9s——128MB buffer pool 下被推过前端 15s 超时(ECONNABORTED)。EXPLAIN ANALYZE 证实最优 covering 索引也要 18.9s,瓶颈是磁盘不是查询形状。 修复(预聚合,云客同款思路): - msg_daily_stats 表:每「渠道×账号×会话×上海自然日」一行(收发计数/ 当日最后时间与方向);迁移 a7d3e9f1c2b4(幂等建表) - 写路径:repos.upsert_messages 落库自动重算本批触及的会话×天(所有 抓取路径汇到该函数,一处挂全;失败不阻塞写入) - 读路径:sessions/staff-chat-stats 表有行时读汇总(namedtuple 同形, 端点主体零改动),空表守卫回退旧聚合(部署回填前不空白) - 自愈:CLI daily-stats 全量重建(幂等);full_sync_all.sh 夜间追加第 11 步——顺带修掉尾部 sqlite3 直连 smoke.db 的硬编码(服务器 MySQL 上会炸) - 口径统一:day 一律 Asia/Shanghai Python 计算(旧 from_unixtime 跟随 DB 会话时区与今日数割裂);统计窗口对齐自然日(文档化语义变化) 测试:12 新增(汇总层 8 + 端点判别/对拍 4,判别手法=回填后删光 message 仍出完整数据),全量 512 绿。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -1,10 +1,17 @@
|
||||
"""GET /api/sessions — 账号会话列表(好友+群,今日消息数+最近联系时间),全本地 message 聚合。
|
||||
"""GET /api/sessions — 账号会话列表(好友+群,今日消息数+最近联系时间)。
|
||||
|
||||
镜像 staff_chat_stats 聚合模式(GROUP BY + case 计数 + Python 侧合并昵称/头像)。
|
||||
today_start 钉死 Asia/Shanghai(UTC+8)当日 0 点,不依赖服务器时区。
|
||||
**双路径(2026-09-03 会话列表超时修复)**:
|
||||
- ``msg_daily_stats`` 有行 → 读日粒度汇总(每会话「今日行 + MAX(last_ts)」,
|
||||
毫秒级;旧实时全量聚合在重账号上实测 3~5s,128MB buffer pool 下更被推过
|
||||
前端 15s 超时)。
|
||||
- 空表(部署回填前守卫)→ 回退旧实时聚合,行为不变。
|
||||
|
||||
today_start 钉死 Asia/Shanghai(UTC+8)当日 0 点,不依赖服务器时区;汇总层
|
||||
``day_key`` 同口径。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import namedtuple
|
||||
from datetime import datetime
|
||||
from typing import Annotated
|
||||
from zoneinfo import ZoneInfo
|
||||
@@ -15,7 +22,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_session
|
||||
from app.api.schemas import SessionListOut, SessionOut
|
||||
from app.db.models import ChatGroup, Customer, Message
|
||||
from app.db.models import ChatGroup, Customer, Message, MsgDailyStats
|
||||
from app.storage.summary import day_key, has_rows as _summary_has_rows
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["sessions"])
|
||||
|
||||
@@ -63,6 +71,51 @@ async def _group_rows(session: AsyncSession, owner: str, channel: str, today_sta
|
||||
return (await session.execute(stmt)).all()
|
||||
|
||||
|
||||
# ── 汇总路径:msg_daily_stats 日行(namedtuple 与旧 Row 同形,端点主体零改动)── #
|
||||
|
||||
_FriendRow = namedtuple("_FriendRow", ["talker_id", "last_chat_time", "today_count"])
|
||||
_GroupRow = namedtuple("_GroupRow", ["room_id", "last_chat_time", "today_count"])
|
||||
|
||||
|
||||
async def _summary_rows(
|
||||
session: AsyncSession, owner: str, channel: str, kind: str, today: int
|
||||
) -> dict[str, tuple[int | None, int]]:
|
||||
"""peer → (last_chat_time, today_count),来自日行聚合。
|
||||
|
||||
last = MAX(last_ts)(日行内时间单调,跨日最大即全史最后一条);
|
||||
today = 当日行的 send+recv。
|
||||
"""
|
||||
base = (
|
||||
MsgDailyStats.channel == channel,
|
||||
MsgDailyStats.owner_account_id == owner,
|
||||
MsgDailyStats.peer_kind == kind,
|
||||
)
|
||||
last = {
|
||||
peer: last_ts
|
||||
for peer, last_ts in (
|
||||
await session.execute(
|
||||
sa.select(MsgDailyStats.peer_id, sa.func.max(MsgDailyStats.last_ts))
|
||||
.where(*base)
|
||||
.group_by(MsgDailyStats.peer_id)
|
||||
)
|
||||
).all()
|
||||
}
|
||||
today_counts = {
|
||||
peer: int(cnt or 0)
|
||||
for peer, cnt in (
|
||||
await session.execute(
|
||||
sa.select(
|
||||
MsgDailyStats.peer_id,
|
||||
sa.func.coalesce(sa.func.sum(MsgDailyStats.send_cnt + MsgDailyStats.recv_cnt), 0),
|
||||
)
|
||||
.where(*base, MsgDailyStats.day == today)
|
||||
.group_by(MsgDailyStats.peer_id)
|
||||
)
|
||||
).all()
|
||||
}
|
||||
return {peer: (last_ts, today_counts.get(peer, 0)) for peer, last_ts in last.items()}
|
||||
|
||||
|
||||
async def _meta(session: AsyncSession, owner: str, channel: str, friend_ids: list[str], room_ids: list[str]):
|
||||
"""查 customer/chat_group 表合并昵称/头像;好友行额外带 delete_flag(is_del 过滤用)。
|
||||
|
||||
@@ -105,8 +158,14 @@ async def list_sessions(
|
||||
``pinned``:filehelper 会话置顶标记(对齐云客置顶区),其余好友/群恒 False。
|
||||
"""
|
||||
today_start = _today_start_ms()
|
||||
f_rows = await _friend_rows(session, owner_account_id, channel, today_start)
|
||||
g_rows = await _group_rows(session, owner_account_id, channel, today_start)
|
||||
if await _summary_has_rows(session):
|
||||
f_map = await _summary_rows(session, owner_account_id, channel, "friend", day_key(today_start))
|
||||
g_map = await _summary_rows(session, owner_account_id, channel, "room", day_key(today_start))
|
||||
f_rows = [_FriendRow(talker_id=p, last_chat_time=t, today_count=c) for p, (t, c) in f_map.items()]
|
||||
g_rows = [_GroupRow(room_id=p, last_chat_time=t, today_count=c) for p, (t, c) in g_map.items()]
|
||||
else:
|
||||
f_rows = await _friend_rows(session, owner_account_id, channel, today_start)
|
||||
g_rows = await _group_rows(session, owner_account_id, channel, today_start)
|
||||
fnames, gnames = await _meta(
|
||||
session, owner_account_id, channel, [r.talker_id for r in f_rows], [r.room_id for r in g_rows]
|
||||
)
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
"""GET /api/staff-chat-stats — 微信员工 5 数字 + 客户明细（本地 message 表实时聚合）。
|
||||
"""GET /api/staff-chat-stats — 微信员工 5 数字 + 客户明细。
|
||||
|
||||
**双路径(2026-09-03 全时段窗 6.9s 超时修复)**:``msg_daily_stats`` 有行时
|
||||
读日粒度汇总(窗口对齐到自然日,毫秒级);空表回退旧实时聚合,行为不变。
|
||||
窗口语义随之统一为「上海自然日」(旧 SQL 日期函数跟随 DB 会话时区,UTC
|
||||
容器下与今日数口径割裂——顺带修复)。
|
||||
|
||||
数据全本地：按 员工微信 wxid × ms-epoch 日期窗 聚合 1:1 私聊（room_id 过滤），
|
||||
不访问云客。5 汇总数从 customers 在 Python 侧派生（spec §5）。
|
||||
跨库日期提取（codebase 首例）镜像 app/storage/repos.py 的方言检测写法。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import namedtuple
|
||||
from typing import Annotated
|
||||
|
||||
import sqlalchemy as sa
|
||||
@@ -15,7 +20,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_session
|
||||
from app.api.schemas import StaffChatCustomerOut, StaffChatStatsOut, StaffChatStatsSummary
|
||||
from app.db.models import Customer, Message
|
||||
from app.db.models import Customer, Message, MsgDailyStats
|
||||
from app.storage.summary import day_key, has_rows as _summary_has_rows
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["staff-chat-stats"])
|
||||
|
||||
@@ -81,6 +87,62 @@ async def _aggregate(session: AsyncSession, owner: str, start: int, end: int) ->
|
||||
return agg_rows, last_map
|
||||
|
||||
|
||||
# ── 汇总路径:msg_daily_stats 日行(namedtuple 与旧 Row 同形)────────────── #
|
||||
|
||||
_AggRow = namedtuple(
|
||||
"_AggRow", ["talker_id", "send_count", "receive_count", "interaction_days", "last_time"]
|
||||
)
|
||||
|
||||
|
||||
async def _aggregate_summary(
|
||||
session: AsyncSession, owner: str, start: int, end: int
|
||||
) -> tuple[list, dict[str, bool | None]]:
|
||||
"""窗口 → [day_key(start), day_key(end)] 自然日区间,按日行 Python 聚合。
|
||||
|
||||
interaction_days = 窗内有消息的自然日数;last_time/last_is_send 取窗内
|
||||
最后一条(last_ts 最大的日行)。窗口对齐自然日是文档化的语义变化
|
||||
(前端日期选择器本就按天取值)。
|
||||
"""
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(
|
||||
MsgDailyStats.peer_id,
|
||||
MsgDailyStats.day,
|
||||
MsgDailyStats.send_cnt,
|
||||
MsgDailyStats.recv_cnt,
|
||||
MsgDailyStats.last_ts,
|
||||
MsgDailyStats.last_is_send,
|
||||
).where(
|
||||
MsgDailyStats.channel == "wechat",
|
||||
MsgDailyStats.owner_account_id == owner,
|
||||
MsgDailyStats.peer_kind == "friend",
|
||||
MsgDailyStats.day >= day_key(start),
|
||||
MsgDailyStats.day <= day_key(end),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
per: dict[str, dict] = {}
|
||||
for peer, _day, send, recv, last_ts, last_is in rows:
|
||||
acc = per.setdefault(
|
||||
peer, {"send": 0, "recv": 0, "days": 0, "last_ts": None, "last_is": None}
|
||||
)
|
||||
acc["send"] += int(send or 0)
|
||||
acc["recv"] += int(recv or 0)
|
||||
acc["days"] += 1
|
||||
if acc["last_ts"] is None or (last_ts is not None and last_ts >= acc["last_ts"]):
|
||||
acc["last_ts"] = last_ts
|
||||
acc["last_is"] = last_is
|
||||
|
||||
agg_rows = [
|
||||
_AggRow(talker_id=p, send_count=a["send"], receive_count=a["recv"],
|
||||
interaction_days=a["days"], last_time=a["last_ts"])
|
||||
for p, a in per.items()
|
||||
]
|
||||
last_map = {p: a["last_is"] for p, a in per.items()}
|
||||
return agg_rows, last_map
|
||||
|
||||
|
||||
async def _customer_names(session: AsyncSession, owner: str, talker_ids: list[str]) -> dict[str, tuple[str, str, str]]:
|
||||
if not talker_ids:
|
||||
return {}
|
||||
@@ -114,7 +176,10 @@ async def get_staff_chat_stats(
|
||||
if start is None:
|
||||
start = end - 3 * _MS_PER_DAY # 默认近 3 天
|
||||
|
||||
agg_rows, last_map = await _aggregate(session, owner_account_id, start, end)
|
||||
if await _summary_has_rows(session):
|
||||
agg_rows, last_map = await _aggregate_summary(session, owner_account_id, start, end)
|
||||
else:
|
||||
agg_rows, last_map = await _aggregate(session, owner_account_id, start, end)
|
||||
talker_ids = [r.talker_id for r in agg_rows]
|
||||
names = await _customer_names(session, owner_account_id, talker_ids)
|
||||
|
||||
|
||||
@@ -740,6 +740,46 @@ class SyncRun(Base):
|
||||
__table_args__ = (Index("ix_sync_run_entity_status", "entity_type", "status"),)
|
||||
|
||||
|
||||
class MsgDailyStats(Base):
|
||||
"""日粒度会话统计:每个「渠道×账号×会话×自然日」一行(2026-09-03)。
|
||||
|
||||
sessions(staff-chat 会话列表)与 staff-chat-stats(5 数字弹窗)的预聚
|
||||
合层——这两端点原先每次请求对 owner 全量 message 做 GROUP BY(重账号
|
||||
5.2 万条实测 3~7s,128MB buffer pool 下更被推过前端 15s 超时)。日行由
|
||||
写路径维护:``repos.upsert_messages`` 落库后自动重算本批触及的
|
||||
「会话×天」;夜间 ``recompute_all`` 全量重建自愈(裸 UPDATE/回填脚本
|
||||
等绕过钩子的写入由它兜底)。
|
||||
|
||||
口径:
|
||||
- ``day`` 一律 Asia/Shanghai 自然日(Python 计算,不用 SQL 日期函数——
|
||||
from_unixtime 跟随 MySQL 会话时区,UTC 容器下与今日数口径割裂)。
|
||||
- ``send_cnt``/``recv_cnt`` 仅计 is_send 真/假;NULL 两边都不计(对齐
|
||||
旧聚合的 case 语义)。
|
||||
- ``last_ts``/``last_is_send`` 为该日最后一条消息的时间与方向。
|
||||
"""
|
||||
|
||||
__tablename__ = "msg_daily_stats"
|
||||
|
||||
channel: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
owner_account_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
# friend = 1:1 私聊(peer=talker_id) / room = 群(peer=room_id)
|
||||
peer_kind: Mapped[str] = mapped_column(String(8), primary_key=True)
|
||||
peer_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
# yyyymmdd,如 20260903(上海自然日)
|
||||
day: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
send_cnt: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=sa.text("0")
|
||||
)
|
||||
recv_cnt: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=sa.text("0")
|
||||
)
|
||||
# 当日最后一条消息(ms);当天无有效方向计数的消息也存在(is_send=NULL)
|
||||
last_ts: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)
|
||||
last_is_send: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
|
||||
updated_at: Mapped[dt.datetime] = _updated_at()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChatGroup",
|
||||
"Customer",
|
||||
@@ -748,6 +788,7 @@ __all__ = [
|
||||
"Media",
|
||||
"Message",
|
||||
"MomentsPost",
|
||||
"MsgDailyStats",
|
||||
"PhoneCall",
|
||||
"Sms",
|
||||
"Staff",
|
||||
|
||||
@@ -53,6 +53,7 @@ that one table. Multi-table routing is a LATER task (YAGNI for M1).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
@@ -74,6 +75,9 @@ from app.db.models import (
|
||||
WechatAccount,
|
||||
WhatsappAccount,
|
||||
)
|
||||
from app.storage.summary import recompute_days, touched_day_keys
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"upsert_chat_group",
|
||||
@@ -409,6 +413,20 @@ async def upsert_messages(session: AsyncSession, recs: list[dict[str, Any]]) ->
|
||||
"""
|
||||
for rec in recs:
|
||||
await _upsert_one(session, Message, rec)
|
||||
|
||||
# 落库后维护日粒度汇总(msg_daily_stats):只重算本批触及的「会话×天」,
|
||||
# 窄时间范围查询毫秒级。所有消息写入路径(页面抓取器微信/WA/群、CLI
|
||||
# 手动同步)都汇到本函数,一处挂全。失败不阻塞消息写入——夜间
|
||||
# recompute_all 自愈兜底(2026-09-03 会话列表超时修复)。
|
||||
keys = touched_day_keys(recs)
|
||||
if keys:
|
||||
try:
|
||||
await recompute_days(session, keys)
|
||||
except Exception:
|
||||
_logger.exception(
|
||||
"daily-stats recompute failed (%d keys); nightly rebuild will heal", len(keys)
|
||||
)
|
||||
|
||||
return len(recs)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
"""日粒度会话汇总(msg_daily_stats)的维护层。
|
||||
|
||||
设计要点(2026-09-03 会话列表超时修复):
|
||||
|
||||
- **day 口径**:Asia/Shanghai 自然日,:func:`day_key` 纯 Python 计算——
|
||||
不用 SQL 日期函数(from_unixtime 跟随 MySQL 会话时区,UTC 容器下与
|
||||
sessions 的上海零点口径割裂;SQLite/MySQL 方言也各自不同)。
|
||||
- **写路径钩子**::func:`repos.upsert_messages` 落库后调
|
||||
:func:`recompute_days` 重算本批触及的「会话×天」(窄时间范围查询,
|
||||
毫秒级)。页面抓取器/WA/群消息/CLI 手动同步全部路径最终都走
|
||||
upsert_messages,一处挂全。
|
||||
- **全量重建**::func:`recompute_all` 清空后逐会话重算——夜间自愈
|
||||
(02:00 full_sync_all.sh 追加一步)兼部署回填,幂等。
|
||||
|
||||
计数语义与旧实时聚合完全一致:is_send=True 计 send、False 计 recv、
|
||||
NULL 两边不计;timestamp 为 NULL 的消息不参与统计。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import logging
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Message, MsgDailyStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SH = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
||||
async def has_rows(session: AsyncSession) -> bool:
|
||||
"""汇总表是否已有数据(读端点的空表守卫:部署回填前自动回退旧实时聚合)。"""
|
||||
return bool(
|
||||
(
|
||||
await session.execute(
|
||||
sa.select(sa.func.count()).select_from(MsgDailyStats)
|
||||
)
|
||||
).scalar_one()
|
||||
)
|
||||
|
||||
|
||||
def day_key(ts_ms: int) -> int:
|
||||
"""ms-epoch → 上海自然日 yyyymmdd。"""
|
||||
d = dt.datetime.fromtimestamp(ts_ms / 1000, tz=_SH)
|
||||
return d.year * 10000 + d.month * 100 + d.day
|
||||
|
||||
|
||||
def _day_bounds_ms(day: int) -> tuple[int, int]:
|
||||
"""上海自然日 → [当日 00:00, 次日 00:00) 的 ms 左闭右开区间。"""
|
||||
y, m, d = day // 10000, (day // 100) % 100, day % 100
|
||||
start = dt.datetime(y, m, d, tzinfo=_SH)
|
||||
return int(start.timestamp() * 1000), int(
|
||||
(start + dt.timedelta(days=1)).timestamp() * 1000
|
||||
)
|
||||
|
||||
|
||||
def _kind_peer(rec: dict) -> tuple[str, str] | None:
|
||||
"""消息记录 → (peer_kind, peer_id);room 优先(群消息按群归账)。"""
|
||||
room = rec.get("room_id")
|
||||
if room:
|
||||
return ("room", str(room))
|
||||
talker = rec.get("talker_id")
|
||||
if talker:
|
||||
return ("friend", str(talker))
|
||||
return None
|
||||
|
||||
|
||||
def touched_day_keys(
|
||||
recs: list[dict],
|
||||
) -> set[tuple[str, str, str, str, int]]:
|
||||
"""从一批消息记录提取要重算的 (channel, owner, kind, peer, day) 集合。"""
|
||||
out: set[tuple[str, str, str, str, int]] = set()
|
||||
for rec in recs:
|
||||
ts = rec.get("timestamp")
|
||||
if not ts:
|
||||
continue
|
||||
kp = _kind_peer(rec)
|
||||
if kp is None:
|
||||
continue
|
||||
out.add(
|
||||
(
|
||||
str(rec.get("channel") or ""),
|
||||
str(rec.get("owner_account_id") or ""),
|
||||
kp[0],
|
||||
kp[1],
|
||||
day_key(int(ts)),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _peer_filter(kind: str, peer: str) -> list:
|
||||
"""kind/peer → message 表过滤条件(与 sessions/staff-chat-stats 口径一致)。"""
|
||||
if kind == "room":
|
||||
return [
|
||||
Message.room_id == peer,
|
||||
Message.room_id.is_not(None),
|
||||
Message.room_id != "",
|
||||
]
|
||||
return [
|
||||
Message.talker_id == peer,
|
||||
sa.or_(Message.room_id.is_(None), Message.room_id == ""),
|
||||
]
|
||||
|
||||
|
||||
async def recompute_days(
|
||||
session: AsyncSession,
|
||||
keys: set[tuple[str, str, str, str, int]] | list[tuple[str, str, str, str, int]],
|
||||
) -> int:
|
||||
"""重算给定 (channel, owner, kind, peer, day) 组合的统计行。
|
||||
|
||||
幂等:先删旧行再按当前 message 真值重建(0 条消息的日子删行不留);
|
||||
同批消息跨页重放自然收敛。查询是窄时间范围(单会话单日),毫秒级。
|
||||
"""
|
||||
n = 0
|
||||
for channel, owner, kind, peer, day in sorted(keys):
|
||||
lo, hi = _day_bounds_ms(day)
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Message.timestamp, Message.is_send).where(
|
||||
Message.channel == channel,
|
||||
Message.owner_account_id == owner,
|
||||
*_peer_filter(kind, peer),
|
||||
Message.timestamp.is_not(None),
|
||||
Message.timestamp >= lo,
|
||||
Message.timestamp < hi,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
send = recv = 0
|
||||
last_ts: int | None = None
|
||||
last_is_send: bool | None = None
|
||||
for ts, is_send in rows:
|
||||
if is_send is True:
|
||||
send += 1
|
||||
elif is_send is False:
|
||||
recv += 1
|
||||
if last_ts is None or ts >= last_ts:
|
||||
last_ts = int(ts)
|
||||
last_is_send = is_send
|
||||
await session.execute(
|
||||
sa.delete(MsgDailyStats).where(
|
||||
MsgDailyStats.channel == channel,
|
||||
MsgDailyStats.owner_account_id == owner,
|
||||
MsgDailyStats.peer_kind == kind,
|
||||
MsgDailyStats.peer_id == peer,
|
||||
MsgDailyStats.day == day,
|
||||
)
|
||||
)
|
||||
if rows:
|
||||
session.add(
|
||||
MsgDailyStats(
|
||||
channel=channel,
|
||||
owner_account_id=owner,
|
||||
peer_kind=kind,
|
||||
peer_id=peer,
|
||||
day=day,
|
||||
send_cnt=send,
|
||||
recv_cnt=recv,
|
||||
last_ts=last_ts,
|
||||
last_is_send=last_is_send,
|
||||
)
|
||||
)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
async def recompute_all(session: AsyncSession) -> int:
|
||||
"""全量重建 msg_daily_stats(夜间自愈 / 部署回填)。
|
||||
|
||||
清空后按会话分组拉取全历史 (timestamp, is_send) 在 Python 侧按上海日
|
||||
分桶——一次清空保证绕过钩子的写入(裸 UPDATE/回填脚本)也能被纠正。
|
||||
返回重算的会话数。夜间 02:00 窗口跑,全库 86 万行量级预计分钟级。
|
||||
"""
|
||||
await session.execute(sa.delete(MsgDailyStats))
|
||||
pair_rows = (
|
||||
await session.execute(
|
||||
sa.select(
|
||||
Message.channel,
|
||||
Message.owner_account_id,
|
||||
Message.room_id,
|
||||
Message.talker_id,
|
||||
)
|
||||
.where(Message.timestamp.is_not(None))
|
||||
.distinct()
|
||||
)
|
||||
).all()
|
||||
|
||||
# 四列 DISTINCT 后同一群会按成员 talker 出现多行 → 归并成 (channel,
|
||||
# owner, kind, peer) 去重,否则同主键日行会插两遍(IntegrityError)。
|
||||
pairs: set[tuple[str, str, str, str]] = set()
|
||||
for channel, owner, room, talker in pair_rows:
|
||||
if room:
|
||||
pairs.add((channel, owner, "room", str(room)))
|
||||
elif talker:
|
||||
pairs.add((channel, owner, "friend", str(talker)))
|
||||
|
||||
total_pairs = 0
|
||||
for channel, owner, kind, peer in sorted(pairs):
|
||||
# 拉全会话消息按日重算(而非逐日窄查):全量重建时一次往返更省
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Message.timestamp, Message.is_send).where(
|
||||
Message.channel == channel,
|
||||
Message.owner_account_id == owner,
|
||||
*_peer_filter(kind, peer),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
days: dict[int, list] = {}
|
||||
for ts, is_send in rows:
|
||||
days.setdefault(day_key(int(ts)), []).append((ts, is_send))
|
||||
for day, day_rows in days.items():
|
||||
send = recv = 0
|
||||
last_ts = None
|
||||
last_is_send = None
|
||||
for ts, is_send in day_rows:
|
||||
if is_send is True:
|
||||
send += 1
|
||||
elif is_send is False:
|
||||
recv += 1
|
||||
if last_ts is None or ts >= last_ts:
|
||||
last_ts = int(ts)
|
||||
last_is_send = is_send
|
||||
session.add(
|
||||
MsgDailyStats(
|
||||
channel=channel,
|
||||
owner_account_id=owner,
|
||||
peer_kind=kind,
|
||||
peer_id=peer,
|
||||
day=day,
|
||||
send_cnt=send,
|
||||
recv_cnt=recv,
|
||||
last_ts=last_ts,
|
||||
last_is_send=last_is_send,
|
||||
)
|
||||
)
|
||||
total_pairs += 1
|
||||
if total_pairs % 500 == 0:
|
||||
await session.flush()
|
||||
return total_pairs
|
||||
@@ -89,6 +89,18 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
help="Suppress per-run JSON progress lines (errors still logged).",
|
||||
)
|
||||
|
||||
# ``daily-stats`` ---------------------------------------------------------
|
||||
p_stats = sub.add_parser(
|
||||
"daily-stats",
|
||||
help="Rebuild msg_daily_stats from the message table.",
|
||||
description=(
|
||||
"Full rebuild of the daily conversation-summary table "
|
||||
"(msg_daily_stats). Used as the nightly self-heal step in "
|
||||
"scripts/full_sync_all.sh and as the one-off deploy backfill "
|
||||
"(see docs/DEPLOY.md). Idempotent."
|
||||
),
|
||||
)
|
||||
|
||||
# ``scheduler`` -----------------------------------------------------------
|
||||
p_sched = sub.add_parser(
|
||||
"scheduler",
|
||||
@@ -274,6 +286,22 @@ def _print_sync_summary(entity: str, results: list[Any]) -> None:
|
||||
print(line, flush=True)
|
||||
|
||||
|
||||
async def _cmd_daily_stats(args: argparse.Namespace) -> int:
|
||||
"""全量重建 msg_daily_stats(夜间自愈 / 部署回填,幂等)。"""
|
||||
settings = get_settings()
|
||||
engine, session_maker = await _init_db(settings.db_url)
|
||||
try:
|
||||
from app.storage.summary import recompute_all
|
||||
|
||||
async with session_maker() as session:
|
||||
pairs = await recompute_all(session)
|
||||
await session.commit()
|
||||
print(f"[daily-stats] rebuilt {pairs} conversation pairs")
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI entry point. Returns a process exit code."""
|
||||
# Configure logging so the scheduler's JSON line + the orchestrator's
|
||||
@@ -292,6 +320,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return asyncio.run(_cmd_sync(args))
|
||||
if args.command == "scheduler":
|
||||
return asyncio.run(_cmd_scheduler(args))
|
||||
if args.command == "daily-stats":
|
||||
return asyncio.run(_cmd_daily_stats(args))
|
||||
except KeyboardInterrupt:
|
||||
# So `python -m app.sync.cli scheduler` exits cleanly on Ctrl-C.
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user