246 lines
8.3 KiB
Python
246 lines
8.3 KiB
Python
|
"""日粒度会话汇总(msg_daily_stats)的维护层。
|
||
|
|
|
||
|
|
设计要点(2026-09-03 会话列表超时修复):
|
||
|
|
|
||
|
|
- **day 口径**:Asia/Shanghai 自然日,:func:`day_key` 纯 Python 计算——
|
||
|
|
不用 SQL 日期函数(from_unixtime 跟随 MySQL 会话时区,UTC 容器下与
|
||
|
|
sessions 的上海零点口径割裂;SQLite/MySQL 方言也各自不同)。
|
||
|
|
- **写路径钩子**::func:`repos.upsert_messages` 落库后调
|
||
|
|
:func:`recompute_days` 重算本批触及的「会话×天」(窄时间范围查询,
|
||
|
|
毫秒级)。页面抓取器/WA/群消息/CLI 手动同步全部路径最终都走
|
||
|
|
upsert_messages,一处挂全。
|
||
|
|
- **全量重建**::func:`recompute_all` 清空后逐会话重算——夜间自愈
|
||
|
|
(02:00 full_sync_all.sh 追加一步)兼部署回填,幂等。
|
||
|
|
|
||
|
|
计数语义与旧实时聚合完全一致:is_send=True 计 send、False 计 recv、
|
||
|
|
NULL 两边不计;timestamp 为 NULL 的消息不参与统计。
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import datetime as dt
|
||
|
|
import logging
|
||
|
|
from zoneinfo import ZoneInfo
|
||
|
|
|
||
|
|
import sqlalchemy as sa
|
||
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
|
|
||
|
|
from app.db.models import Message, MsgDailyStats
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
_SH = ZoneInfo("Asia/Shanghai")
|
||
|
|
|
||
|
|
|
||
|
|
async def has_rows(session: AsyncSession) -> bool:
|
||
|
|
"""汇总表是否已有数据(读端点的空表守卫:部署回填前自动回退旧实时聚合)。"""
|
||
|
|
return bool(
|
||
|
|
(
|
||
|
|
await session.execute(
|
||
|
|
sa.select(sa.func.count()).select_from(MsgDailyStats)
|
||
|
|
)
|
||
|
|
).scalar_one()
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def day_key(ts_ms: int) -> int:
|
||
|
|
"""ms-epoch → 上海自然日 yyyymmdd。"""
|
||
|
|
d = dt.datetime.fromtimestamp(ts_ms / 1000, tz=_SH)
|
||
|
|
return d.year * 10000 + d.month * 100 + d.day
|
||
|
|
|
||
|
|
|
||
|
|
def _day_bounds_ms(day: int) -> tuple[int, int]:
|
||
|
|
"""上海自然日 → [当日 00:00, 次日 00:00) 的 ms 左闭右开区间。"""
|
||
|
|
y, m, d = day // 10000, (day // 100) % 100, day % 100
|
||
|
|
start = dt.datetime(y, m, d, tzinfo=_SH)
|
||
|
|
return int(start.timestamp() * 1000), int(
|
||
|
|
(start + dt.timedelta(days=1)).timestamp() * 1000
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _kind_peer(rec: dict) -> tuple[str, str] | None:
|
||
|
|
"""消息记录 → (peer_kind, peer_id);room 优先(群消息按群归账)。"""
|
||
|
|
room = rec.get("room_id")
|
||
|
|
if room:
|
||
|
|
return ("room", str(room))
|
||
|
|
talker = rec.get("talker_id")
|
||
|
|
if talker:
|
||
|
|
return ("friend", str(talker))
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def touched_day_keys(
|
||
|
|
recs: list[dict],
|
||
|
|
) -> set[tuple[str, str, str, str, int]]:
|
||
|
|
"""从一批消息记录提取要重算的 (channel, owner, kind, peer, day) 集合。"""
|
||
|
|
out: set[tuple[str, str, str, str, int]] = set()
|
||
|
|
for rec in recs:
|
||
|
|
ts = rec.get("timestamp")
|
||
|
|
if not ts:
|
||
|
|
continue
|
||
|
|
kp = _kind_peer(rec)
|
||
|
|
if kp is None:
|
||
|
|
continue
|
||
|
|
out.add(
|
||
|
|
(
|
||
|
|
str(rec.get("channel") or ""),
|
||
|
|
str(rec.get("owner_account_id") or ""),
|
||
|
|
kp[0],
|
||
|
|
kp[1],
|
||
|
|
day_key(int(ts)),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def _peer_filter(kind: str, peer: str) -> list:
|
||
|
|
"""kind/peer → message 表过滤条件(与 sessions/staff-chat-stats 口径一致)。"""
|
||
|
|
if kind == "room":
|
||
|
|
return [
|
||
|
|
Message.room_id == peer,
|
||
|
|
Message.room_id.is_not(None),
|
||
|
|
Message.room_id != "",
|
||
|
|
]
|
||
|
|
return [
|
||
|
|
Message.talker_id == peer,
|
||
|
|
sa.or_(Message.room_id.is_(None), Message.room_id == ""),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
async def recompute_days(
|
||
|
|
session: AsyncSession,
|
||
|
|
keys: set[tuple[str, str, str, str, int]] | list[tuple[str, str, str, str, int]],
|
||
|
|
) -> int:
|
||
|
|
"""重算给定 (channel, owner, kind, peer, day) 组合的统计行。
|
||
|
|
|
||
|
|
幂等:先删旧行再按当前 message 真值重建(0 条消息的日子删行不留);
|
||
|
|
同批消息跨页重放自然收敛。查询是窄时间范围(单会话单日),毫秒级。
|
||
|
|
"""
|
||
|
|
n = 0
|
||
|
|
for channel, owner, kind, peer, day in sorted(keys):
|
||
|
|
lo, hi = _day_bounds_ms(day)
|
||
|
|
rows = (
|
||
|
|
await session.execute(
|
||
|
|
sa.select(Message.timestamp, Message.is_send).where(
|
||
|
|
Message.channel == channel,
|
||
|
|
Message.owner_account_id == owner,
|
||
|
|
*_peer_filter(kind, peer),
|
||
|
|
Message.timestamp.is_not(None),
|
||
|
|
Message.timestamp >= lo,
|
||
|
|
Message.timestamp < hi,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
send = recv = 0
|
||
|
|
last_ts: int | None = None
|
||
|
|
last_is_send: bool | None = None
|
||
|
|
for ts, is_send in rows:
|
||
|
|
if is_send is True:
|
||
|
|
send += 1
|
||
|
|
elif is_send is False:
|
||
|
|
recv += 1
|
||
|
|
if last_ts is None or ts >= last_ts:
|
||
|
|
last_ts = int(ts)
|
||
|
|
last_is_send = is_send
|
||
|
|
await session.execute(
|
||
|
|
sa.delete(MsgDailyStats).where(
|
||
|
|
MsgDailyStats.channel == channel,
|
||
|
|
MsgDailyStats.owner_account_id == owner,
|
||
|
|
MsgDailyStats.peer_kind == kind,
|
||
|
|
MsgDailyStats.peer_id == peer,
|
||
|
|
MsgDailyStats.day == day,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
if rows:
|
||
|
|
session.add(
|
||
|
|
MsgDailyStats(
|
||
|
|
channel=channel,
|
||
|
|
owner_account_id=owner,
|
||
|
|
peer_kind=kind,
|
||
|
|
peer_id=peer,
|
||
|
|
day=day,
|
||
|
|
send_cnt=send,
|
||
|
|
recv_cnt=recv,
|
||
|
|
last_ts=last_ts,
|
||
|
|
last_is_send=last_is_send,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
n += 1
|
||
|
|
return n
|
||
|
|
|
||
|
|
|
||
|
|
async def recompute_all(session: AsyncSession) -> int:
|
||
|
|
"""全量重建 msg_daily_stats(夜间自愈 / 部署回填)。
|
||
|
|
|
||
|
|
清空后按会话分组拉取全历史 (timestamp, is_send) 在 Python 侧按上海日
|
||
|
|
分桶——一次清空保证绕过钩子的写入(裸 UPDATE/回填脚本)也能被纠正。
|
||
|
|
返回重算的会话数。夜间 02:00 窗口跑,全库 86 万行量级预计分钟级。
|
||
|
|
"""
|
||
|
|
await session.execute(sa.delete(MsgDailyStats))
|
||
|
|
pair_rows = (
|
||
|
|
await session.execute(
|
||
|
|
sa.select(
|
||
|
|
Message.channel,
|
||
|
|
Message.owner_account_id,
|
||
|
|
Message.room_id,
|
||
|
|
Message.talker_id,
|
||
|
|
)
|
||
|
|
.where(Message.timestamp.is_not(None))
|
||
|
|
.distinct()
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
|
||
|
|
# 四列 DISTINCT 后同一群会按成员 talker 出现多行 → 归并成 (channel,
|
||
|
|
# owner, kind, peer) 去重,否则同主键日行会插两遍(IntegrityError)。
|
||
|
|
pairs: set[tuple[str, str, str, str]] = set()
|
||
|
|
for channel, owner, room, talker in pair_rows:
|
||
|
|
if room:
|
||
|
|
pairs.add((channel, owner, "room", str(room)))
|
||
|
|
elif talker:
|
||
|
|
pairs.add((channel, owner, "friend", str(talker)))
|
||
|
|
|
||
|
|
total_pairs = 0
|
||
|
|
for channel, owner, kind, peer in sorted(pairs):
|
||
|
|
# 拉全会话消息按日重算(而非逐日窄查):全量重建时一次往返更省
|
||
|
|
rows = (
|
||
|
|
await session.execute(
|
||
|
|
sa.select(Message.timestamp, Message.is_send).where(
|
||
|
|
Message.channel == channel,
|
||
|
|
Message.owner_account_id == owner,
|
||
|
|
*_peer_filter(kind, peer),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
days: dict[int, list] = {}
|
||
|
|
for ts, is_send in rows:
|
||
|
|
days.setdefault(day_key(int(ts)), []).append((ts, is_send))
|
||
|
|
for day, day_rows in days.items():
|
||
|
|
send = recv = 0
|
||
|
|
last_ts = None
|
||
|
|
last_is_send = None
|
||
|
|
for ts, is_send in day_rows:
|
||
|
|
if is_send is True:
|
||
|
|
send += 1
|
||
|
|
elif is_send is False:
|
||
|
|
recv += 1
|
||
|
|
if last_ts is None or ts >= last_ts:
|
||
|
|
last_ts = int(ts)
|
||
|
|
last_is_send = is_send
|
||
|
|
session.add(
|
||
|
|
MsgDailyStats(
|
||
|
|
channel=channel,
|
||
|
|
owner_account_id=owner,
|
||
|
|
peer_kind=kind,
|
||
|
|
peer_id=peer,
|
||
|
|
day=day,
|
||
|
|
send_cnt=send,
|
||
|
|
recv_cnt=recv,
|
||
|
|
last_ts=last_ts,
|
||
|
|
last_is_send=last_is_send,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
total_pairs += 1
|
||
|
|
if total_pairs % 500 == 0:
|
||
|
|
await session.flush()
|
||
|
|
return total_pairs
|