yunke/yunke-server/app/storage/summary.py

"""日粒度会话汇总(msg_daily_stats)的维护层。

设计要点(2026-09-03 会话列表超时修复):

- **day 口径**:Asia/Shanghai 自然日,:func:`day_key` 纯 Python 计算——
  不用 SQL 日期函数(from_unixtime 跟随 MySQL 会话时区,UTC 容器下与
  sessions 的上海零点口径割裂;SQLite/MySQL 方言也各自不同)。
- **写路径钩子**::func:`repos.upsert_messages` 落库后调
  :func:`recompute_days` 重算本批触及的「会话×天」(窄时间范围查询,
  毫秒级)。页面抓取器/WA/群消息/CLI 手动同步全部路径最终都走
  upsert_messages,一处挂全。
- **全量重建**::func:`recompute_all` 清空后逐会话重算——夜间自愈
  (02:00 full_sync_all.sh 追加一步)兼部署回填,幂等。

计数语义与旧实时聚合完全一致:is_send=True 计 send、False 计 recv、
NULL 两边不计;timestamp 为 NULL 的消息不参与统计。

首响/回复率三列(2026-09-07 批1,A1+A2)——「消息串」口径(与 8-24
spec 对齐,定案):

- **串**:同会话内客户连续消息(is_send=False)不被员工消息(is_send=True)
  打断的段;is_send=None(unknown)不参与串判定(跳过——不打断也不起串)。
- **首响样本**:每个被回串一个样本 fr_ms = 员工首条回复 ts − 串末条客户
  消息 ts。fr_cnt ≡ threads_answered(被回串数=样本数),不单列。
- **日切近似**(v1):单日内计算,日末未闭合的串计 threads 不计 answered,
  跨夜的串在次日段重新起算——8-24 spec 的 24h 窗口径的简化,读端拼均值
  (fr_sum_ms / threads_answered)不受影响,仅跨夜串的归属日有偏差。
"""

from __future__ import annotations

import datetime as dt
import logging
from zoneinfo import ZoneInfo

import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession

from app.db.models import Message, MsgDailyStats

logger = logging.getLogger(__name__)

_SH = ZoneInfo("Asia/Shanghai")


async def has_rows(session: AsyncSession) -> bool:
    """汇总表是否已有数据(读端点的空表守卫:部署回填前自动回退旧实时聚合)。

    表还不存在(漏跑迁移的升级)与空表同视——回退旧路径,不 500。
    """
    try:
        return bool(
            (
                await session.execute(
                    sa.select(sa.func.count()).select_from(MsgDailyStats)
                )
            ).scalar_one()
        )
    except (sa.exc.OperationalError, sa.exc.ProgrammingError):
        return False


def day_key(ts_ms: int) -> int:
    """ms-epoch → 上海自然日 yyyymmdd。"""
    d = dt.datetime.fromtimestamp(ts_ms / 1000, tz=_SH)
    return d.year * 10000 + d.month * 100 + d.day


def _day_bounds_ms(day: int) -> tuple[int, int]:
    """上海自然日 → [当日 00:00, 次日 00:00) 的 ms 左闭右开区间。"""
    y, m, d = day // 10000, (day // 100) % 100, day % 100
    start = dt.datetime(y, m, d, tzinfo=_SH)
    return int(start.timestamp() * 1000), int(
        (start + dt.timedelta(days=1)).timestamp() * 1000
    )


def _kind_peer(rec: dict) -> tuple[str, str] | None:
    """消息记录 → (peer_kind, peer_id);room 优先(群消息按群归账)。"""
    room = rec.get("room_id")
    if room:
        return ("room", str(room))
    talker = rec.get("talker_id")
    if talker:
        return ("friend", str(talker))
    return None


def touched_day_keys(
    recs: list[dict],
) -> set[tuple[str, str, str, str, int]]:
    """从一批消息记录提取要重算的 (channel, owner, kind, peer, day) 集合。"""
    out: set[tuple[str, str, str, str, int]] = set()
    for rec in recs:
        ts = rec.get("timestamp")
        if not ts:
            continue
        kp = _kind_peer(rec)
        if kp is None:
            continue
        out.add(
            (
                str(rec.get("channel") or ""),
                str(rec.get("owner_account_id") or ""),
                kp[0],
                kp[1],
                day_key(int(ts)),
            )
        )
    return out


def _peer_filter(kind: str, peer: str) -> list:
    """kind/peer → message 表过滤条件(与 sessions/staff-chat-stats 口径一致)。"""
    if kind == "room":
        return [
            Message.room_id == peer,
            Message.room_id.is_not(None),
            Message.room_id != "",
        ]
    return [
        Message.talker_id == peer,
        sa.or_(Message.room_id.is_(None), Message.room_id == ""),
    ]


def _thread_stats(rows: list) -> tuple[int, int, int]:
    """消息串口径三值:(cust_threads, threads_answered, fr_sum_ms)。

    - 串 = 客户连续消息(is_send=False)不被员工消息(is_send=True)打断
      的段;is_send=None(unknown)不参与串判定——跳过,不打断也不起串。
    - 每个被回串一个首响样本 fr_ms = 员工首条回复 ts − 串**末条**客户消息
      ts;fr_sum_ms 为样本时长和(展示用首响时长 = fr_sum_ms/answered)。
    - 日切近似 v1:日末未闭合的串计 threads 不计 answered(跨夜的串在次日
      段重新起算——8-24 spec 的 24h 窗口径的简化)。
    - rows 须先按 ts 升序(调用方 sorted;SQL 无 ORDER BY,乱序会错算串)。
    """
    threads = answered = 0
    fr_sum = 0
    in_thread = False
    thread_last_ts: int | None = None
    for ts, is_send in rows:
        if is_send is False:
            if not in_thread:
                in_thread = True
                threads += 1
            thread_last_ts = int(ts)
        elif is_send is True:
            if in_thread and thread_last_ts is not None:
                answered += 1
                fr_sum += int(ts) - thread_last_ts
                in_thread = False
        # is_send=None:跳过(不参与串判定)
    return threads, answered, fr_sum


async def recompute_days(
    session: AsyncSession,
    keys: set[tuple[str, str, str, str, int]] | list[tuple[str, str, str, str, int]],
) -> int:
    """重算给定 (channel, owner, kind, peer, day) 组合的统计行。

    幂等:先删旧行再按当前 message 真值重建(0 条消息的日子删行不留);
    同批消息跨页重放自然收敛。查询是窄时间范围(单会话单日),毫秒级。
    """
    n = 0
    for channel, owner, kind, peer, day in sorted(keys):
        lo, hi = _day_bounds_ms(day)
        rows = (
            await session.execute(
                sa.select(Message.timestamp, Message.is_send).where(
                    Message.channel == channel,
                    Message.owner_account_id == owner,
                    *_peer_filter(kind, peer),
                    Message.timestamp.is_not(None),
                    Message.timestamp >= lo,
                    Message.timestamp < hi,
                )
            )
        ).all()
        send = recv = 0
        last_ts: int | None = None
        last_is_send: bool | None = None
        # 串算法需要严格时序:先排序(SQL 无 ORDER BY,旧代码遍历中 last_ts
        # 用 >= 兜乱序;排序后该兜底语义不变——max ts 与并列末条方向一致)。
        ordered = sorted(rows, key=lambda r: r[0])
        for ts, is_send in ordered:
            if is_send is True:
                send += 1
            elif is_send is False:
                recv += 1
            if last_ts is None or ts >= last_ts:
                last_ts = int(ts)
                last_is_send = is_send
        threads, answered, fr_sum = _thread_stats(ordered)
        await session.execute(
            sa.delete(MsgDailyStats).where(
                MsgDailyStats.channel == channel,
                MsgDailyStats.owner_account_id == owner,
                MsgDailyStats.peer_kind == kind,
                MsgDailyStats.peer_id == peer,
                MsgDailyStats.day == day,
            )
        )
        if rows:
            session.add(
                MsgDailyStats(
                    channel=channel,
                    owner_account_id=owner,
                    peer_kind=kind,
                    peer_id=peer,
                    day=day,
                    send_cnt=send,
                    recv_cnt=recv,
                    last_ts=last_ts,
                    last_is_send=last_is_send,
                    cust_threads=threads,
                    threads_answered=answered,
                    fr_sum_ms=fr_sum,
                )
            )
        n += 1
    return n


async def recompute_all(session: AsyncSession) -> int:
    """全量重建 msg_daily_stats(夜间自愈 / 部署回填)。

    清空后按会话分组拉取全历史 (timestamp, is_send) 在 Python 侧按上海日
    分桶——一次清空保证绕过钩子的写入(裸 UPDATE/回填脚本)也能被纠正。
    返回重算的会话数。夜间 02:00 窗口跑,全库 86 万行量级预计分钟级。
    """
    await session.execute(sa.delete(MsgDailyStats))
    pair_rows = (
        await session.execute(
            sa.select(
                Message.channel,
                Message.owner_account_id,
                Message.room_id,
                Message.talker_id,
            )
            .where(Message.timestamp.is_not(None))
            .distinct()
        )
    ).all()

    # 四列 DISTINCT 后同一群会按成员 talker 出现多行 → 归并成 (channel,
    # owner, kind, peer) 去重,否则同主键日行会插两遍(IntegrityError)。
    pairs: set[tuple[str, str, str, str]] = set()
    for channel, owner, room, talker in pair_rows:
        if room:
            pairs.add((channel, owner, "room", str(room)))
        elif talker:
            pairs.add((channel, owner, "friend", str(talker)))

    total_pairs = 0
    for channel, owner, kind, peer in sorted(pairs):
        # 拉全会话消息按日重算(而非逐日窄查):全量重建时一次往返更省
        rows = (
            await session.execute(
                sa.select(Message.timestamp, Message.is_send).where(
                    Message.channel == channel,
                    Message.owner_account_id == owner,
                    *_peer_filter(kind, peer),
                )
            )
        ).all()
        days: dict[int, list] = {}
        for ts, is_send in rows:
            days.setdefault(day_key(int(ts)), []).append((ts, is_send))
        for day, day_rows in days.items():
            send = recv = 0
            last_ts = None
            last_is_send = None
            ordered = sorted(day_rows, key=lambda r: r[0])  # 串算法需严格时序
            for ts, is_send in ordered:
                if is_send is True:
                    send += 1
                elif is_send is False:
                    recv += 1
                if last_ts is None or ts >= last_ts:
                    last_ts = int(ts)
                    last_is_send = is_send
            threads, answered, fr_sum = _thread_stats(ordered)
            session.add(
                MsgDailyStats(
                    channel=channel,
                    owner_account_id=owner,
                    peer_kind=kind,
                    peer_id=peer,
                    day=day,
                    send_cnt=send,
                    recv_cnt=recv,
                    last_ts=last_ts,
                    last_is_send=last_is_send,
                    cust_threads=threads,
                    threads_answered=answered,
                    fr_sum_ms=fr_sum,
                )
            )
        total_pairs += 1
        if total_pairs % 500 == 0:
            await session.flush()
    return total_pairs


# 公共别名(2026-09-04):analytics 层复用日界计算,避免各处复制时区逻辑。
day_bounds_ms = _day_bounds_ms