Files
yunke/yunke-server/app/api/routers/staff_chat_stats.py
T
浅念 421998be54 perf(api): 会话/5数字弹窗超时修复——日粒度汇总层 msg_daily_stats
根因(服务器实测):sessions 对 owner 全量 message GROUP BY,重账号 5.2万条
3~5s;staff-chat-stats 全时段窗 6.9s——128MB buffer pool 下被推过前端
15s 超时(ECONNABORTED)。EXPLAIN ANALYZE 证实最优 covering 索引也要
18.9s,瓶颈是磁盘不是查询形状。

修复(预聚合,云客同款思路):
- msg_daily_stats 表:每「渠道×账号×会话×上海自然日」一行(收发计数/
  当日最后时间与方向);迁移 a7d3e9f1c2b4(幂等建表)
- 写路径:repos.upsert_messages 落库自动重算本批触及的会话×天(所有
  抓取路径汇到该函数,一处挂全;失败不阻塞写入)
- 读路径:sessions/staff-chat-stats 表有行时读汇总(namedtuple 同形,
  端点主体零改动),空表守卫回退旧聚合(部署回填前不空白)
- 自愈:CLI daily-stats 全量重建(幂等);full_sync_all.sh 夜间追加第 11
  步——顺带修掉尾部 sqlite3 直连 smoke.db 的硬编码(服务器 MySQL 上会炸)
- 口径统一:day 一律 Asia/Shanghai Python 计算(旧 from_unixtime 跟随
  DB 会话时区与今日数割裂);统计窗口对齐自然日(文档化语义变化)

测试:12 新增(汇总层 8 + 端点判别/对拍 4,判别手法=回填后删光 message
仍出完整数据),全量 512 绿。

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-09-03 16:37:36 +08:00

219 lines
8.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""GET /api/staff-chat-stats — 微信员工 5 数字 + 客户明细。
**双路径(2026-09-03 全时段窗 6.9s 超时修复)**:``msg_daily_stats`` 有行时
读日粒度汇总(窗口对齐到自然日,毫秒级);空表回退旧实时聚合,行为不变。
窗口语义随之统一为「上海自然日」(旧 SQL 日期函数跟随 DB 会话时区,UTC
容器下与今日数口径割裂——顺带修复)。
数据全本地:按 员工微信 wxid × ms-epoch 日期窗 聚合 1:1 私聊(room_id 过滤),
不访问云客。5 汇总数从 customers 在 Python 侧派生(spec §5)。
"""
from __future__ import annotations
import time
from collections import namedtuple
from typing import Annotated
import sqlalchemy as sa
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_session
from app.api.schemas import StaffChatCustomerOut, StaffChatStatsOut, StaffChatStatsSummary
from app.db.models import Customer, Message, MsgDailyStats
from app.storage.summary import day_key, has_rows as _summary_has_rows
router = APIRouter(prefix="/api", tags=["staff-chat-stats"])
SessionDep = Annotated[AsyncSession, Depends(get_session)]
_MS_PER_DAY = 86_400_000
def _private_window(msg, owner: str, start: int, end: int):
"""1:1 私聊 + 窗口过滤条件(channel 固定 wechat),排除 timestamp IS NULL。"""
return [
msg.channel == "wechat",
msg.owner_account_id == owner,
sa.or_(msg.room_id.is_(None), msg.room_id == ""),
msg.timestamp.is_not(None),
msg.timestamp >= start,
msg.timestamp <= end,
]
async def _aggregate(session: AsyncSession, owner: str, start: int, end: int) -> tuple[list, dict[str, bool | None]]:
"""返回 [(talker_id, send, recv, interaction_days, last_time)] + last_is_send map。"""
# 跨库日期提取:从 ms-epoch 取日期。镜像 repos.py 方言检测。
dialect = session.bind.name # 镜像 app/storage/repos.py(.name 等价 .dialect.name)
if dialect == "sqlite":
day_expr = sa.func.date(Message.timestamp / 1000, "unixepoch")
else: # mysql(生产)
day_expr = sa.func.date(sa.func.from_unixtime(Message.timestamp / 1000))
base = _private_window(Message, owner, start, end)
agg_stmt = (
sa.select(
Message.talker_id,
sa.func.sum(sa.case((Message.is_send.is_(True), 1), else_=0)).label("send_count"),
sa.func.sum(sa.case((Message.is_send.is_(False), 1), else_=0)).label("receive_count"),
sa.func.count(sa.func.distinct(day_expr)).label("interaction_days"),
sa.func.max(Message.timestamp).label("last_time"),
)
.where(*base)
.group_by(Message.talker_id)
.order_by(Message.talker_id)
)
agg_rows = (await session.execute(agg_stmt)).all()
# 每客户最后一条消息方向:ROW_NUMBER 窗口取 rn=1(tie-break msg_id 保证确定性)
inner = (
sa.select(
Message.talker_id,
Message.is_send,
sa.func.row_number()
.over(
partition_by=Message.talker_id,
order_by=(Message.timestamp.desc(), Message.msg_id.desc()),
)
.label("rn"),
)
.where(*base)
)
sub = inner.subquery("last_msg")
last_stmt = sa.select(sub.c.talker_id, sub.c.is_send).where(sub.c.rn == 1)
last_map = {tid: is_send for tid, is_send in (await session.execute(last_stmt)).all()}
return agg_rows, last_map
# ── 汇总路径:msg_daily_stats 日行(namedtuple 与旧 Row 同形)────────────── #
_AggRow = namedtuple(
"_AggRow", ["talker_id", "send_count", "receive_count", "interaction_days", "last_time"]
)
async def _aggregate_summary(
session: AsyncSession, owner: str, start: int, end: int
) -> tuple[list, dict[str, bool | None]]:
"""窗口 → [day_key(start), day_key(end)] 自然日区间,按日行 Python 聚合。
interaction_days = 窗内有消息的自然日数;last_time/last_is_send 取窗内
最后一条(last_ts 最大的日行)。窗口对齐自然日是文档化的语义变化
(前端日期选择器本就按天取值)。
"""
rows = (
await session.execute(
sa.select(
MsgDailyStats.peer_id,
MsgDailyStats.day,
MsgDailyStats.send_cnt,
MsgDailyStats.recv_cnt,
MsgDailyStats.last_ts,
MsgDailyStats.last_is_send,
).where(
MsgDailyStats.channel == "wechat",
MsgDailyStats.owner_account_id == owner,
MsgDailyStats.peer_kind == "friend",
MsgDailyStats.day >= day_key(start),
MsgDailyStats.day <= day_key(end),
)
)
).all()
per: dict[str, dict] = {}
for peer, _day, send, recv, last_ts, last_is in rows:
acc = per.setdefault(
peer, {"send": 0, "recv": 0, "days": 0, "last_ts": None, "last_is": None}
)
acc["send"] += int(send or 0)
acc["recv"] += int(recv or 0)
acc["days"] += 1
if acc["last_ts"] is None or (last_ts is not None and last_ts >= acc["last_ts"]):
acc["last_ts"] = last_ts
acc["last_is"] = last_is
agg_rows = [
_AggRow(talker_id=p, send_count=a["send"], receive_count=a["recv"],
interaction_days=a["days"], last_time=a["last_ts"])
for p, a in per.items()
]
last_map = {p: a["last_is"] for p, a in per.items()}
return agg_rows, last_map
async def _customer_names(session: AsyncSession, owner: str, talker_ids: list[str]) -> dict[str, tuple[str, str, str]]:
if not talker_ids:
return {}
stmt = sa.select(
Customer.friend_id,
Customer.friend_remark,
Customer.friend_nickname,
Customer.real_name,
Customer.head_image_url,
).where(
Customer.channel == "wechat",
Customer.owner_account_id == owner,
Customer.friend_id.in_(talker_ids),
)
info: dict[str, tuple[str, str, str]] = {}
for fid, remark, nick, real, head in (await session.execute(stmt)).all():
info[fid] = (remark or nick or real or fid, real or "", head or "")
return info
@router.get("/staff-chat-stats")
async def get_staff_chat_stats(
session: SessionDep,
owner_account_id: Annotated[str, Query(min_length=1)],
start: Annotated[int | None, Query(ge=0)] = None,
end: Annotated[int | None, Query(ge=0)] = None,
) -> StaffChatStatsOut:
"""微信员工沟通统计:5 数字 + 客户明细(全本地 message 聚合,1:1 私聊)。"""
if end is None:
end = int(time.time() * 1000)
if start is None:
start = end - 3 * _MS_PER_DAY # 默认近 3 天
if await _summary_has_rows(session):
agg_rows, last_map = await _aggregate_summary(session, owner_account_id, start, end)
else:
agg_rows, last_map = await _aggregate(session, owner_account_id, start, end)
talker_ids = [r.talker_id for r in agg_rows]
names = await _customer_names(session, owner_account_id, talker_ids)
customers: list[StaffChatCustomerOut] = []
for r in agg_rows:
tid = r.talker_id
last_is_send = last_map.get(tid) # bool | None(None=最后一条方向未知)
customers.append(
StaffChatCustomerOut(
talker_id=tid,
friend_name=(names.get(tid) or (tid, "", ""))[0],
real_name=(names.get(tid) or ("", "", ""))[1] or None,
head_image_url=(names.get(tid) or ("", "", ""))[2] or None,
send_count=int(r.send_count or 0),
receive_count=int(r.receive_count or 0),
interaction_days=int(r.interaction_days or 0),
# True→1, False→0, None→None(方向未知不计入两率;summary 用 ==1/==0 自动排除 None)
last_is_send=int(last_is_send) if last_is_send is not None else None,
last_time=r.last_time,
)
)
summary = StaffChatStatsSummary(
chat_count=len(customers),
effective_count=sum(1 for c in customers if c.send_count >= 1 and c.receive_count >= 1),
follow_count=sum(1 for c in customers if c.interaction_days >= 2),
last_reply_count=sum(1 for c in customers if c.last_is_send == 1),
no_reply_count=sum(1 for c in customers if c.last_is_send == 0),
)
return StaffChatStatsOut(
owner_account_id=owner_account_id,
start=start,
end=end,
summary=summary,
customers=customers,
)