浅念
2175cb0169
- repos: on_duplicate只更新实际插入列(fetched_at等Python-default列, SQLite excluded容忍/MySQL new.*直接报Unknown column) - repos: _coerce_int增加日期字符串→ms epoch分支 + _INT_FIELDS补 last_update/last_chat_time/timestamp/update_time(字符串直通BigInteger) - models: wechat_account.db_size Integer→BigInteger(实测2.17GB超32位上限) - compose: mysql不映射宿主机3306(本机已占;容器内服务名互联) - docs/DEPLOY.md: 服务器部署全指南(架构/步骤/数据迁移/运维命令/注意事项) - 容器实测:Playwright抓取55账号→MySQL upsert全通 Co-Authored-By: Claude <noreply@anthropic.com>
703 lines
32 KiB
Python
703 lines
32 KiB
Python
"""Core ORM models for the yunke data-mirror system.
|
|
|
|
Every model maps an entity observed in the yunke CRM API (see
|
|
``docs/superpowers/specs/2026-08-01-yunke-data-dictionary.md``) to a SQL table.
|
|
Models target **MySQL in production** but keep column types portable so the
|
|
suite can exercise them on **in-memory SQLite** (see ``tests/test_models.py``).
|
|
|
|
Conventions
|
|
-----------
|
|
* SQLAlchemy 2.0 ``Mapped``/``mapped_column`` style; all classes extend
|
|
``app.db.base.Base``.
|
|
* API field names are mapped to ``snake_case`` columns; the original payload is
|
|
preserved verbatim in ``raw_json`` (``sa.JSON``).
|
|
* Common audit columns on every source-mirrored table:
|
|
- ``version`` — source update stamp (string when the API gives a version
|
|
token, BigInteger when it is a ms timestamp).
|
|
- ``raw_json`` — the original API record (``sa.JSON``).
|
|
- ``fetched_at`` — when we pulled the row (server default ``now()``).
|
|
- ``updated_at`` — when the row was last written (server default + onupdate).
|
|
* ``channel`` is only added where it is *meaningful*:
|
|
- Communication tables (``Customer``, ``Message``, ``Media``, ``ChatGroup``,
|
|
``GroupMember``, ``Sms``) use ``channel`` in {``wechat``, ``whatsapp``,
|
|
``sms``}.
|
|
- ``PhoneCall`` uses ``channel`` to mean call-type in {``phone_call``,
|
|
``voice_call``} (calls are not tied to a messaging channel).
|
|
- ``SyncState`` uses ``channel`` to scope a sync cursor (together with
|
|
``entity_type``).
|
|
- ``Staff``, ``Department``, ``WechatAccount``, ``WhatsappAccount``,
|
|
``YkAccount``, ``SyncRun`` carry no ``channel`` column.
|
|
* Primary keys use the natural source id where one exists; otherwise an
|
|
integer surrogate PK is used (``SyncState``, ``YkAccount``, ``SyncRun``).
|
|
* ``GroupMember`` uses a composite PK ``(channel, room_id, member_id)``.
|
|
|
|
The ``Message`` model currently maps to a single physical ``message`` table.
|
|
Real-world volume is expected at millions of rows/month, so the long-term plan
|
|
is time-sharded physical tables named via :func:`message_table_name`. That
|
|
sharding is a LATER task — for now every message row lives in ``message`` and
|
|
the helper below is provided so call-sites can already agree on the naming
|
|
convention.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
from typing import Any
|
|
|
|
import sqlalchemy as sa
|
|
from sqlalchemy import BigInteger, Boolean, DateTime, Index, Integer, String, Text
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
from sqlalchemy.types import JSON
|
|
|
|
from app.db.base import Base
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def message_table_name(channel: str, timestamp: int | dt.datetime) -> str:
|
|
"""Return the conventional sharded table name for a message.
|
|
|
|
Format: ``message_<channel>_<YYYYMM>`` where ``YYYYMM`` is derived from the
|
|
message timestamp. ``timestamp`` may be either:
|
|
|
|
* a Unix epoch in **milliseconds** (``int``) — the form used by the
|
|
``Message.timestamp`` column and the yunke API, or
|
|
* a ``datetime.datetime`` instance (any tz).
|
|
|
|
Examples
|
|
--------
|
|
>>> import datetime as dt
|
|
>>> message_table_name("wechat", 1_700_000_000_000)
|
|
'message_wechat_202311'
|
|
>>> message_table_name("whatsapp", dt.datetime(2024, 1, 5))
|
|
'message_whatsapp_202401'
|
|
"""
|
|
if isinstance(timestamp, dt.datetime):
|
|
ts = timestamp
|
|
else:
|
|
# Treat int as ms-since-epoch (UTC). ``float`` also tolerated.
|
|
ts = dt.datetime.fromtimestamp(int(timestamp) / 1000, tz=dt.UTC)
|
|
return f"message_{channel}_{ts.strftime('%Y%m')}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared column defaults
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _fetched_at() -> Mapped[dt.datetime]:
|
|
return mapped_column(
|
|
DateTime, server_default=sa.func.now(), nullable=False
|
|
)
|
|
|
|
|
|
def _updated_at() -> Mapped[dt.datetime]:
|
|
return mapped_column(
|
|
DateTime,
|
|
server_default=sa.func.now(),
|
|
onupdate=sa.func.now(),
|
|
nullable=False,
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# 1. Organization & people
|
|
# ===========================================================================
|
|
|
|
|
|
class Department(Base):
|
|
"""云客 部门/分组 — source: ``POST /usercenter/dept/tree``."""
|
|
|
|
__tablename__ = "department"
|
|
|
|
dept_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
parent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
path: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
|
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
|
|
class Staff(Base):
|
|
"""员工 — embedded in account lists (``userId`` + ``userName``)."""
|
|
|
|
__tablename__ = "staff"
|
|
|
|
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sale_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
dept_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
is_on_job: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
|
|
# ===========================================================================
|
|
# 2. Staff messaging accounts (channel identity, so no `channel` column)
|
|
# ===========================================================================
|
|
|
|
|
|
class WechatAccount(Base):
|
|
"""员工微信号 — source: ``POST /wechatweb/wxUser/list`` (wechatId PK)."""
|
|
|
|
__tablename__ = "wechat_account"
|
|
|
|
wx_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
nick_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
alias: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
user_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
contacts_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
group_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
send_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
receive_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
is_on_job: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
db_size: Mapped[int | None] = mapped_column(BigInteger, nullable=True) # 字节数可超 32 位 INT(实测 2.17GB,2026-08-21)
|
|
online_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
|
last_update: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
|
|
class WhatsappAccount(Base):
|
|
"""员工 WhatsApp 账号 — source: ``POST /qqweb/WhatsAppUser/businessSalesList``.
|
|
|
|
PK is ``uid`` of the form ``<phone>@s.whatsapp.net``.
|
|
"""
|
|
|
|
__tablename__ = "whatsapp_account"
|
|
|
|
uid: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
user_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
nick_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sale_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sales_bind_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
contacts_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
group_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
talker_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
friend_add_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
is_on_job: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
last_update: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
|
|
# ===========================================================================
|
|
# 3. Communication entities (carry a meaningful `channel` column)
|
|
# ===========================================================================
|
|
|
|
|
|
class Customer(Base):
|
|
"""客户/好友 (WeChat friend OR WhatsApp contact OR SMS peer).
|
|
|
|
Source: ``POST /wechatweb/wxFriends/friend_list/es`` (WeChat) or
|
|
``POST /qqweb/WhatsAppUser/businessFriendList`` (WhatsApp). PK is the
|
|
source friend id: ``friendWxId`` for WeChat, ``<phone>@s.whatsapp.net``
|
|
for WhatsApp.
|
|
"""
|
|
|
|
__tablename__ = "customer"
|
|
|
|
friend_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
channel: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
owner_account_id: Mapped[str | None] = mapped_column(
|
|
String(128), nullable=True
|
|
) # wxId (WeChat) / uid (WhatsApp) — owning staff account
|
|
friend_nickname: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
friend_remark: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
real_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
wechat_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
gender: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
region: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
sale_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
dept_route: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
dept_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
head_image_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
last_chat_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
add_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True) # ms epoch; from raw_json.addTime
|
|
send_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
receive_num: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
message_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
delete_flag: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
is_block: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
friend_state: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
Index(
|
|
"ix_customer_owner_friend",
|
|
"channel",
|
|
"owner_account_id",
|
|
"friend_id",
|
|
),
|
|
)
|
|
|
|
|
|
class ChatGroup(Base):
|
|
"""聊天群 — source: ``POST /wechatweb/chatroom/getChatroomList``.
|
|
|
|
PK ``room_id`` is the source ``chatRoomId`` (ends with ``@chatroom``).
|
|
The raw ``memberIds`` (semicolon-joined wxids) is kept verbatim in
|
|
``member_ids``; the parsed members are mirrored into :class:`GroupMember`.
|
|
"""
|
|
|
|
__tablename__ = "chat_group"
|
|
|
|
room_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
channel: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
room_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
owner_account_id: Mapped[str | None] = mapped_column(
|
|
String(128), nullable=True
|
|
) # wechatId / uid of owning staff account
|
|
owner_account_nickname: Mapped[str | None] = mapped_column(
|
|
String(255), nullable=True
|
|
)
|
|
member_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
member_ids: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
head_image_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
owner_user_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
message_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
own_message_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
loss_members: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
add_members: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
last_chat_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
update_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
Index(
|
|
"ix_chat_group_owner",
|
|
"channel",
|
|
"owner_account_id",
|
|
"room_id",
|
|
),
|
|
)
|
|
|
|
|
|
class GroupMember(Base):
|
|
"""群成员 — composite PK (channel, room_id, member_id).
|
|
|
|
Source: the ``memberIds`` field of a chatroom payload, parsed into one row
|
|
per member. Per-member nickname/avatar may require a separate API call
|
|
(see data-dictionary §9.6) and are therefore nullable.
|
|
"""
|
|
|
|
__tablename__ = "group_member"
|
|
|
|
channel: Mapped[str] = mapped_column(String(16), primary_key=True)
|
|
room_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
member_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
|
nickname: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
avatar: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
display_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
|
|
class Message(Base):
|
|
"""聊天消息 — WeChat ``msgSvrId`` / WhatsApp ``keyId`` as natural PK.
|
|
|
|
Sources:
|
|
* WeChat: ``POST /wechatweb/msg/chats``
|
|
* WhatsApp: ``POST /qqweb/WhatsAppUser/chatRecords``
|
|
|
|
The model currently maps to a **single** ``message`` table. The real
|
|
sharding across ``message_<channel>_<YYYYMM>`` physical tables (see
|
|
:func:`message_table_name`) is a LATER task; this single-table mapping is
|
|
sufficient for ingestion v1.
|
|
"""
|
|
|
|
__tablename__ = "message"
|
|
|
|
msg_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
channel: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
owner_account_id: Mapped[str | None] = mapped_column(
|
|
String(128), nullable=True
|
|
) # wechatId / uid of sender staff account
|
|
talker_id: Mapped[str | None] = mapped_column(
|
|
String(128), nullable=True
|
|
) # talker / friendUid — the other party
|
|
ori_talker: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
room_id: Mapped[str | None] = mapped_column(
|
|
String(128), nullable=True
|
|
) # @chatroom id; NULL/empty = direct message
|
|
is_send: Mapped[bool | None] = mapped_column(Boolean, nullable=True) # mine/isSend
|
|
type: Mapped[str | None] = mapped_column(
|
|
String(32), nullable=True
|
|
) # normalized enum label (text/image/voice/.../unknown); see design §4.3
|
|
type_raw: Mapped[int | None] = mapped_column(
|
|
Integer, nullable=True
|
|
) # original upstream type code (e.g. WeChat 1=text, -2=divider)
|
|
text: Mapped[str | None] = mapped_column(Text, nullable=True) # content/text
|
|
file_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
file_th_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
file_hd_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
md5: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
md5_th: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
md5_hd: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
duration: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
transcription_text: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
img_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
original_image: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
pic_type: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
pay_msg_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
refer_msg: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
sender_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
sender_head_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
timestamp: Mapped[int | None] = mapped_column(BigInteger, nullable=True) # ms epoch
|
|
is_del: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
|
|
# sha256 media_hash of the localized media blob (FK -> Media.media_hash);
|
|
# NULL until sync_message_images downloads it. Populated by the
|
|
# message_images sync pass; rendering prefers /api/media/{this} over the
|
|
# OSS file_url / file_th_url / file_hd_url URLs. For image messages the
|
|
# localized blob is typically the thumbnail (file_th_url) — small + fast
|
|
# + sufficient for chat-timeline display.
|
|
media_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
# Hot query path: per-account, per-talker, time-ordered pagination.
|
|
Index(
|
|
"ix_message_channel_owner_talker_time",
|
|
"channel",
|
|
"owner_account_id",
|
|
"talker_id",
|
|
"timestamp",
|
|
),
|
|
# Group-chat hot path: per-account, per-room, time-ordered pagination.
|
|
# Group messages filter on room_id (not talker_id, which is the *other
|
|
# party* in a DM); without this index MySQL scans the talker index then
|
|
# refilters on room_id, which on high-volume accounts degrades fast.
|
|
Index(
|
|
"ix_message_channel_owner_room_time",
|
|
"channel",
|
|
"owner_account_id",
|
|
"room_id",
|
|
"timestamp",
|
|
),
|
|
)
|
|
|
|
|
|
class Media(Base):
|
|
"""Locally cached media (image / voice / video / file) — keyed by hash.
|
|
|
|
Deduplicates downloads across messages that reference the same file (via
|
|
``md5``). ``local_path`` is set once the blob has been fetched into the
|
|
local media store.
|
|
"""
|
|
|
|
__tablename__ = "media"
|
|
|
|
media_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
channel: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
media_type: Mapped[str | None] = mapped_column(
|
|
String(32), nullable=True
|
|
) # image|voice|video|file|...
|
|
original_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
original_th_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
original_hd_url: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
local_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
mime_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
duration: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
Index("ix_media_type_channel", "media_type", "channel"),
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# 4. Phone calls & SMS (prefix /pc/)
|
|
# ===========================================================================
|
|
|
|
|
|
class PhoneCall(Base):
|
|
"""电话/语音通话记录 — source: ``POST /pc/callLog/list`` (phone calls) and
|
|
``POST /wechatweb/callRecord/queryWeChatVoiceList`` (voice calls).
|
|
|
|
``channel`` here means **call-type** in {``phone_call``, ``voice_call``}
|
|
(calls are not tied to a messaging channel). PK is the source ``id``.
|
|
"""
|
|
|
|
__tablename__ = "phone_call"
|
|
|
|
call_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
channel: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
start_call_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
call_duration: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
ring_time: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
call_status: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
incoming_call: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
user_id_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
department_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
call_number: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sim_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sim_number: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
imei: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
record_file: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
# sha256 media_hash of the localized recording blob (FK -> Media.media_hash);
|
|
# NULL until sync_call_recordings downloads it. Populated by the
|
|
# call_recordings sync pass; playback prefers /api/media/{this} over the
|
|
# OSS record_file URL.
|
|
record_media_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
is_upload: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
download_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
ai_asr: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
qc_status: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
plan_customer_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
Index("ix_phone_call_user_time", "user_id", "start_call_time"),
|
|
)
|
|
|
|
|
|
class VoiceClip(Base):
|
|
"""长语音片段 — source: ``POST /wechatweb/call/list`` (WeChat + WhatsApp
|
|
share the same endpoint, distinguished by the employee's channel).
|
|
|
|
Distinct from :class:`PhoneCall` (which holds ``callLog/list`` phone calls
|
|
and ``queryWeChatVoiceList`` voice-call records): a voice_clip is one of the
|
|
long audio segments listed in the 云客 微信长语音 / WA wavoicecalllist modals.
|
|
PK is the source ``id`` (a cross-employee-globally-unique bigint).
|
|
|
|
``channel`` here is the **messaging channel** in {``wechat``, ``whatsapp``}
|
|
— stamped LOCALLY by the sync orchestrator based on which staff-account
|
|
table owns the scraped ``user_id`` (WechatAccount -> ``wechat``,
|
|
WhatsappAccount -> ``whatsapp``). The upstream endpoint is shared and does
|
|
NOT distinguish the channel in its response, so the channel must come from
|
|
the employee mapping, not the record.
|
|
|
|
``start_time`` / ``end_time`` arrive as ms-epoch **strings** (e.g.
|
|
``"1786072733378"``); the scraper normalizes them to ``int``.
|
|
"""
|
|
|
|
__tablename__ = "voice_clip"
|
|
|
|
voice_clip_id: Mapped[int] = mapped_column(
|
|
BigInteger, primary_key=True, autoincrement=False
|
|
)
|
|
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
user_real_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
wx_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
talker: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
|
is_send: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
company_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
start_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
end_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
oss_file_name: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
|
mp3_duration: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
duration_format: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
|
channel: Mapped[str] = mapped_column(String(16), nullable=False)
|
|
|
|
# sha256 media_hash of the localized audio blob (FK -> Media.media_hash);
|
|
# NULL until sync_voice_clip_audio downloads it. Populated by the
|
|
# voice_clip_audio sync pass; playback prefers /api/media/{this} over the
|
|
# OSS oss_file_name URL.
|
|
media_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
|
|
# LOCAL-only manual match: references customer.friend_id when an operator
|
|
# has associated this clip with a specific customer (好友). NULL = unmatched.
|
|
# The mirror never pushes this back to 云客 — it's our own bookkeeping for
|
|
# the 手动匹配 UI (spec §7). Read paths resolve the customer name via join.
|
|
matched_friend_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (Index("ix_voice_clip_user_time", "user_id", "start_time"),)
|
|
|
|
|
|
class Sms(Base):
|
|
"""短信记录 — source: ``POST /pc/customer/smsLogsList``."""
|
|
|
|
__tablename__ = "sms"
|
|
|
|
sms_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
channel: Mapped[str] = mapped_column(
|
|
String(16), nullable=False, default="sms"
|
|
) # always 'sms' today; kept for symmetry
|
|
customer_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
content: Mapped[str | None] = mapped_column(Text, nullable=True) # full sms text
|
|
content_preview: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
sender: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
direction: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sender_department: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
sender_department_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
receive_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
create_time: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
sim_phone: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
sim_number: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
source: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (Index("ix_sms_user_time", "user_id", "create_time"),)
|
|
|
|
|
|
# ===========================================================================
|
|
# 5. Operational / bookkeeping tables (local; no raw_json/version)
|
|
# ===========================================================================
|
|
|
|
|
|
class SyncState(Base):
|
|
"""Resumable sync cursor — scoped by (entity_type, channel, scope_id).
|
|
|
|
``scope_id`` qualifies the cursor for a sub-stream (e.g. the owning staff
|
|
account id when syncing per-account message streams). ``last_version``
|
|
holds the opaque source update stamp; ``last_timestamp`` the ms epoch.
|
|
"""
|
|
|
|
__tablename__ = "sync_state"
|
|
|
|
state_id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
entity_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
channel: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
|
scope_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
last_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
last_timestamp: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
|
last_synced_at: Mapped[dt.datetime | None] = mapped_column(
|
|
DateTime, nullable=True
|
|
)
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
Index(
|
|
"ix_sync_state_scope",
|
|
"entity_type",
|
|
"channel",
|
|
"scope_id",
|
|
unique=True,
|
|
),
|
|
)
|
|
|
|
|
|
class YkAccount(Base):
|
|
"""云客 CRM login account — credentials / session for the upstream API.
|
|
|
|
The session cookie / password is stored encrypted with
|
|
``Settings.encryption_key`` (Fernet). ``user_id`` + ``company`` together
|
|
identify a yunke session (the two ``userid`` / ``company`` request headers).
|
|
"""
|
|
|
|
__tablename__ = "yk_account"
|
|
|
|
account_id: Mapped[int] = mapped_column(
|
|
Integer, primary_key=True, autoincrement=True
|
|
)
|
|
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
company: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
username: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
session_blob_enc: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
last_login_at: Mapped[dt.datetime | None] = mapped_column(
|
|
DateTime, nullable=True
|
|
)
|
|
is_active: Mapped[bool] = mapped_column(
|
|
Boolean, nullable=False, default=True, server_default=sa.text("1")
|
|
)
|
|
|
|
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (
|
|
Index("ix_yk_account_user_company", "user_id", "company", unique=True),
|
|
)
|
|
|
|
|
|
class SyncRun(Base):
|
|
"""Audit log for one sync execution (start / end / status / counts)."""
|
|
|
|
__tablename__ = "sync_run"
|
|
|
|
run_id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
entity_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
channel: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
|
scope_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
status: Mapped[str] = mapped_column(String(16), nullable=False) # running|success|failed
|
|
started_at: Mapped[dt.datetime] = mapped_column(
|
|
DateTime, server_default=sa.func.now(), nullable=False
|
|
)
|
|
finished_at: Mapped[dt.datetime | None] = mapped_column(DateTime, nullable=True)
|
|
records_fetched: Mapped[int] = mapped_column(
|
|
Integer, nullable=False, default=0, server_default=sa.text("0")
|
|
)
|
|
records_upserted: Mapped[int] = mapped_column(
|
|
Integer, nullable=False, default=0, server_default=sa.text("0")
|
|
)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
|
|
updated_at: Mapped[dt.datetime] = _updated_at()
|
|
|
|
__table_args__ = (Index("ix_sync_run_entity_status", "entity_type", "status"),)
|
|
|
|
|
|
__all__ = [
|
|
"ChatGroup",
|
|
"Customer",
|
|
"Department",
|
|
"GroupMember",
|
|
"Media",
|
|
"Message",
|
|
"PhoneCall",
|
|
"Sms",
|
|
"Staff",
|
|
"SyncRun",
|
|
"SyncState",
|
|
"VoiceClip",
|
|
"WechatAccount",
|
|
"WhatsappAccount",
|
|
"YkAccount",
|
|
"message_table_name",
|
|
]
|