feat(db): moments_post 表+upsert(朋友圈实体,PK=云客data.id,create_time统一ms)
- MomentsPost 模型:moments_post 表,PK=id(云客 getFriendsterInfo data.id), wechat_id/create_time 建索引,praise/comment/image 存 JSON 列 - 迁移 524da8d2d597(手写 create_table,已应用到 smoke.db 并在临时 SQLite 全链路 upgrade/downgrade 验证) - repos.upsert_moments_posts:批量幂等 upsert;附带修 _build_column_values 对 JSON 列表的误 join(标量列 join 保持不变,dept_route 旧测试仍过) - 修复 3 个 chat_group 旧测试缺 owner_account_id(2026-08-24 复合主键遗留) - 全套 430 测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -26,7 +26,7 @@ Conventions
|
||||
- ``SyncState`` uses ``channel`` to scope a sync cursor (together with
|
||||
``entity_type``).
|
||||
- ``Staff``, ``Department``, ``WechatAccount``, ``WhatsappAccount``,
|
||||
``YkAccount``, ``SyncRun`` carry no ``channel`` column.
|
||||
``YkAccount``, ``SyncRun``, ``MomentsPost`` carry no ``channel`` column.
|
||||
* Primary keys use the natural source id where one exists; otherwise an
|
||||
integer surrogate PK is used (``SyncState``, ``YkAccount``, ``SyncRun``).
|
||||
* ``GroupMember`` uses a composite PK ``(channel, room_id, member_id)``.
|
||||
@@ -592,6 +592,33 @@ class Sms(Base):
|
||||
__table_args__ = (Index("ix_sms_user_time", "user_id", "create_time"),)
|
||||
|
||||
|
||||
class MomentsPost(Base):
|
||||
"""员工朋友圈帖子 — source: ``POST /wechatweb/wechatFriendster/getFriendsterInfo``.
|
||||
|
||||
PK = 云客 ``data.id``。``create_time`` 统一 **ms epoch**（云客返回秒，
|
||||
scraper ×1000）。WeChat-only 实体（无 ``channel`` 列），所属员工微信号
|
||||
由 ``wechat_id`` 标识。
|
||||
"""
|
||||
|
||||
__tablename__ = "moments_post"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
wechat_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
sns_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
author_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
author_wxid: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
author_avatar: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
content: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
type_raw: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
create_time: Mapped[int | None] = mapped_column(BigInteger, index=True)
|
||||
praise_json: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
|
||||
comment_json: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
|
||||
image_urls: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
||||
raw_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||
fetched_at: Mapped[dt.datetime] = _fetched_at()
|
||||
updated_at: Mapped[dt.datetime] = _updated_at()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 5. Operational / bookkeeping tables (local; no raw_json/version)
|
||||
# ===========================================================================
|
||||
@@ -696,6 +723,7 @@ __all__ = [
|
||||
"GroupMember",
|
||||
"Media",
|
||||
"Message",
|
||||
"MomentsPost",
|
||||
"PhoneCall",
|
||||
"Sms",
|
||||
"Staff",
|
||||
|
||||
@@ -36,6 +36,10 @@ dict (see :func:`_coerce_bool`, :func:`_coerce_int`):
|
||||
* ``raw_json``: the original record dict, preserved verbatim. When the input
|
||||
already carries a ``raw_json`` key (scraper output does), that value wins;
|
||||
otherwise the input dict itself is stored.
|
||||
* JSON-typed list columns (``MomentsPost.praise_json`` / ``comment_json`` /
|
||||
``image_urls``): list values are kept verbatim (stored as JSON arrays). The
|
||||
list -> ``";"``-join coercion only applies to lists headed for SCALAR
|
||||
String columns (e.g. ``Customer.dept_route``).
|
||||
|
||||
Sharding note
|
||||
-------------
|
||||
@@ -60,6 +64,7 @@ from app.db.models import (
|
||||
GroupMember,
|
||||
Media,
|
||||
Message,
|
||||
MomentsPost,
|
||||
PhoneCall,
|
||||
Sms,
|
||||
Staff,
|
||||
@@ -75,6 +80,7 @@ __all__ = [
|
||||
"upsert_department",
|
||||
"upsert_media",
|
||||
"upsert_messages",
|
||||
"upsert_moments_posts",
|
||||
"upsert_phone_call",
|
||||
"upsert_sms",
|
||||
"upsert_staff",
|
||||
@@ -220,12 +226,16 @@ def _build_column_values(
|
||||
on unknown columns).
|
||||
* Boolean / int-as-str fields are coerced via :func:`_coerce_bool` /
|
||||
:func:`_coerce_int`.
|
||||
* List values headed for SCALAR columns (e.g. ``Customer.dept_route``) are
|
||||
joined into a ``";"``-separated string so the bind succeeds; list values
|
||||
headed for JSON columns (e.g. ``MomentsPost.praise_json`` /
|
||||
``image_urls``) are kept verbatim as JSON arrays.
|
||||
* ``raw_json`` defaults to the entire input dict when not explicitly
|
||||
provided, so the verbatim upstream record is always recoverable.
|
||||
"""
|
||||
col_names = {c.name for c in model.__table__.columns}
|
||||
cols = {c.name: c for c in model.__table__.columns}
|
||||
values: dict[str, Any] = {}
|
||||
for col_name in col_names:
|
||||
for col_name, col in cols.items():
|
||||
if col_name in rec:
|
||||
v = rec[col_name]
|
||||
if col_name in _BOOL_FIELDS:
|
||||
@@ -235,14 +245,15 @@ def _build_column_values(
|
||||
# Some upstream fields are lists (e.g. Customer.deptRoute is a
|
||||
# list of dept ids) but map to scalar String columns — join so the
|
||||
# bind succeeds and the data is still recoverable (raw_json keeps
|
||||
# the original list).
|
||||
if isinstance(v, list):
|
||||
# the original list). JSON-typed columns keep the list verbatim
|
||||
# (MomentsPost.praise_json / comment_json / image_urls).
|
||||
if isinstance(v, list) and not isinstance(col.type, sa.JSON):
|
||||
v = ";".join(str(x) for x in v)
|
||||
values[col_name] = v
|
||||
# Preserve the verbatim source record. If the scraper already set raw_json
|
||||
# (it does), that wins; otherwise the input dict itself is the source of
|
||||
# truth.
|
||||
if "raw_json" in col_names and "raw_json" not in values:
|
||||
if "raw_json" in cols and "raw_json" not in values:
|
||||
values["raw_json"] = rec
|
||||
# Sanitize lone surrogate chars (e.g. \udf1e from emoji fragments in the
|
||||
# upstream API) that break SQLite/MySQL UTF-8 encoding. Replaces them with
|
||||
@@ -602,6 +613,28 @@ async def upsert_sms(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
await _upsert_one(session, Sms, rec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MomentsPost (员工朋友圈) upsert — 员工聊天窗口 Phase 1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def upsert_moments_posts(session: AsyncSession, recs: list[dict[str, Any]]) -> int:
|
||||
"""Batch idempotent upsert for :class:`MomentsPost` (PK = ``id``).
|
||||
|
||||
Same ``id`` twice -> one row, with non-PK fields refreshed from the later
|
||||
record. ``create_time`` goes through the ``_INT_FIELDS`` int coercion (it
|
||||
must land as a ms-epoch int; 云客 returns seconds, the scraper ×1000).
|
||||
JSON list columns (``praise_json`` / ``comment_json`` / ``image_urls``)
|
||||
are stored verbatim as JSON arrays — the generic list->";".join coercion
|
||||
in :func:`_build_column_values` only applies to scalar columns. Returns
|
||||
the number of records processed (input length); the DB collapses
|
||||
duplicate ``id`` values to one row.
|
||||
"""
|
||||
for rec in recs:
|
||||
await _upsert_one(session, MomentsPost, rec)
|
||||
return len(recs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VoiceClip (长语音) upsert
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user