feat(db): moments_post 表+upsert(朋友圈实体,PK=云客data.id,create_time统一ms)
- MomentsPost 模型:moments_post 表,PK=id(云客 getFriendsterInfo data.id), wechat_id/create_time 建索引,praise/comment/image 存 JSON 列 - 迁移 524da8d2d597(手写 create_table,已应用到 smoke.db 并在临时 SQLite 全链路 upgrade/downgrade 验证) - repos.upsert_moments_posts:批量幂等 upsert;附带修 _build_column_values 对 JSON 列表的误 join(标量列 join 保持不变,dept_route 旧测试仍过) - 修复 3 个 chat_group 旧测试缺 owner_account_id(2026-08-24 复合主键遗留) - 全套 430 测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -36,6 +36,10 @@ dict (see :func:`_coerce_bool`, :func:`_coerce_int`):
|
||||
* ``raw_json``: the original record dict, preserved verbatim. When the input
|
||||
already carries a ``raw_json`` key (scraper output does), that value wins;
|
||||
otherwise the input dict itself is stored.
|
||||
* JSON-typed list columns (``MomentsPost.praise_json`` / ``comment_json`` /
|
||||
``image_urls``): list values are kept verbatim (stored as JSON arrays). The
|
||||
list -> ``";"``-join coercion only applies to lists headed for SCALAR
|
||||
String columns (e.g. ``Customer.dept_route``).
|
||||
|
||||
Sharding note
|
||||
-------------
|
||||
@@ -60,6 +64,7 @@ from app.db.models import (
|
||||
GroupMember,
|
||||
Media,
|
||||
Message,
|
||||
MomentsPost,
|
||||
PhoneCall,
|
||||
Sms,
|
||||
Staff,
|
||||
@@ -75,6 +80,7 @@ __all__ = [
|
||||
"upsert_department",
|
||||
"upsert_media",
|
||||
"upsert_messages",
|
||||
"upsert_moments_posts",
|
||||
"upsert_phone_call",
|
||||
"upsert_sms",
|
||||
"upsert_staff",
|
||||
@@ -220,12 +226,16 @@ def _build_column_values(
|
||||
on unknown columns).
|
||||
* Boolean / int-as-str fields are coerced via :func:`_coerce_bool` /
|
||||
:func:`_coerce_int`.
|
||||
* List values headed for SCALAR columns (e.g. ``Customer.dept_route``) are
|
||||
joined into a ``";"``-separated string so the bind succeeds; list values
|
||||
headed for JSON columns (e.g. ``MomentsPost.praise_json`` /
|
||||
``image_urls``) are kept verbatim as JSON arrays.
|
||||
* ``raw_json`` defaults to the entire input dict when not explicitly
|
||||
provided, so the verbatim upstream record is always recoverable.
|
||||
"""
|
||||
col_names = {c.name for c in model.__table__.columns}
|
||||
cols = {c.name: c for c in model.__table__.columns}
|
||||
values: dict[str, Any] = {}
|
||||
for col_name in col_names:
|
||||
for col_name, col in cols.items():
|
||||
if col_name in rec:
|
||||
v = rec[col_name]
|
||||
if col_name in _BOOL_FIELDS:
|
||||
@@ -235,14 +245,15 @@ def _build_column_values(
|
||||
# Some upstream fields are lists (e.g. Customer.deptRoute is a
|
||||
# list of dept ids) but map to scalar String columns — join so the
|
||||
# bind succeeds and the data is still recoverable (raw_json keeps
|
||||
# the original list).
|
||||
if isinstance(v, list):
|
||||
# the original list). JSON-typed columns keep the list verbatim
|
||||
# (MomentsPost.praise_json / comment_json / image_urls).
|
||||
if isinstance(v, list) and not isinstance(col.type, sa.JSON):
|
||||
v = ";".join(str(x) for x in v)
|
||||
values[col_name] = v
|
||||
# Preserve the verbatim source record. If the scraper already set raw_json
|
||||
# (it does), that wins; otherwise the input dict itself is the source of
|
||||
# truth.
|
||||
if "raw_json" in col_names and "raw_json" not in values:
|
||||
if "raw_json" in cols and "raw_json" not in values:
|
||||
values["raw_json"] = rec
|
||||
# Sanitize lone surrogate chars (e.g. \udf1e from emoji fragments in the
|
||||
# upstream API) that break SQLite/MySQL UTF-8 encoding. Replaces them with
|
||||
@@ -602,6 +613,28 @@ async def upsert_sms(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
await _upsert_one(session, Sms, rec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MomentsPost (员工朋友圈) upsert — 员工聊天窗口 Phase 1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def upsert_moments_posts(session: AsyncSession, recs: list[dict[str, Any]]) -> int:
|
||||
"""Batch idempotent upsert for :class:`MomentsPost` (PK = ``id``).
|
||||
|
||||
Same ``id`` twice -> one row, with non-PK fields refreshed from the later
|
||||
record. ``create_time`` goes through the ``_INT_FIELDS`` int coercion (it
|
||||
must land as a ms-epoch int; 云客 returns seconds, the scraper ×1000).
|
||||
JSON list columns (``praise_json`` / ``comment_json`` / ``image_urls``)
|
||||
are stored verbatim as JSON arrays — the generic list->";".join coercion
|
||||
in :func:`_build_column_values` only applies to scalar columns. Returns
|
||||
the number of records processed (input length); the DB collapses
|
||||
duplicate ``id`` values to one row.
|
||||
"""
|
||||
for rec in recs:
|
||||
await _upsert_one(session, MomentsPost, rec)
|
||||
return len(recs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VoiceClip (长语音) upsert
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user