fix(repos): int 强转跳过 String 声明列(列类型优先于 _INT_FIELDS 按名集合)
Task 1 评审遗留:_INT_FIELDS 含 type_raw(为 Message int code 加),但
MomentsPost.type_raw 是 String(16)——按名强转会把非数字 code(如
'text')置 None,违背列声明。_build_column_values 现在检查目标列
声明类型:isinstance(col.type, sa.String)(含 Text)则跳过强转,
Message.type_raw(Integer) 照旧。另修正 upsert_chat_group 文档串里
过时的 'PK = room_id'(复合键自 8a99940)。
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -23,9 +23,11 @@ dict (see :func:`_coerce_bool`, :func:`_coerce_int`):
|
|||||||
``is_upload``): ``1`` / ``"1"`` / ``True`` -> ``True``; ``0`` / ``"0"`` /
|
``is_upload``): ``1`` / ``"1"`` / ``True`` -> ``True``; ``0`` / ``"0"`` /
|
||||||
``False`` -> ``False``; ``None`` preserved.
|
``False`` -> ``False``; ``None`` preserved.
|
||||||
* ``gender``: ``"0"`` / ``"1"`` / ``"2"`` (str) -> ``int`` when not ``None``.
|
* ``gender``: ``"0"`` / ``"1"`` / ``"2"`` (str) -> ``int`` when not ``None``.
|
||||||
* ``type_raw`` (Message): the raw upstream type code, coerced to
|
* ``type_raw``: the raw upstream type code. Coerced to ``int | None`` **only
|
||||||
``int | None``. ``Message.type`` is a normalized label string and is
|
when the target column is an Integer/BigInteger column** (``Message.type_raw``)
|
||||||
stored as-is (no int coercion).
|
— a same-named column declared as String (``MomentsPost.type_raw``) keeps the
|
||||||
|
value verbatim (column declaration wins over the global by-name set).
|
||||||
|
``Message.type`` / ``MomentsPost`` label strings are stored as-is.
|
||||||
* PhoneCall int fields (``call_duration`` / ``start_call_time`` /
|
* PhoneCall int fields (``call_duration`` / ``start_call_time`` /
|
||||||
``call_status`` / ``incoming_call`` / ``qc_status``): coerced to
|
``call_status`` / ``incoming_call`` / ``qc_status``): coerced to
|
||||||
``int | None``. The voice-call scraper already parses these to ints
|
``int | None``. The voice-call scraper already parses these to ints
|
||||||
@@ -240,7 +242,12 @@ def _build_column_values(
|
|||||||
v = rec[col_name]
|
v = rec[col_name]
|
||||||
if col_name in _BOOL_FIELDS:
|
if col_name in _BOOL_FIELDS:
|
||||||
v = _coerce_bool(v)
|
v = _coerce_bool(v)
|
||||||
elif col_name in _INT_FIELDS:
|
elif col_name in _INT_FIELDS and not isinstance(col.type, sa.String):
|
||||||
|
# 列声明类型优先于全局按名集合(2026-08-25 评审遗留修复):
|
||||||
|
# 同名列在不同表可声明不同类型 —— Message.type_raw 是
|
||||||
|
# Integer(照旧强转),而 MomentsPost.type_raw 是 String(16),
|
||||||
|
# 强转会把非数字 code(如 "text")置 None,违背列声明。
|
||||||
|
# sa.Text 继承 sa.String,同样跳过。
|
||||||
v = _coerce_int(v)
|
v = _coerce_int(v)
|
||||||
# Some upstream fields are lists (e.g. Customer.deptRoute is a
|
# Some upstream fields are lists (e.g. Customer.deptRoute is a
|
||||||
# list of dept ids) but map to scalar String columns — join so the
|
# list of dept ids) but map to scalar String columns — join so the
|
||||||
@@ -512,7 +519,8 @@ async def upsert_whatsapp_account(session: AsyncSession, rec: dict[str, Any]) ->
|
|||||||
async def upsert_chat_group(session: AsyncSession, rec: dict[str, Any]) -> None:
|
async def upsert_chat_group(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||||
"""Idempotently upsert one :class:`ChatGroup` AND its :class:`GroupMember` rows.
|
"""Idempotently upsert one :class:`ChatGroup` AND its :class:`GroupMember` rows.
|
||||||
|
|
||||||
PK = ``room_id``. The group row is upserted first; then, for every id in
|
PK = composite ``(room_id, channel, owner_account_id)`` (同一群在每个
|
||||||
|
员工名下各一行). The group row is upserted first; then, for every id in
|
||||||
the record's ``member_ids_list`` (the scraper-parsed, empties-dropped
|
the record's ``member_ids_list`` (the scraper-parsed, empties-dropped
|
||||||
member wxid list), a :class:`GroupMember` row is upserted against the
|
member wxid list), a :class:`GroupMember` row is upserted against the
|
||||||
composite PK ``(channel, room_id, member_id)``. Re-scraping a group
|
composite PK ``(channel, room_id, member_id)``. Re-scraping a group
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from app.db.models import (
|
|||||||
WhatsappAccount,
|
WhatsappAccount,
|
||||||
)
|
)
|
||||||
from app.storage.repos import (
|
from app.storage.repos import (
|
||||||
|
_build_column_values,
|
||||||
_coerce_bool,
|
_coerce_bool,
|
||||||
_coerce_int,
|
_coerce_int,
|
||||||
upsert_chat_group,
|
upsert_chat_group,
|
||||||
@@ -1459,3 +1460,63 @@ async def test_upsert_moments_posts_idempotent_keeps_json_lists_and_coerces_crea
|
|||||||
assert rows[0].type_raw == "1" # MomentsPost type_raw is a string code
|
assert rows[0].type_raw == "1" # MomentsPost type_raw is a string code
|
||||||
finally:
|
finally:
|
||||||
await engine.dispose()
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# int 强转 vs String 列声明(2026-08-25 评审遗留修复)
|
||||||
|
# _INT_FIELDS 按列名全局命中,但列类型优先:目标列声明为 String(Text 继承)
|
||||||
|
# 则跳过强转 —— MomentsPost.type_raw 是 String(16),非数字 code(如 "text")
|
||||||
|
# 不能被 _coerce_int 置 None;Message.type_raw(Integer)照旧强转。
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def test_int_coercion_skipped_for_string_columns():
|
||||||
|
"""String 声明的 type_raw 原样保留;Integer 声明的照旧强转。"""
|
||||||
|
# MomentsPost.type_raw = String(16) -> 不强转
|
||||||
|
vals = _build_column_values(
|
||||||
|
MomentsPost, {"id": "x", "wechat_id": "w", "type_raw": "1"}
|
||||||
|
)
|
||||||
|
assert vals["type_raw"] == "1"
|
||||||
|
vals = _build_column_values(
|
||||||
|
MomentsPost, {"id": "x", "wechat_id": "w", "type_raw": "text"}
|
||||||
|
)
|
||||||
|
assert vals["type_raw"] == "text"
|
||||||
|
|
||||||
|
# Message.type_raw = Integer -> "1" -> 1,非数字 -> None
|
||||||
|
vals = _build_column_values(
|
||||||
|
Message, {"msg_id": "m", "channel": "wechat", "type_raw": "1"}
|
||||||
|
)
|
||||||
|
assert vals["type_raw"] == 1
|
||||||
|
vals = _build_column_values(
|
||||||
|
Message, {"msg_id": "m", "channel": "wechat", "type_raw": "text"}
|
||||||
|
)
|
||||||
|
assert vals["type_raw"] is None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_moments_post_non_numeric_type_raw_round_trip():
|
||||||
|
"""非数字 type_raw code 入库读回仍是原字符串(不被 _coerce_int 置 None)。"""
|
||||||
|
engine = await _make_engine()
|
||||||
|
try:
|
||||||
|
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
await upsert_moments_posts(
|
||||||
|
session,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "p1",
|
||||||
|
"wechat_id": "wxid_x",
|
||||||
|
"type_raw": "video", # 假设未来出现的非数字 code
|
||||||
|
"create_time": 1776302051000,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
sa.select(MomentsPost).execution_options(populate_existing=True)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0].type_raw == "video"
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|||||||
Reference in New Issue
Block a user