fix: split message type into normalized label + raw code
The scraper was writing a normalized type label ("text"/"unknown"/...) into
Message.type, but the column was Integer, so the storage layer coerced it to
NULL and the raw code was only recoverable from raw_json. Align with design
§4.3, which specifies two fields: `type` (normalized enum) and `type_raw`
(original code).
- Model (Message): type -> String(32) normalized label; add type_raw Integer
for the raw upstream type code.
- Scraper: _normalize_message sets both `type` (label) and `type_raw`
(coerced int from messageType/type); _is_divider now checks raw -2.
- Repos: type_raw joins _INT_FIELDS; Message.type stored as-is (string).
- Migration 96975050fa79: add type_raw + alter type Integer->String(32),
wrapped in batch_alter_table so it runs on SQLite and MySQL.
- env.py: enable render_as_batch on SQLite (dev/test) for portable batch
migrations; MySQL still uses direct ALTER.
- Tests updated end-to-end (scraper, repos, models).
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -72,10 +72,10 @@ def test_coerce_int_handles_documented_cases():
|
||||
|
||||
|
||||
def test_coerce_int_returns_none_for_non_numeric_string():
|
||||
"""A non-numeric string (e.g. normalized 'type' label) -> None.
|
||||
"""A non-numeric string (e.g. a label) -> None.
|
||||
|
||||
The column is INTEGER; a non-numeric value cannot be stored. The raw value
|
||||
remains recoverable from ``raw_json``.
|
||||
Integer columns (``type_raw``, ``gender``, ...) cannot store a
|
||||
non-numeric value; the coercion collapses it to ``None``.
|
||||
"""
|
||||
assert _coerce_int("text") is None
|
||||
assert _coerce_int("unknown") is None
|
||||
@@ -229,7 +229,8 @@ async def test_upsert_customers_batch_returns_count():
|
||||
|
||||
|
||||
async def test_upsert_messages_coerces_types():
|
||||
"""is_send '1' -> True; is_del 0 -> False; same msg_id twice -> 1 row."""
|
||||
"""is_send '1' -> True; is_del 0 -> False; type label stored as string;
|
||||
type_raw coerced to int; same msg_id twice -> 1 row."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
@@ -239,7 +240,8 @@ async def test_upsert_messages_coerces_types():
|
||||
"channel": "wechat",
|
||||
"is_send": "1",
|
||||
"is_del": 0,
|
||||
"type": "1",
|
||||
"type": "text", # normalized label — stored as-is (no coercion)
|
||||
"type_raw": "1", # raw code arrives as str -> coerced to int
|
||||
"timestamp": 1_700_000_000_000,
|
||||
}
|
||||
await upsert_messages(session, [rec])
|
||||
@@ -252,17 +254,18 @@ async def test_upsert_messages_coerces_types():
|
||||
).scalar_one()
|
||||
assert fetched.is_send is True
|
||||
assert fetched.is_del is False
|
||||
assert fetched.type == 1
|
||||
assert fetched.type == "text" # string label preserved
|
||||
assert fetched.type_raw == 1 # raw code coerced to int
|
||||
assert fetched.timestamp == 1_700_000_000_000
|
||||
|
||||
# Upsert same msg_id again with different text -> row updated, not
|
||||
# duplicated.
|
||||
rec2 = dict(rec, text="updated")
|
||||
# Upsert same msg_id again with different text + new raw code ->
|
||||
# row updated, not duplicated.
|
||||
rec2 = dict(rec, text="updated", type="unknown", type_raw=3)
|
||||
await upsert_messages(session, [rec2])
|
||||
await session.commit()
|
||||
|
||||
# ``populate_existing=True`` bypasses the identity-map cache so we
|
||||
# observe the second upsert's text value.
|
||||
# observe the second upsert's values.
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(Message)
|
||||
@@ -272,6 +275,34 @@ async def test_upsert_messages_coerces_types():
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].text == "updated"
|
||||
assert rows[0].type == "unknown"
|
||||
assert rows[0].type_raw == 3
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_upsert_messages_preserves_none_type_and_type_raw():
|
||||
"""Missing / None type & type_raw stay None (no spurious coercion)."""
|
||||
engine = await _make_engine()
|
||||
try:
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with SessionLocal() as session:
|
||||
rec = {
|
||||
"msg_id": "m_none",
|
||||
"channel": "wechat",
|
||||
"type": None,
|
||||
"type_raw": None,
|
||||
}
|
||||
await upsert_messages(session, [rec])
|
||||
await session.commit()
|
||||
|
||||
fetched = (
|
||||
await session.execute(
|
||||
sa.select(Message).where(Message.msg_id == "m_none")
|
||||
)
|
||||
).scalar_one()
|
||||
assert fetched.type is None
|
||||
assert fetched.type_raw is None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user