fix: split message type into normalized label + raw code

The scraper was writing a normalized type label ("text"/"unknown"/...) into
Message.type, but the column was Integer, so the storage layer coerced it to
NULL and the raw code was only recoverable from raw_json. Align with design
§4.3, which specifies two fields: `type` (normalized enum) and `type_raw`
(original code).

- Model (Message): type -> String(32) normalized label; add type_raw Integer
  for the raw upstream type code.
- Scraper: _normalize_message sets both `type` (label) and `type_raw`
  (coerced int from messageType/type); _is_divider now checks raw -2.
- Repos: type_raw joins _INT_FIELDS; Message.type stored as-is (string).
- Migration 96975050fa79: add type_raw + alter type Integer->String(32),
  wrapped in batch_alter_table so it runs on SQLite and MySQL.
- env.py: enable render_as_batch on SQLite (dev/test) for portable batch
  migrations; MySQL still uses direct ALTER.
- Tests updated end-to-end (scraper, repos, models).

Co-Authored-By: Claude <noreply@anthropic.com>

yunke-server/tests/test_models.py

@@ -179,7 +179,8 @@ async def test_message_natural_pk():
                 owner_account_id="wxid_staff",
                 talker_id="wxid_abc",
                 is_send=False,
-                type=1,
+                type="text",
+                type_raw=1,
                 text="hello",
                 timestamp=1700000000000,
             )
@@ -195,6 +196,8 @@ async def test_message_natural_pk():
             assert fetched.text == "hello"
             assert fetched.timestamp == 1700000000000
             assert fetched.is_send is False
+            assert fetched.type == "text"
+            assert fetched.type_raw == 1
     finally:
         await engine.dispose()
 

yunke-server/tests/test_repos.py

@@ -72,10 +72,10 @@ def test_coerce_int_handles_documented_cases():
 
 
 def test_coerce_int_returns_none_for_non_numeric_string():
-    """A non-numeric string (e.g. normalized 'type' label) -> None.
+    """A non-numeric string (e.g. a label) -> None.
 
-    The column is INTEGER; a non-numeric value cannot be stored. The raw value
-    remains recoverable from ``raw_json``.
+    Integer columns (``type_raw``, ``gender``, ...) cannot store a
+    non-numeric value; the coercion collapses it to ``None``.
     """
     assert _coerce_int("text") is None
     assert _coerce_int("unknown") is None
@@ -229,7 +229,8 @@ async def test_upsert_customers_batch_returns_count():
 
 
 async def test_upsert_messages_coerces_types():
-    """is_send '1' -> True; is_del 0 -> False; same msg_id twice -> 1 row."""
+    """is_send '1' -> True; is_del 0 -> False; type label stored as string;
+    type_raw coerced to int; same msg_id twice -> 1 row."""
     engine = await _make_engine()
     try:
         SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
@@ -239,7 +240,8 @@ async def test_upsert_messages_coerces_types():
                 "channel": "wechat",
                 "is_send": "1",
                 "is_del": 0,
-                "type": "1",
+                "type": "text",  # normalized label — stored as-is (no coercion)
+                "type_raw": "1",  # raw code arrives as str -> coerced to int
                 "timestamp": 1_700_000_000_000,
             }
             await upsert_messages(session, [rec])
@@ -252,17 +254,18 @@ async def test_upsert_messages_coerces_types():
             ).scalar_one()
             assert fetched.is_send is True
             assert fetched.is_del is False
-            assert fetched.type == 1
+            assert fetched.type == "text"  # string label preserved
+            assert fetched.type_raw == 1  # raw code coerced to int
             assert fetched.timestamp == 1_700_000_000_000
 
-            # Upsert same msg_id again with different text -> row updated, not
-            # duplicated.
-            rec2 = dict(rec, text="updated")
+            # Upsert same msg_id again with different text + new raw code ->
+            # row updated, not duplicated.
+            rec2 = dict(rec, text="updated", type="unknown", type_raw=3)
             await upsert_messages(session, [rec2])
             await session.commit()
 
             # ``populate_existing=True`` bypasses the identity-map cache so we
-            # observe the second upsert's text value.
+            # observe the second upsert's values.
             rows = (
                 await session.execute(
                     sa.select(Message)
@@ -272,6 +275,34 @@ async def test_upsert_messages_coerces_types():
             ).scalars().all()
             assert len(rows) == 1
             assert rows[0].text == "updated"
+            assert rows[0].type == "unknown"
+            assert rows[0].type_raw == 3
+    finally:
+        await engine.dispose()
+
+
+async def test_upsert_messages_preserves_none_type_and_type_raw():
+    """Missing / None type & type_raw stay None (no spurious coercion)."""
+    engine = await _make_engine()
+    try:
+        SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+        async with SessionLocal() as session:
+            rec = {
+                "msg_id": "m_none",
+                "channel": "wechat",
+                "type": None,
+                "type_raw": None,
+            }
+            await upsert_messages(session, [rec])
+            await session.commit()
+
+            fetched = (
+                await session.execute(
+                    sa.select(Message).where(Message.msg_id == "m_none")
+                )
+            ).scalar_one()
+            assert fetched.type is None
+            assert fetched.type_raw is None
     finally:
         await engine.dispose()
 

yunke-server/tests/test_scrape_wechat_messages.py

@@ -106,7 +106,8 @@ async def test_scrape_wechat_messages_drops_divider_and_paginates_backward():
     assert r1["ori_talker"] == "wxid_friend_0001"
     assert r1["room_id"] == ""              # raw value carried through
     assert r1["is_send"] == 0               # pulled from `mine`
-    assert r1["type"] == "text"
+    assert r1["type"] == "text"             # normalized label (design §4.3)
+    assert r1["type_raw"] == 1              # original raw code preserved
     assert r1["text"] == "hi alice"
     assert r1["timestamp"] == E1 + 600000
     assert r1["sender_name"] == "Alice Wang"
@@ -119,6 +120,7 @@ async def test_scrape_wechat_messages_drops_divider_and_paginates_backward():
     assert r2["ori_talker"] == "room12345@chatroom"
     assert r2["is_send"] == 1
     assert r2["type"] == "unknown"          # unmapped code passes through
+    assert r2["type_raw"] == 3              # raw image-type code preserved
     assert r2["file_url"] == "https://cdn.example.com/img2.jpg"
     assert r2["file_th_url"] == "https://cdn.example.com/img2_th.jpg"
     assert r2["file_hd_url"] == "https://cdn.example.com/img2_hd.jpg"
@@ -133,6 +135,7 @@ async def test_scrape_wechat_messages_drops_divider_and_paginates_backward():
     # m3 — voice / transcription.
     assert r3["msg_id"] == "999999"
     assert r3["type"] == "unknown"
+    assert r3["type_raw"] == 34             # raw voice-type code preserved
     assert r3["duration"] == 5
     assert r3["transcription_text"] == "hello from yesterday"
     assert r3["pay_msg_id"] == "pay-abc-123"