fix: sanitize lone surrogate chars in stored data + backfill script

Upstream API records may contain lone surrogate chars (emoji fragments)
that break SQLite/MySQL utf-8 encoding. Now sanitized to U+FFFD before
storage. Includes backfill_customers.py for full customer backfill.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-05 11:57:12 +08:00
parent a92fa2927c
commit c8311bb496
2 changed files with 105 additions and 0 deletions
+20
View File
@@ -211,9 +211,29 @@ def _build_column_values(
# truth.
if "raw_json" in col_names and "raw_json" not in values:
values["raw_json"] = rec
# Sanitize lone surrogate chars (e.g. \udf1e from emoji fragments in the
# upstream API) that break SQLite/MySQL UTF-8 encoding. Replaces them with
# the Unicode replacement character (U+FFFD).
for k, v in values.items():
values[k] = _sanitize_surrogates(v)
return values
def _sanitize_surrogates(obj: Any) -> Any:
"""Recursively remove lone surrogate characters from strings/dicts/lists."""
if isinstance(obj, str):
try:
obj.encode("utf-8") # fast-path: clean string, no copy
return obj
except UnicodeEncodeError:
return obj.encode("utf-8", "replace").decode("utf-8")
if isinstance(obj, dict):
return {k: _sanitize_surrogates(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_sanitize_surrogates(v) for v in obj]
return obj
def _build_upsert_stmt(
model: type[DeclarativeBase], values: dict[str, Any], dialect_name: str
) -> sa.Insert: