fix: sanitize lone surrogate chars in stored data + backfill script
Upstream API records may contain lone surrogate chars (emoji fragments) that break SQLite/MySQL utf-8 encoding. Now sanitized to U+FFFD before storage. Includes backfill_customers.py for full customer backfill. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -211,9 +211,29 @@ def _build_column_values(
|
||||
# truth.
|
||||
if "raw_json" in col_names and "raw_json" not in values:
|
||||
values["raw_json"] = rec
|
||||
# Sanitize lone surrogate chars (e.g. \udf1e from emoji fragments in the
|
||||
# upstream API) that break SQLite/MySQL UTF-8 encoding. Replaces them with
|
||||
# the Unicode replacement character (U+FFFD).
|
||||
for k, v in values.items():
|
||||
values[k] = _sanitize_surrogates(v)
|
||||
return values
|
||||
|
||||
|
||||
def _sanitize_surrogates(obj: Any) -> Any:
|
||||
"""Recursively remove lone surrogate characters from strings/dicts/lists."""
|
||||
if isinstance(obj, str):
|
||||
try:
|
||||
obj.encode("utf-8") # fast-path: clean string, no copy
|
||||
return obj
|
||||
except UnicodeEncodeError:
|
||||
return obj.encode("utf-8", "replace").decode("utf-8")
|
||||
if isinstance(obj, dict):
|
||||
return {k: _sanitize_surrogates(v) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_sanitize_surrogates(v) for v in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _build_upsert_stmt(
|
||||
model: type[DeclarativeBase], values: dict[str, Any], dialect_name: str
|
||||
) -> sa.Insert:
|
||||
|
||||
Reference in New Issue
Block a user