fix: sanitize lone surrogate chars in stored data + backfill script

Upstream API records may contain lone surrogate chars (emoji fragments) that break SQLite/MySQL utf-8 encoding. Now sanitized to U+FFFD before storage. Includes backfill_customers.py for full customer backfill. Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-05 11:57:12 +08:00
parent a92fa2927c
commit c8311bb496
2 changed files with 105 additions and 0 deletions
@@ -0,0 +1,85 @@
+"""Full backfill of customers (WeChat + WhatsApp) into the DB.
+
+Unlike the seed scripts (tiny demo slices), this paginates the ENTIRE friend
+list for both channels and upserts everything. Slow (anti-风控 paced), commits
+per batch so it's resumable. Run in background; logs progress to stdout.
+"""
+import asyncio
+import sys
+import time
+
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.db import models  # noqa: F401
+from app.scraper.wechat import scrape_departments, scrape_wechat_friends
+from app.scraper.whatsapp import scrape_whatsapp_friends
+from app.storage.repos import upsert_customers
+from scripts.smoke_wechat import _push_settings_into_environ, build_client_from_env
+
+BATCH = 100  # upsert + commit every N records
+PAGE_SIZE = 50
+
+
+def _log(msg: str) -> None:
+    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
+
+
+async def _root_dept(client) -> str | None:
+    async for d in scrape_departments(client):
+        if not d.get("parent_id"):
+            return d.get("dept_id")
+    return None
+
+
+async def _backfill_channel(label: str, agen, session_maker) -> int:
+    total = 0
+    batch: list[dict] = []
+    async for rec in agen:
+        batch.append(rec)
+        if len(batch) >= BATCH:
+            async with session_maker() as s:
+                await upsert_customers(s, batch)
+                await s.commit()
+            total += len(batch)
+            _log(f"{label}: {total} upserted...")
+            batch = []
+    if batch:
+        async with session_maker() as s:
+            await upsert_customers(s, batch)
+            await s.commit()
+        total += len(batch)
+    _log(f"{label}: DONE — {total} customers")
+    return total
+
+
+async def main() -> None:
+    _push_settings_into_environ()
+    client = build_client_from_env()
+    if client is None:
+        _log("no auth env"); sys.exit(2)
+    engine = create_async_engine("sqlite+aiosqlite:///./smoke.db")
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+
+    _log("resolving root department...")
+    root = await _root_dept(client)
+    _log(f"root dept: {root}")
+
+    _log("=== WeChat customers (full) ===")
+    wx = await _backfill_channel(
+        "wechat", scrape_wechat_friends(client, page_size=PAGE_SIZE), sm
+    )
+
+    _log("=== WhatsApp customers (full) ===")
+    wa = await _backfill_channel(
+        "whatsapp",
+        scrape_whatsapp_friends(client, department_id=root or "", page_size=PAGE_SIZE),
+        sm,
+    )
+
+    _log(f"\nTOTAL: {wx} wechat + {wa} whatsapp = {wx + wa} customers")
+    await client.aclose()
+    await engine.dispose()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())