feat(scripts): 全渠道全量重同步脚本(页面抓取+API全覆盖)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-15 14:38:01 +08:00
parent 2d1943a74e
commit 7beae8cf55
2 changed files with 115 additions and 0 deletions
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Full re-sync via page scraper (zero 风控)
+cd "$(dirname "$0")/.."
+export DB_URL="sqlite+aiosqlite:///./smoke.db"
+PY=".venv/Scripts/python.exe"
+
+echo "$(date) === 1/3: Refreshing ALL wechat customers ==="
+$PY scripts/scrape_via_page.py --entity friends --max-employees 100
+echo "$(date) === Friends done ==="
+
+echo "$(date) === 2/3: Refreshing ALL wechat groups ==="
+$PY scripts/scrape_via_page.py --entity groups --max-employees 100
+echo "$(date) === Groups done ==="
+
+echo "$(date) === 3/3: Scraping messages (1000 conversations) ==="
+$PY scripts/scrape_via_page.py --entity messages --max-conversations 1000
+echo "$(date) === Messages done ==="
+
+echo "$(date) === FULL RE-SYNC COMPLETE ==="
@@ -0,0 +1,96 @@
+#!/bin/bash
+# FULL re-sync: ALL channels, ALL entities (page scraper + API)
+# Runs after the page scraper (full_resync.sh) completes.
+cd "$(dirname "$0")/.."
+export DB_URL="sqlite+aiosqlite:///./smoke.db"
+PY=".venv/Scripts/python.exe"
+
+echo "=============================================="
+echo "$(date) === COMPREHENSIVE FULL RE-SYNC ==="
+echo "=============================================="
+
+# ---- Phase 1: Page scraper (zero 风控) ----
+echo "$(date) --- 1. Page scraper: WeChat customers ---"
+$PY scripts/scrape_via_page.py --entity friends --max-employees 100
+echo "$(date) friends done"
+
+echo "$(date) --- 2. Page scraper: WeChat groups ---"
+$PY scripts/scrape_via_page.py --entity groups --max-employees 100
+echo "$(date) groups done"
+
+echo "$(date) --- 3. Page scraper: WeChat messages (1000 convs) ---"
+$PY scripts/scrape_via_page.py --entity messages --max-conversations 1000
+echo "$(date) wechat messages done"
+
+# ---- Phase 2: API sync for remaining entities ----
+echo "$(date) --- 4. API: departments + accounts ---"
+$PY -m app.sync.cli sync --entity departments
+$PY -m app.sync.cli sync --entity accounts
+echo "$(date) accounts done"
+
+echo "$(date) --- 5. API: WeChat voice calls ---"
+$PY -m app.sync.cli sync --entity voice_calls
+echo "$(date) voice_calls done"
+
+echo "$(date) --- 6. API: WA accounts ---"
+$PY -m app.sync.cli sync --entity wa_accounts
+echo "$(date) wa_accounts done"
+
+echo "$(date) --- 7. API: WA friends ---"
+$PY -m app.sync.cli sync --entity wa_friends
+echo "$(date) wa_friends done"
+
+echo "$(date) --- 8. API: WA voice calls ---"
+$PY -m app.sync.cli sync --entity wa_voice_calls
+echo "$(date) wa_voice_calls done"
+
+echo "$(date) --- 9. API: Voice clips ---"
+$PY -m app.sync.cli sync --entity voice_clips
+echo "$(date) voice_clips done"
+
+echo "$(date) --- 10. API: Voice clip audio download ---"
+$PY -m app.sync.cli sync --entity voice_clip_audio
+echo "$(date) voice_clip_audio done"
+
+echo "$(date) --- 11. API: Call recordings download ---"
+$PY -m app.sync.cli sync --entity call_recordings
+echo "$(date) call_recordings done"
+
+echo "$(date) --- 12. API: Message images download ---"
+$PY -m app.sync.cli sync --entity message_images
+echo "$(date) message_images done"
+
+echo "$(date) --- 13. API: WA messages (300 req cap) ---"
+$PY -m app.sync.cli sync --entity wa_messages
+echo "$(date) wa_messages done"
+
+echo "$(date) --- 14. API: WeChat messages (additional, 300 req cap) ---"
+$PY -m app.sync.cli sync --entity messages
+echo "$(date) messages done"
+
+echo "=============================================="
+echo "$(date) === FULL RE-SYNC COMPLETE ==="
+echo "=============================================="
+
+# Show final data summary
+$PY -c "
+import sqlite3
+con = sqlite3.connect('smoke.db')
+c = con.cursor()
+print()
+print('=== FINAL DATA SUMMARY ===')
+for label, sql in [
+    ('wechat accounts', \"SELECT COUNT(*) FROM wechat_account\"),
+    ('whatsapp accounts', \"SELECT COUNT(*) FROM whatsapp_account\"),
+    ('wechat customers', \"SELECT COUNT(*) FROM customer WHERE channel='wechat'\"),
+    ('whatsapp customers', \"SELECT COUNT(*) FROM customer WHERE channel='whatsapp'\"),
+    ('wechat groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='wechat'\"),
+    ('whatsapp groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='whatsapp'\"),
+    ('wechat messages', \"SELECT COUNT(*) FROM message WHERE channel='wechat'\"),
+    ('whatsapp messages', \"SELECT COUNT(*) FROM message WHERE channel='whatsapp'\"),
+    ('voice calls', \"SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')\"),
+    ('voice clips', 'SELECT COUNT(*) FROM voice_clip'),
+    ('media files', 'SELECT COUNT(*) FROM media'),
+]:
+    print(f'  {label:22s} {c.execute(sql).fetchone()[0]:>6}')
+"