From 7beae8cf5550830a4e942fa64f8f825d43cb0ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E5=BF=B5?= Date: Sat, 15 Aug 2026 14:38:01 +0800 Subject: [PATCH] =?UTF-8?q?feat(scripts):=20=E5=85=A8=E6=B8=A0=E9=81=93?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E9=87=8D=E5=90=8C=E6=AD=A5=E8=84=9A=E6=9C=AC?= =?UTF-8?q?(=E9=A1=B5=E9=9D=A2=E6=8A=93=E5=8F=96+API=E5=85=A8=E8=A6=86?= =?UTF-8?q?=E7=9B=96)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- yunke-server/scripts/full_resync.sh | 19 ++++++ yunke-server/scripts/full_sync_all.sh | 96 +++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 yunke-server/scripts/full_resync.sh create mode 100644 yunke-server/scripts/full_sync_all.sh diff --git a/yunke-server/scripts/full_resync.sh b/yunke-server/scripts/full_resync.sh new file mode 100644 index 0000000..6f6000f --- /dev/null +++ b/yunke-server/scripts/full_resync.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Full re-sync via page scraper (zero 风控) +cd "$(dirname "$0")/.." +export DB_URL="sqlite+aiosqlite:///./smoke.db" +PY=".venv/Scripts/python.exe" + +echo "$(date) === 1/3: Refreshing ALL wechat customers ===" +$PY scripts/scrape_via_page.py --entity friends --max-employees 100 +echo "$(date) === Friends done ===" + +echo "$(date) === 2/3: Refreshing ALL wechat groups ===" +$PY scripts/scrape_via_page.py --entity groups --max-employees 100 +echo "$(date) === Groups done ===" + +echo "$(date) === 3/3: Scraping messages (1000 conversations) ===" +$PY scripts/scrape_via_page.py --entity messages --max-conversations 1000 +echo "$(date) === Messages done ===" + +echo "$(date) === FULL RE-SYNC COMPLETE ===" diff --git a/yunke-server/scripts/full_sync_all.sh b/yunke-server/scripts/full_sync_all.sh new file mode 100644 index 0000000..59b58e2 --- /dev/null +++ b/yunke-server/scripts/full_sync_all.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# FULL re-sync: ALL channels, ALL entities (page scraper + API) +# Runs after the page scraper (full_resync.sh) completes. +cd "$(dirname "$0")/.." +export DB_URL="sqlite+aiosqlite:///./smoke.db" +PY=".venv/Scripts/python.exe" + +echo "==============================================" +echo "$(date) === COMPREHENSIVE FULL RE-SYNC ===" +echo "==============================================" + +# ---- Phase 1: Page scraper (zero 风控) ---- +echo "$(date) --- 1. Page scraper: WeChat customers ---" +$PY scripts/scrape_via_page.py --entity friends --max-employees 100 +echo "$(date) friends done" + +echo "$(date) --- 2. Page scraper: WeChat groups ---" +$PY scripts/scrape_via_page.py --entity groups --max-employees 100 +echo "$(date) groups done" + +echo "$(date) --- 3. Page scraper: WeChat messages (1000 convs) ---" +$PY scripts/scrape_via_page.py --entity messages --max-conversations 1000 +echo "$(date) wechat messages done" + +# ---- Phase 2: API sync for remaining entities ---- +echo "$(date) --- 4. API: departments + accounts ---" +$PY -m app.sync.cli sync --entity departments +$PY -m app.sync.cli sync --entity accounts +echo "$(date) accounts done" + +echo "$(date) --- 5. API: WeChat voice calls ---" +$PY -m app.sync.cli sync --entity voice_calls +echo "$(date) voice_calls done" + +echo "$(date) --- 6. API: WA accounts ---" +$PY -m app.sync.cli sync --entity wa_accounts +echo "$(date) wa_accounts done" + +echo "$(date) --- 7. API: WA friends ---" +$PY -m app.sync.cli sync --entity wa_friends +echo "$(date) wa_friends done" + +echo "$(date) --- 8. API: WA voice calls ---" +$PY -m app.sync.cli sync --entity wa_voice_calls +echo "$(date) wa_voice_calls done" + +echo "$(date) --- 9. API: Voice clips ---" +$PY -m app.sync.cli sync --entity voice_clips +echo "$(date) voice_clips done" + +echo "$(date) --- 10. API: Voice clip audio download ---" +$PY -m app.sync.cli sync --entity voice_clip_audio +echo "$(date) voice_clip_audio done" + +echo "$(date) --- 11. API: Call recordings download ---" +$PY -m app.sync.cli sync --entity call_recordings +echo "$(date) call_recordings done" + +echo "$(date) --- 12. API: Message images download ---" +$PY -m app.sync.cli sync --entity message_images +echo "$(date) message_images done" + +echo "$(date) --- 13. API: WA messages (300 req cap) ---" +$PY -m app.sync.cli sync --entity wa_messages +echo "$(date) wa_messages done" + +echo "$(date) --- 14. API: WeChat messages (additional, 300 req cap) ---" +$PY -m app.sync.cli sync --entity messages +echo "$(date) messages done" + +echo "==============================================" +echo "$(date) === FULL RE-SYNC COMPLETE ===" +echo "==============================================" + +# Show final data summary +$PY -c " +import sqlite3 +con = sqlite3.connect('smoke.db') +c = con.cursor() +print() +print('=== FINAL DATA SUMMARY ===') +for label, sql in [ + ('wechat accounts', \"SELECT COUNT(*) FROM wechat_account\"), + ('whatsapp accounts', \"SELECT COUNT(*) FROM whatsapp_account\"), + ('wechat customers', \"SELECT COUNT(*) FROM customer WHERE channel='wechat'\"), + ('whatsapp customers', \"SELECT COUNT(*) FROM customer WHERE channel='whatsapp'\"), + ('wechat groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='wechat'\"), + ('whatsapp groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='whatsapp'\"), + ('wechat messages', \"SELECT COUNT(*) FROM message WHERE channel='wechat'\"), + ('whatsapp messages', \"SELECT COUNT(*) FROM message WHERE channel='whatsapp'\"), + ('voice calls', \"SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')\"), + ('voice clips', 'SELECT COUNT(*) FROM voice_clip'), + ('media files', 'SELECT COUNT(*) FROM media'), +]: + print(f' {label:22s} {c.execute(sql).fetchone()[0]:>6}') +"