diff --git a/yunke-server/scripts/full_sync_all.sh b/yunke-server/scripts/full_sync_all.sh index 59b58e2..53907a3 100644 --- a/yunke-server/scripts/full_sync_all.sh +++ b/yunke-server/scripts/full_sync_all.sh @@ -1,78 +1,38 @@ #!/bin/bash -# FULL re-sync: ALL channels, ALL entities (page scraper + API) -# Runs after the page scraper (full_resync.sh) completes. +# FULL page-based re-sync: ALL channels, ALL entities — ZERO API calls, ZERO 风控 cd "$(dirname "$0")/.." export DB_URL="sqlite+aiosqlite:///./smoke.db" PY=".venv/Scripts/python.exe" echo "==============================================" -echo "$(date) === COMPREHENSIVE FULL RE-SYNC ===" +echo "$(date) === FULL PAGE-BASED RE-SYNC (NO API) ===" echo "==============================================" -# ---- Phase 1: Page scraper (zero 风控) ---- -echo "$(date) --- 1. Page scraper: WeChat customers ---" +echo "$(date) --- 1/10 WeChat accounts ---" +$PY scripts/scrape_via_page.py --entity accounts +echo "$(date) --- 2/10 WA accounts ---" +$PY scripts/scrape_via_page.py --entity wa-accounts +echo "$(date) --- 3/10 WeChat customers ---" $PY scripts/scrape_via_page.py --entity friends --max-employees 100 -echo "$(date) friends done" - -echo "$(date) --- 2. Page scraper: WeChat groups ---" +echo "$(date) --- 4/10 WA customers ---" +$PY scripts/scrape_via_page.py --entity wa-friends +echo "$(date) --- 5/10 WeChat groups ---" $PY scripts/scrape_via_page.py --entity groups --max-employees 100 -echo "$(date) groups done" - -echo "$(date) --- 3. Page scraper: WeChat messages (1000 convs) ---" +echo "$(date) --- 6/10 Voice calls (WeChat + WA) ---" +$PY scripts/scrape_via_page.py --entity voice-calls +echo "$(date) --- 7/10 Voice clips ---" +$PY scripts/scrape_via_page.py --entity voice-clips +echo "$(date) --- 8/10 WeChat messages ---" $PY scripts/scrape_via_page.py --entity messages --max-conversations 1000 -echo "$(date) wechat messages done" - -# ---- Phase 2: API sync for remaining entities ---- -echo "$(date) --- 4. API: departments + accounts ---" -$PY -m app.sync.cli sync --entity departments -$PY -m app.sync.cli sync --entity accounts -echo "$(date) accounts done" - -echo "$(date) --- 5. API: WeChat voice calls ---" -$PY -m app.sync.cli sync --entity voice_calls -echo "$(date) voice_calls done" - -echo "$(date) --- 6. API: WA accounts ---" -$PY -m app.sync.cli sync --entity wa_accounts -echo "$(date) wa_accounts done" - -echo "$(date) --- 7. API: WA friends ---" -$PY -m app.sync.cli sync --entity wa_friends -echo "$(date) wa_friends done" - -echo "$(date) --- 8. API: WA voice calls ---" -$PY -m app.sync.cli sync --entity wa_voice_calls -echo "$(date) wa_voice_calls done" - -echo "$(date) --- 9. API: Voice clips ---" -$PY -m app.sync.cli sync --entity voice_clips -echo "$(date) voice_clips done" - -echo "$(date) --- 10. API: Voice clip audio download ---" -$PY -m app.sync.cli sync --entity voice_clip_audio -echo "$(date) voice_clip_audio done" - -echo "$(date) --- 11. API: Call recordings download ---" -$PY -m app.sync.cli sync --entity call_recordings -echo "$(date) call_recordings done" - -echo "$(date) --- 12. API: Message images download ---" -$PY -m app.sync.cli sync --entity message_images -echo "$(date) message_images done" - -echo "$(date) --- 13. API: WA messages (300 req cap) ---" -$PY -m app.sync.cli sync --entity wa_messages -echo "$(date) wa_messages done" - -echo "$(date) --- 14. API: WeChat messages (additional, 300 req cap) ---" -$PY -m app.sync.cli sync --entity messages -echo "$(date) messages done" +echo "$(date) --- 9/10 WA messages ---" +$PY scripts/scrape_via_page.py --entity wa-messages --max-conversations 500 +echo "$(date) --- 10/10 Media download (audio+images) ---" +$PY scripts/scrape_via_page.py --entity media echo "==============================================" echo "$(date) === FULL RE-SYNC COMPLETE ===" echo "==============================================" -# Show final data summary $PY -c " import sqlite3 con = sqlite3.connect('smoke.db')