浅念
97 lines
3.6 KiB
Bash
97 lines
3.6 KiB
Bash
#!/bin/bash
|
|
# FULL re-sync: ALL channels, ALL entities (page scraper + API)
|
|
# Runs after the page scraper (full_resync.sh) completes.
|
|
cd "$(dirname "$0")/.."
|
|
export DB_URL="sqlite+aiosqlite:///./smoke.db"
|
|
PY=".venv/Scripts/python.exe"
|
|
|
|
echo "=============================================="
|
|
echo "$(date) === COMPREHENSIVE FULL RE-SYNC ==="
|
|
echo "=============================================="
|
|
|
|
# ---- Phase 1: Page scraper (zero 风控) ----
|
|
echo "$(date) --- 1. Page scraper: WeChat customers ---"
|
|
$PY scripts/scrape_via_page.py --entity friends --max-employees 100
|
|
echo "$(date) friends done"
|
|
|
|
echo "$(date) --- 2. Page scraper: WeChat groups ---"
|
|
$PY scripts/scrape_via_page.py --entity groups --max-employees 100
|
|
echo "$(date) groups done"
|
|
|
|
echo "$(date) --- 3. Page scraper: WeChat messages (1000 convs) ---"
|
|
$PY scripts/scrape_via_page.py --entity messages --max-conversations 1000
|
|
echo "$(date) wechat messages done"
|
|
|
|
# ---- Phase 2: API sync for remaining entities ----
|
|
echo "$(date) --- 4. API: departments + accounts ---"
|
|
$PY -m app.sync.cli sync --entity departments
|
|
$PY -m app.sync.cli sync --entity accounts
|
|
echo "$(date) accounts done"
|
|
|
|
echo "$(date) --- 5. API: WeChat voice calls ---"
|
|
$PY -m app.sync.cli sync --entity voice_calls
|
|
echo "$(date) voice_calls done"
|
|
|
|
echo "$(date) --- 6. API: WA accounts ---"
|
|
$PY -m app.sync.cli sync --entity wa_accounts
|
|
echo "$(date) wa_accounts done"
|
|
|
|
echo "$(date) --- 7. API: WA friends ---"
|
|
$PY -m app.sync.cli sync --entity wa_friends
|
|
echo "$(date) wa_friends done"
|
|
|
|
echo "$(date) --- 8. API: WA voice calls ---"
|
|
$PY -m app.sync.cli sync --entity wa_voice_calls
|
|
echo "$(date) wa_voice_calls done"
|
|
|
|
echo "$(date) --- 9. API: Voice clips ---"
|
|
$PY -m app.sync.cli sync --entity voice_clips
|
|
echo "$(date) voice_clips done"
|
|
|
|
echo "$(date) --- 10. API: Voice clip audio download ---"
|
|
$PY -m app.sync.cli sync --entity voice_clip_audio
|
|
echo "$(date) voice_clip_audio done"
|
|
|
|
echo "$(date) --- 11. API: Call recordings download ---"
|
|
$PY -m app.sync.cli sync --entity call_recordings
|
|
echo "$(date) call_recordings done"
|
|
|
|
echo "$(date) --- 12. API: Message images download ---"
|
|
$PY -m app.sync.cli sync --entity message_images
|
|
echo "$(date) message_images done"
|
|
|
|
echo "$(date) --- 13. API: WA messages (300 req cap) ---"
|
|
$PY -m app.sync.cli sync --entity wa_messages
|
|
echo "$(date) wa_messages done"
|
|
|
|
echo "$(date) --- 14. API: WeChat messages (additional, 300 req cap) ---"
|
|
$PY -m app.sync.cli sync --entity messages
|
|
echo "$(date) messages done"
|
|
|
|
echo "=============================================="
|
|
echo "$(date) === FULL RE-SYNC COMPLETE ==="
|
|
echo "=============================================="
|
|
|
|
# Show final data summary
|
|
$PY -c "
|
|
import sqlite3
|
|
con = sqlite3.connect('smoke.db')
|
|
c = con.cursor()
|
|
print()
|
|
print('=== FINAL DATA SUMMARY ===')
|
|
for label, sql in [
|
|
('wechat accounts', \"SELECT COUNT(*) FROM wechat_account\"),
|
|
('whatsapp accounts', \"SELECT COUNT(*) FROM whatsapp_account\"),
|
|
('wechat customers', \"SELECT COUNT(*) FROM customer WHERE channel='wechat'\"),
|
|
('whatsapp customers', \"SELECT COUNT(*) FROM customer WHERE channel='whatsapp'\"),
|
|
('wechat groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='wechat'\"),
|
|
('whatsapp groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='whatsapp'\"),
|
|
('wechat messages', \"SELECT COUNT(*) FROM message WHERE channel='wechat'\"),
|
|
('whatsapp messages', \"SELECT COUNT(*) FROM message WHERE channel='whatsapp'\"),
|
|
('voice calls', \"SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')\"),
|
|
('voice clips', 'SELECT COUNT(*) FROM voice_clip'),
|
|
('media files', 'SELECT COUNT(*) FROM media'),
|
|
]:
|
|
print(f' {label:22s} {c.execute(sql).fetchone()[0]:>6}')
|
|
"
|