浅念
90c28f4c7b
- scrape_via_page:_normalize_moments(createTime秒×1000,缺id丢弃)+ _iter_moments_pages 翻页内核(fetch回调注入,sleep可注入,hashNext假/ 信封异常/max_pages 上限即止)+ scrape_moments(逐员工遍历wechat_account, form小驼峰,首页code≠10000告警cookie失效,换页重试惯例)+CLI注册moments - media:download_moments_images(image_urls JSON数组展开去重,MomentsPost无 回链列→用Media.original_url做已下载标记,分片IN防SQLite变量上限,批间commit) 接入scrape_media第4源 - full_sync_all.sh:9c步moments(media前,同轮图片即可本地化)+汇总加moments计数 - scheduler:4h messages job 末尾追加 moments --max-pages 3 浅扫(全量翻页 由每日full承担);测试断言7条命令序列 - tests/test_scrape_moments.py:归一化(秒→ms/空安全/坏createTime)+翻页内核 (两页累积停止/hashNext假单页/max_pages防死循环/丢id行/坏信封)+图片本地化 (一帖两图/幂等跳过/非http跳过+500容错);全套440测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
75 lines
3.5 KiB
Bash
75 lines
3.5 KiB
Bash
#!/bin/bash
|
|
# FULL page-based re-sync: ALL channels, ALL entities — ZERO API calls, ZERO 风控
|
|
cd "$(dirname "$0")/.."
|
|
# DB_URL:外部已设则沿用(容器里由 docker-compose 注入 MySQL);本地默认 SQLite
|
|
export DB_URL="${DB_URL:-sqlite+aiosqlite:///./smoke.db}"
|
|
# 可移植 python:容器里 python 在 PATH;本地 Windows 用 venv 的解释器(2026-08-21 部署阻塞点#3)
|
|
if [ -f ".venv/Scripts/python.exe" ]; then
|
|
PY=".venv/Scripts/python.exe"
|
|
else
|
|
PY="python"
|
|
fi
|
|
|
|
echo "=============================================="
|
|
echo "$(date) === FULL PAGE-BASED RE-SYNC (NO API) ==="
|
|
echo "=============================================="
|
|
|
|
echo "$(date) --- 1/10 WeChat accounts ---"
|
|
$PY scripts/scrape_via_page.py --entity accounts
|
|
echo "$(date) --- 2/10 WA accounts ---"
|
|
$PY scripts/scrape_via_page.py --entity wa-accounts
|
|
echo "$(date) --- 3/10 WeChat customers ---"
|
|
$PY scripts/scrape_via_page.py --entity friends --max-employees 100
|
|
echo "$(date) --- 4/10 WA customers ---"
|
|
$PY scripts/scrape_via_page.py --entity wa-friends
|
|
echo "$(date) --- 5/10 WeChat groups ---"
|
|
$PY scripts/scrape_via_page.py --entity groups --max-employees 100
|
|
echo "$(date) --- 5b/10 WA groups ---"
|
|
$PY scripts/scrape_via_page.py --entity wa-groups
|
|
echo "$(date) --- 6/10 Voice calls (WeChat + WA) ---"
|
|
$PY scripts/scrape_via_page.py --entity voice-calls
|
|
# 语音重扫会把 record_media_hash 覆盖回 None —— 必须紧跟录音重链接
|
|
$PY -m app.sync.cli sync --entity call_recordings
|
|
echo "$(date) --- 7/10 Voice clips ---"
|
|
$PY scripts/scrape_via_page.py --entity voice-clips
|
|
echo "$(date) --- 8/10 WeChat messages ---"
|
|
$PY scripts/scrape_via_page.py --entity messages --max-conversations 99999
|
|
echo "$(date) --- 8b/10 WeChat group messages + members ---"
|
|
$PY scripts/scrape_via_page.py --entity group-messages --max-conversations 99999
|
|
$PY -m scripts.backfill_group_members
|
|
echo "$(date) --- 9/10 WA messages ---"
|
|
$PY scripts/scrape_via_page.py --entity wa-messages --max-conversations 99999
|
|
echo "$(date) --- 9b/10 WA group messages ---"
|
|
$PY scripts/scrape_via_page.py --entity wa-group-messages --max-conversations 99999
|
|
echo "$(date) --- 9c/10 Moments (员工朋友圈) ---"
|
|
$PY scripts/scrape_via_page.py --entity moments
|
|
echo "$(date) --- 10/10 Media download (audio+images) ---"
|
|
$PY scripts/scrape_via_page.py --entity media
|
|
|
|
echo "=============================================="
|
|
echo "$(date) === FULL RE-SYNC COMPLETE ==="
|
|
echo "=============================================="
|
|
|
|
$PY -c "
|
|
import sqlite3
|
|
con = sqlite3.connect('smoke.db')
|
|
c = con.cursor()
|
|
print()
|
|
print('=== FINAL DATA SUMMARY ===')
|
|
for label, sql in [
|
|
('wechat accounts', \"SELECT COUNT(*) FROM wechat_account\"),
|
|
('whatsapp accounts', \"SELECT COUNT(*) FROM whatsapp_account\"),
|
|
('wechat customers', \"SELECT COUNT(*) FROM customer WHERE channel='wechat'\"),
|
|
('whatsapp customers', \"SELECT COUNT(*) FROM customer WHERE channel='whatsapp'\"),
|
|
('wechat groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='wechat'\"),
|
|
('whatsapp groups', \"SELECT COUNT(*) FROM chat_group WHERE channel='whatsapp'\"),
|
|
('wechat messages', \"SELECT COUNT(*) FROM message WHERE channel='wechat'\"),
|
|
('whatsapp messages', \"SELECT COUNT(*) FROM message WHERE channel='whatsapp'\"),
|
|
('voice calls', \"SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')\"),
|
|
('voice clips', 'SELECT COUNT(*) FROM voice_clip'),
|
|
('moments posts', 'SELECT COUNT(*) FROM moments_post'),
|
|
('media files', 'SELECT COUNT(*) FROM media'),
|
|
]:
|
|
print(f' {label:22s} {c.execute(sql).fetchone()[0]:>6}')
|
|
"
|