feat(sync): 聊天图片本地化(message media_hash+download_message_images+local-first rendering)

- Message 模型新增 media_hash 列 (sha256 FK -> Media.media_hash)
- Alembic 迁移 e5f6a7b8c9d0_message_media_hash (batch_alter_table, SQLite/MySQL 通用)
- app/scraper/media.py:download_message_images 批量下载器,镜像 download_call_recordings
  - 候选: media_hash IS NULL AND channel IN ('wechat','whatsapp') AND type IN ('image','video','file') AND 任一 file_*_url LIKE 'http%'
  - image 类型优先 file_th_url (缩略图,小快省),video/file 优先 file_url
  - per-row try/except + per-run tried set,失败不挂循环
- sync_message_images orchestrator 入口 + ENTITY_MESSAGE_IMAGES 常量 (提交自身 SyncRun,镜像 call_recordings)
- scheduler: _run_message_images + ENTITY_FUNCS['message_images'] + 加入 daily all job (尾部)
- MessageOut schema 暴露 media_hash 字段
- MessageBubble.vue local-first: media_hash 优先 /api/media/{hash},否则回退 OSS file_url/file_th_url/file_hd_url
- types.ts: Message 接口添加 media_hash 字段
- 新增 test_message_images_downloader.py (6 tests, 镜像 test_recording_downloader)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-11 18:24:11 +08:00
parent af7f8d71c8
commit 5ce33232c1
9 changed files with 678 additions and 5 deletions
+63
View File
@@ -53,6 +53,7 @@ from app.db.models import Department, SyncRun, WechatAccount, WhatsappAccount
from app.scraper.call import scrape_phone_calls, scrape_sms
from app.scraper.media import (
download_call_recordings,
download_message_images,
download_message_media,
download_voice_clip_audio,
)
@@ -92,11 +93,13 @@ logger = logging.getLogger(__name__)
__all__ = [
"ENTITY_CALL_RECORDINGS",
"ENTITY_MESSAGE_IMAGES",
"ENTITY_VOICE_CLIP_AUDIO",
"SyncResult",
"sync_all_wechat",
"sync_call_recordings",
"sync_departments",
"sync_message_images",
"sync_phone_calls",
"sync_sms",
"sync_voice_clip_audio",
@@ -157,6 +160,12 @@ ENTITY_VOICE_CLIP = "voice_clip"
# object store (records a SyncRun with this entity_type; no SyncState row
# because the voice_clip table itself is the cursor). Mirrors call_recordings.
ENTITY_VOICE_CLIP_AUDIO = "voice_clip_audio"
# Message image / media localization pass: not a scrape — reads already-mirrored
# ``message`` rows whose ``file_url`` / ``file_th_url`` / ``file_hd_url`` point
# at OSS and have no ``media_hash`` yet, downloads each into the local object
# store (records a SyncRun with this entity_type; no SyncState row because the
# message table itself is the cursor). Mirrors call_recordings / voice_clip_audio.
ENTITY_MESSAGE_IMAGES = "message_images"
CHANNEL_WECHAT = "wechat"
CHANNEL_WHATSAPP = "whatsapp"
@@ -1152,6 +1161,60 @@ async def sync_voice_clip_audio(
)
async def sync_message_images(
session: AsyncSession, client: YunkeClient, store: ObjectStore
) -> SyncResult:
"""Download localized copies of chat-message media for already-mirrored
messages.
Not a scrape: reads local ``message`` rows whose ``file_url`` /
``file_th_url`` / ``file_hd_url`` point at OSS, have ``type`` in
('image', 'video', 'file'), are on the wechat / whatsapp channel, and have
no ``media_hash`` yet; downloads each and links it. Records a ``SyncRun``
(``entity_type='message_images'``); ``fetched`` = messages needing
download, ``upserted`` = messages localized.
Mirrors :func:`sync_call_recordings` / :func:`sync_voice_clip_audio`
exactly (same commit semantics): commits its own ``SyncRun`` row because
:func:`download_message_images` commits per batch internally, so the run
bookkeeping must land in the same transaction boundary to stay consistent
with what was actually persisted.
Failure policy mirrors every other ``sync_*``: a downloader exception is
recorded as a ``status='failed'`` ``SyncRun`` and a failed
:class:`SyncResult` is returned (no re-raise), so a broken image pass does
NOT abort a multi-entity batch.
"""
run = await _start_run(session, ENTITY_MESSAGE_IMAGES)
try:
attempted, localized = await download_message_images(
client, store, session
)
await _finish_run(
session, run, status="success", fetched=attempted, upserted=localized
)
await session.commit()
return SyncResult(
entity=ENTITY_MESSAGE_IMAGES,
status="success",
fetched=attempted,
upserted=localized,
)
except Exception as exc:
logger.exception("sync_message_images failed")
await _finish_run(
session, run, status="failed", fetched=0, upserted=0, error=repr(exc)
)
await session.commit()
return SyncResult(
entity=ENTITY_MESSAGE_IMAGES,
status="failed",
fetched=0,
upserted=0,
error=repr(exc),
)
# ---------------------------------------------------------------------------
# Coordinator
# ---------------------------------------------------------------------------