feat(sync): 聊天图片本地化(message media_hash+download_message_images+local-first rendering)
- Message 模型新增 media_hash 列 (sha256 FK -> Media.media_hash)
- Alembic 迁移 e5f6a7b8c9d0_message_media_hash (batch_alter_table, SQLite/MySQL 通用)
- app/scraper/media.py:download_message_images 批量下载器,镜像 download_call_recordings
- 候选: media_hash IS NULL AND channel IN ('wechat','whatsapp') AND type IN ('image','video','file') AND 任一 file_*_url LIKE 'http%'
- image 类型优先 file_th_url (缩略图,小快省),video/file 优先 file_url
- per-row try/except + per-run tried set,失败不挂循环
- sync_message_images orchestrator 入口 + ENTITY_MESSAGE_IMAGES 常量 (提交自身 SyncRun,镜像 call_recordings)
- scheduler: _run_message_images + ENTITY_FUNCS['message_images'] + 加入 daily all job (尾部)
- MessageOut schema 暴露 media_hash 字段
- MessageBubble.vue local-first: media_hash 优先 /api/media/{hash},否则回退 OSS file_url/file_th_url/file_hd_url
- types.ts: Message 接口添加 media_hash 字段
- 新增 test_message_images_downloader.py (6 tests, 镜像 test_recording_downloader)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -53,6 +53,7 @@ from app.db.models import Department, SyncRun, WechatAccount, WhatsappAccount
|
||||
from app.scraper.call import scrape_phone_calls, scrape_sms
|
||||
from app.scraper.media import (
|
||||
download_call_recordings,
|
||||
download_message_images,
|
||||
download_message_media,
|
||||
download_voice_clip_audio,
|
||||
)
|
||||
@@ -92,11 +93,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"ENTITY_CALL_RECORDINGS",
|
||||
"ENTITY_MESSAGE_IMAGES",
|
||||
"ENTITY_VOICE_CLIP_AUDIO",
|
||||
"SyncResult",
|
||||
"sync_all_wechat",
|
||||
"sync_call_recordings",
|
||||
"sync_departments",
|
||||
"sync_message_images",
|
||||
"sync_phone_calls",
|
||||
"sync_sms",
|
||||
"sync_voice_clip_audio",
|
||||
@@ -157,6 +160,12 @@ ENTITY_VOICE_CLIP = "voice_clip"
|
||||
# object store (records a SyncRun with this entity_type; no SyncState row
|
||||
# because the voice_clip table itself is the cursor). Mirrors call_recordings.
|
||||
ENTITY_VOICE_CLIP_AUDIO = "voice_clip_audio"
|
||||
# Message image / media localization pass: not a scrape — reads already-mirrored
|
||||
# ``message`` rows whose ``file_url`` / ``file_th_url`` / ``file_hd_url`` point
|
||||
# at OSS and have no ``media_hash`` yet, downloads each into the local object
|
||||
# store (records a SyncRun with this entity_type; no SyncState row because the
|
||||
# message table itself is the cursor). Mirrors call_recordings / voice_clip_audio.
|
||||
ENTITY_MESSAGE_IMAGES = "message_images"
|
||||
|
||||
CHANNEL_WECHAT = "wechat"
|
||||
CHANNEL_WHATSAPP = "whatsapp"
|
||||
@@ -1152,6 +1161,60 @@ async def sync_voice_clip_audio(
|
||||
)
|
||||
|
||||
|
||||
async def sync_message_images(
|
||||
session: AsyncSession, client: YunkeClient, store: ObjectStore
|
||||
) -> SyncResult:
|
||||
"""Download localized copies of chat-message media for already-mirrored
|
||||
messages.
|
||||
|
||||
Not a scrape: reads local ``message`` rows whose ``file_url`` /
|
||||
``file_th_url`` / ``file_hd_url`` point at OSS, have ``type`` in
|
||||
('image', 'video', 'file'), are on the wechat / whatsapp channel, and have
|
||||
no ``media_hash`` yet; downloads each and links it. Records a ``SyncRun``
|
||||
(``entity_type='message_images'``); ``fetched`` = messages needing
|
||||
download, ``upserted`` = messages localized.
|
||||
|
||||
Mirrors :func:`sync_call_recordings` / :func:`sync_voice_clip_audio`
|
||||
exactly (same commit semantics): commits its own ``SyncRun`` row because
|
||||
:func:`download_message_images` commits per batch internally, so the run
|
||||
bookkeeping must land in the same transaction boundary to stay consistent
|
||||
with what was actually persisted.
|
||||
|
||||
Failure policy mirrors every other ``sync_*``: a downloader exception is
|
||||
recorded as a ``status='failed'`` ``SyncRun`` and a failed
|
||||
:class:`SyncResult` is returned (no re-raise), so a broken image pass does
|
||||
NOT abort a multi-entity batch.
|
||||
"""
|
||||
run = await _start_run(session, ENTITY_MESSAGE_IMAGES)
|
||||
try:
|
||||
attempted, localized = await download_message_images(
|
||||
client, store, session
|
||||
)
|
||||
await _finish_run(
|
||||
session, run, status="success", fetched=attempted, upserted=localized
|
||||
)
|
||||
await session.commit()
|
||||
return SyncResult(
|
||||
entity=ENTITY_MESSAGE_IMAGES,
|
||||
status="success",
|
||||
fetched=attempted,
|
||||
upserted=localized,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("sync_message_images failed")
|
||||
await _finish_run(
|
||||
session, run, status="failed", fetched=0, upserted=0, error=repr(exc)
|
||||
)
|
||||
await session.commit()
|
||||
return SyncResult(
|
||||
entity=ENTITY_MESSAGE_IMAGES,
|
||||
status="failed",
|
||||
fetched=0,
|
||||
upserted=0,
|
||||
error=repr(exc),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coordinator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user