feat(sync): 聊天图片本地化(message media_hash+download_message_images+local-first rendering)

- Message 模型新增 media_hash 列 (sha256 FK -> Media.media_hash)
- Alembic 迁移 e5f6a7b8c9d0_message_media_hash (batch_alter_table, SQLite/MySQL 通用)
- app/scraper/media.py:download_message_images 批量下载器,镜像 download_call_recordings
  - 候选: media_hash IS NULL AND channel IN ('wechat','whatsapp') AND type IN ('image','video','file') AND 任一 file_*_url LIKE 'http%'
  - image 类型优先 file_th_url (缩略图,小快省),video/file 优先 file_url
  - per-row try/except + per-run tried set,失败不挂循环
- sync_message_images orchestrator 入口 + ENTITY_MESSAGE_IMAGES 常量 (提交自身 SyncRun,镜像 call_recordings)
- scheduler: _run_message_images + ENTITY_FUNCS['message_images'] + 加入 daily all job (尾部)
- MessageOut schema 暴露 media_hash 字段
- MessageBubble.vue local-first: media_hash 优先 /api/media/{hash},否则回退 OSS file_url/file_th_url/file_hd_url
- types.ts: Message 接口添加 media_hash 字段
- 新增 test_message_images_downloader.py (6 tests, 镜像 test_recording_downloader)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-11 18:24:11 +08:00
parent af7f8d71c8
commit 5ce33232c1
9 changed files with 678 additions and 5 deletions
+139 -1
View File
@@ -37,13 +37,14 @@ import httpx
import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Media, PhoneCall, VoiceClip
from app.db.models import Media, Message, PhoneCall, VoiceClip
from app.session.yunke_client import YunkeClient
from app.storage.object_store import ObjectStore
from app.storage.repos import upsert_media
__all__ = [
"download_call_recordings",
"download_message_images",
"download_message_media",
"download_url",
"download_voice_clip_audio",
@@ -374,3 +375,140 @@ async def download_voice_clip_audio(
localized += 1
await session.commit()
return attempted, localized
def _pick_message_image_url(
*,
type_: str | None,
file_url: str | None,
file_th_url: str | None,
file_hd_url: str | None,
) -> str | None:
"""Pick the best downloadable http(s) URL for a message media row.
For ``image`` messages, prefer the thumbnail (``file_th_url``): thumbnails
are small, fast to fetch, and exactly what the chat timeline renders, so
localizing the thumbnail is sufficient for display and easy on disk + bandwidth.
Falls back to ``file_url`` then ``file_hd_url`` when no http(s) thumbnail is
available.
For every other media type (``video`` / ``file``), prefer ``file_url`` (the
original), then ``file_hd_url``, then ``file_th_url``.
Returns ``None`` when no candidate is an http(s) URL (e.g. type-3 opaque ids
or relative filenames). Only ``http://`` / ``https://`` URLs are downloadable.
"""
if type_ == "image":
order = (file_th_url, file_url, file_hd_url)
else:
order = (file_url, file_hd_url, file_th_url)
for url in order:
if _is_http_url(url):
return url # type: ignore[return-value]
return None
async def download_message_images(
client: YunkeClient,
store: ObjectStore,
session: AsyncSession,
*,
batch_size: int = 50,
) -> tuple[int, int]:
"""Download chat-message media (image / video / file) OSS URLs into the
local object store and link them via ``message.media_hash``.
Iterates ``message`` rows that carry an http(s) URL on any of
``file_url`` / ``file_th_url`` / ``file_hd_url``, have ``type`` in
('image', 'video', 'file'), are on the wechat / whatsapp channel, and do
NOT yet carry a ``media_hash``. For each: pick the best available URL
(image: thumbnail-first; other: original-first — see
:func:`_pick_message_image_url`), fetch bytes (:func:`download_url`),
store content-addressed (:meth:`ObjectStore.put_by_hash`, sha256 dedup),
upsert a ``Media`` row, and write ``media_hash`` back onto the message.
Returns ``(attempted, localized)``. Resilient: a per-row download/store
failure is logged and counted as attempted-but-not-localized; no exception
escapes. Idempotent within a run: a per-run ``tried`` set excludes already-
attempted msg_ids from later batches so a persistently-failing row is NOT
re-fetched infinitely (it stays ``media_hash IS NULL`` and is retried on
the NEXT run). Rows already carrying ``media_hash`` are skipped; identical
bytes are stored once.
Mirrors :func:`download_call_recordings` / :func:`download_voice_clip_audio`
exactly. The message ``type`` label filter restricts to media-bearing rows
so text / system / unknown messages are never selected.
"""
attempted = 0
localized = 0
tried: set[str] = set()
while True:
stmt = (
sa.select(
Message.msg_id,
Message.type,
Message.channel,
Message.file_url,
Message.file_th_url,
Message.file_hd_url,
)
.where(Message.media_hash.is_(None))
.where(Message.channel.in_(("wechat", "whatsapp")))
.where(Message.type.in_(("image", "video", "file")))
.where(
sa.or_(
Message.file_url.like("http%"),
Message.file_th_url.like("http%"),
Message.file_hd_url.like("http%"),
)
)
.limit(batch_size)
)
if tried:
stmt = stmt.where(Message.msg_id.not_in(tried))
rows = (await session.execute(stmt)).all()
if not rows:
break
for msg_id, type_, channel, file_url, file_th_url, file_hd_url in rows:
attempted += 1
tried.add(msg_id)
url = _pick_message_image_url(
type_=type_,
file_url=file_url,
file_th_url=file_th_url,
file_hd_url=file_hd_url,
)
if url is None:
# WHERE clause matched at least one http(s) URL, but the
# preferred slot for this type isn't http(s) — defensive, in
# practice the OR above guarantees _pick returns a value.
continue
try:
data = await download_url(client, url)
except (httpx.HTTPError, OSError) as exc:
logger.warning(
"message media download failed for %s: %s", msg_id, exc
)
continue
ext = _ext_from_url(url)
sha256_hex, path = await store.put_by_hash(data, ext=ext)
await upsert_media(
session,
media_hash=sha256_hex,
channel=channel or "wechat",
media_type=type_,
original_url=file_url,
original_th_url=file_th_url,
original_hd_url=file_hd_url,
local_path=str(path),
size_bytes=len(data),
mime_type=guess_mime_type(ext),
)
await session.execute(
sa.update(Message)
.where(Message.msg_id == msg_id)
.values(media_hash=sha256_hex)
)
localized += 1
await session.commit()
return attempted, localized