feat(sync): 聊天图片本地化(message media_hash+download_message_images+local-first rendering)
- Message 模型新增 media_hash 列 (sha256 FK -> Media.media_hash)
- Alembic 迁移 e5f6a7b8c9d0_message_media_hash (batch_alter_table, SQLite/MySQL 通用)
- app/scraper/media.py:download_message_images 批量下载器,镜像 download_call_recordings
- 候选: media_hash IS NULL AND channel IN ('wechat','whatsapp') AND type IN ('image','video','file') AND 任一 file_*_url LIKE 'http%'
- image 类型优先 file_th_url (缩略图,小快省),video/file 优先 file_url
- per-row try/except + per-run tried set,失败不挂循环
- sync_message_images orchestrator 入口 + ENTITY_MESSAGE_IMAGES 常量 (提交自身 SyncRun,镜像 call_recordings)
- scheduler: _run_message_images + ENTITY_FUNCS['message_images'] + 加入 daily all job (尾部)
- MessageOut schema 暴露 media_hash 字段
- MessageBubble.vue local-first: media_hash 优先 /api/media/{hash},否则回退 OSS file_url/file_th_url/file_hd_url
- types.ts: Message 接口添加 media_hash 字段
- 新增 test_message_images_downloader.py (6 tests, 镜像 test_recording_downloader)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -37,13 +37,14 @@ import httpx
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Media, PhoneCall, VoiceClip
|
||||
from app.db.models import Media, Message, PhoneCall, VoiceClip
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from app.storage.object_store import ObjectStore
|
||||
from app.storage.repos import upsert_media
|
||||
|
||||
__all__ = [
|
||||
"download_call_recordings",
|
||||
"download_message_images",
|
||||
"download_message_media",
|
||||
"download_url",
|
||||
"download_voice_clip_audio",
|
||||
@@ -374,3 +375,140 @@ async def download_voice_clip_audio(
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
|
||||
def _pick_message_image_url(
|
||||
*,
|
||||
type_: str | None,
|
||||
file_url: str | None,
|
||||
file_th_url: str | None,
|
||||
file_hd_url: str | None,
|
||||
) -> str | None:
|
||||
"""Pick the best downloadable http(s) URL for a message media row.
|
||||
|
||||
For ``image`` messages, prefer the thumbnail (``file_th_url``): thumbnails
|
||||
are small, fast to fetch, and exactly what the chat timeline renders, so
|
||||
localizing the thumbnail is sufficient for display and easy on disk + bandwidth.
|
||||
Falls back to ``file_url`` then ``file_hd_url`` when no http(s) thumbnail is
|
||||
available.
|
||||
|
||||
For every other media type (``video`` / ``file``), prefer ``file_url`` (the
|
||||
original), then ``file_hd_url``, then ``file_th_url``.
|
||||
|
||||
Returns ``None`` when no candidate is an http(s) URL (e.g. type-3 opaque ids
|
||||
or relative filenames). Only ``http://`` / ``https://`` URLs are downloadable.
|
||||
"""
|
||||
if type_ == "image":
|
||||
order = (file_th_url, file_url, file_hd_url)
|
||||
else:
|
||||
order = (file_url, file_hd_url, file_th_url)
|
||||
for url in order:
|
||||
if _is_http_url(url):
|
||||
return url # type: ignore[return-value]
|
||||
return None
|
||||
|
||||
|
||||
async def download_message_images(
|
||||
client: YunkeClient,
|
||||
store: ObjectStore,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
batch_size: int = 50,
|
||||
) -> tuple[int, int]:
|
||||
"""Download chat-message media (image / video / file) OSS URLs into the
|
||||
local object store and link them via ``message.media_hash``.
|
||||
|
||||
Iterates ``message`` rows that carry an http(s) URL on any of
|
||||
``file_url`` / ``file_th_url`` / ``file_hd_url``, have ``type`` in
|
||||
('image', 'video', 'file'), are on the wechat / whatsapp channel, and do
|
||||
NOT yet carry a ``media_hash``. For each: pick the best available URL
|
||||
(image: thumbnail-first; other: original-first — see
|
||||
:func:`_pick_message_image_url`), fetch bytes (:func:`download_url`),
|
||||
store content-addressed (:meth:`ObjectStore.put_by_hash`, sha256 dedup),
|
||||
upsert a ``Media`` row, and write ``media_hash`` back onto the message.
|
||||
|
||||
Returns ``(attempted, localized)``. Resilient: a per-row download/store
|
||||
failure is logged and counted as attempted-but-not-localized; no exception
|
||||
escapes. Idempotent within a run: a per-run ``tried`` set excludes already-
|
||||
attempted msg_ids from later batches so a persistently-failing row is NOT
|
||||
re-fetched infinitely (it stays ``media_hash IS NULL`` and is retried on
|
||||
the NEXT run). Rows already carrying ``media_hash`` are skipped; identical
|
||||
bytes are stored once.
|
||||
|
||||
Mirrors :func:`download_call_recordings` / :func:`download_voice_clip_audio`
|
||||
exactly. The message ``type`` label filter restricts to media-bearing rows
|
||||
so text / system / unknown messages are never selected.
|
||||
"""
|
||||
attempted = 0
|
||||
localized = 0
|
||||
tried: set[str] = set()
|
||||
while True:
|
||||
stmt = (
|
||||
sa.select(
|
||||
Message.msg_id,
|
||||
Message.type,
|
||||
Message.channel,
|
||||
Message.file_url,
|
||||
Message.file_th_url,
|
||||
Message.file_hd_url,
|
||||
)
|
||||
.where(Message.media_hash.is_(None))
|
||||
.where(Message.channel.in_(("wechat", "whatsapp")))
|
||||
.where(Message.type.in_(("image", "video", "file")))
|
||||
.where(
|
||||
sa.or_(
|
||||
Message.file_url.like("http%"),
|
||||
Message.file_th_url.like("http%"),
|
||||
Message.file_hd_url.like("http%"),
|
||||
)
|
||||
)
|
||||
.limit(batch_size)
|
||||
)
|
||||
if tried:
|
||||
stmt = stmt.where(Message.msg_id.not_in(tried))
|
||||
rows = (await session.execute(stmt)).all()
|
||||
if not rows:
|
||||
break
|
||||
for msg_id, type_, channel, file_url, file_th_url, file_hd_url in rows:
|
||||
attempted += 1
|
||||
tried.add(msg_id)
|
||||
url = _pick_message_image_url(
|
||||
type_=type_,
|
||||
file_url=file_url,
|
||||
file_th_url=file_th_url,
|
||||
file_hd_url=file_hd_url,
|
||||
)
|
||||
if url is None:
|
||||
# WHERE clause matched at least one http(s) URL, but the
|
||||
# preferred slot for this type isn't http(s) — defensive, in
|
||||
# practice the OR above guarantees _pick returns a value.
|
||||
continue
|
||||
try:
|
||||
data = await download_url(client, url)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
logger.warning(
|
||||
"message media download failed for %s: %s", msg_id, exc
|
||||
)
|
||||
continue
|
||||
ext = _ext_from_url(url)
|
||||
sha256_hex, path = await store.put_by_hash(data, ext=ext)
|
||||
await upsert_media(
|
||||
session,
|
||||
media_hash=sha256_hex,
|
||||
channel=channel or "wechat",
|
||||
media_type=type_,
|
||||
original_url=file_url,
|
||||
original_th_url=file_th_url,
|
||||
original_hd_url=file_hd_url,
|
||||
local_path=str(path),
|
||||
size_bytes=len(data),
|
||||
mime_type=guess_mime_type(ext),
|
||||
)
|
||||
await session.execute(
|
||||
sa.update(Message)
|
||||
.where(Message.msg_id == msg_id)
|
||||
.values(media_hash=sha256_hex)
|
||||
)
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
Reference in New Issue
Block a user