feat(sync): 长语音音频本地化(voice_clip media_hash+download+local-first playback)
Mirror download_call_recordings pattern for voice_clips:
- VoiceClip.media_hash column + Alembic migration (c4d5e6f7a8b9)
- VoiceClipOut.media_hash + frontend VoiceClip.media_hash type
- download_voice_clip_audio in app/scraper/media.py (resilient, sha256-dedup,
same download_url/upsert_media helpers as call recordings)
- sync_voice_clip_audio orchestrator wrapper (commits own SyncRun, mirrors
sync_call_recordings semantics)
- scheduler: _run_voice_clip_audio + ENTITY_FUNCS['voice_clip_audio'] +
appended to daily _run_all job (after voice_clips)
- VoiceClipModal playClip local-first: /api/media/{media_hash} with OSS
oss_file_name fallback (mirrors CallList.playRecord)
Tested: 376 pytest pass; vue-tsc clean; migration applied; 1523/1533 clips
localized (10 pending are genuine upstream HTTP 404s — resilient skip).
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -37,7 +37,7 @@ import httpx
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Media, PhoneCall
|
||||
from app.db.models import Media, PhoneCall, VoiceClip
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from app.storage.object_store import ObjectStore
|
||||
from app.storage.repos import upsert_media
|
||||
@@ -46,6 +46,7 @@ __all__ = [
|
||||
"download_call_recordings",
|
||||
"download_message_media",
|
||||
"download_url",
|
||||
"download_voice_clip_audio",
|
||||
"guess_mime_type",
|
||||
"media_type_from_message",
|
||||
]
|
||||
@@ -293,3 +294,83 @@ async def download_call_recordings(
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
|
||||
async def download_voice_clip_audio(
|
||||
client: YunkeClient,
|
||||
store: ObjectStore,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
batch_size: int = 50,
|
||||
) -> tuple[int, int]:
|
||||
"""Download voice_clip audio (``voice_clip.oss_file_name`` OSS URLs) into
|
||||
the local object store and link them via ``voice_clip.media_hash``.
|
||||
|
||||
Mirrors :func:`download_call_recordings` exactly: iterates ``voice_clip``
|
||||
rows that carry an http(s) ``oss_file_name`` but no ``media_hash`` yet. For
|
||||
each: fetch bytes (:func:`download_url`), store content-addressed
|
||||
(:meth:`ObjectStore.put_by_hash`, sha256 dedup), upsert a ``Media`` row
|
||||
(``media_type='audio'``), and write ``media_hash`` back onto the clip.
|
||||
|
||||
Returns ``(attempted, localized)``. Resilient: a per-row download/store
|
||||
failure is logged and counted as attempted-but-not-localized; no exception
|
||||
escapes. Idempotent within a run: a per-run ``tried`` set excludes already-
|
||||
attempted voice_clip_ids from later batches so a persistently-failing row
|
||||
is NOT re-fetched infinitely (it stays ``media_hash IS NULL`` and is
|
||||
retried on the NEXT run). Rows already carrying ``media_hash`` are skipped;
|
||||
identical bytes are stored once.
|
||||
"""
|
||||
attempted = 0
|
||||
localized = 0
|
||||
tried: set[int] = set()
|
||||
while True:
|
||||
stmt = (
|
||||
sa.select(
|
||||
VoiceClip.voice_clip_id,
|
||||
VoiceClip.oss_file_name,
|
||||
VoiceClip.channel,
|
||||
)
|
||||
.where(
|
||||
VoiceClip.oss_file_name.is_not(None),
|
||||
VoiceClip.oss_file_name.like("http%"),
|
||||
VoiceClip.media_hash.is_(None),
|
||||
)
|
||||
.limit(batch_size)
|
||||
)
|
||||
if tried:
|
||||
stmt = stmt.where(VoiceClip.voice_clip_id.not_in(tried))
|
||||
rows = (await session.execute(stmt)).all()
|
||||
if not rows:
|
||||
break
|
||||
for voice_clip_id, oss_file_name, channel in rows:
|
||||
attempted += 1
|
||||
tried.add(voice_clip_id)
|
||||
try:
|
||||
data = await download_url(client, oss_file_name)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
logger.warning(
|
||||
"voice_clip audio download failed for %s: %s",
|
||||
voice_clip_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
ext = _ext_from_url(oss_file_name)
|
||||
sha256_hex, path = await store.put_by_hash(data, ext=ext)
|
||||
await upsert_media(
|
||||
session,
|
||||
media_hash=sha256_hex,
|
||||
channel=channel or "wechat",
|
||||
media_type="audio",
|
||||
original_url=oss_file_name,
|
||||
local_path=str(path),
|
||||
size_bytes=len(data),
|
||||
mime_type=guess_mime_type(ext),
|
||||
)
|
||||
await session.execute(
|
||||
sa.update(VoiceClip)
|
||||
.where(VoiceClip.voice_clip_id == voice_clip_id)
|
||||
.values(media_hash=sha256_hex)
|
||||
)
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
Reference in New Issue
Block a user