feat(sync): 长语音音频本地化(voice_clip media_hash+download+local-first playback)

Mirror download_call_recordings pattern for voice_clips:
- VoiceClip.media_hash column + Alembic migration (c4d5e6f7a8b9)
- VoiceClipOut.media_hash + frontend VoiceClip.media_hash type
- download_voice_clip_audio in app/scraper/media.py (resilient, sha256-dedup,
  same download_url/upsert_media helpers as call recordings)
- sync_voice_clip_audio orchestrator wrapper (commits own SyncRun, mirrors
  sync_call_recordings semantics)
- scheduler: _run_voice_clip_audio + ENTITY_FUNCS['voice_clip_audio'] +
  appended to daily _run_all job (after voice_clips)
- VoiceClipModal playClip local-first: /api/media/{media_hash} with OSS
  oss_file_name fallback (mirrors CallList.playRecord)

Tested: 376 pytest pass; vue-tsc clean; migration applied; 1523/1533 clips
localized (10 pending are genuine upstream HTTP 404s — resilient skip).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-11 12:06:26 +08:00
parent 8b19c299bb
commit 538a0dc635
8 changed files with 210 additions and 4 deletions
+82 -1
View File
@@ -37,7 +37,7 @@ import httpx
import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Media, PhoneCall
from app.db.models import Media, PhoneCall, VoiceClip
from app.session.yunke_client import YunkeClient
from app.storage.object_store import ObjectStore
from app.storage.repos import upsert_media
@@ -46,6 +46,7 @@ __all__ = [
"download_call_recordings",
"download_message_media",
"download_url",
"download_voice_clip_audio",
"guess_mime_type",
"media_type_from_message",
]
@@ -293,3 +294,83 @@ async def download_call_recordings(
localized += 1
await session.commit()
return attempted, localized
async def download_voice_clip_audio(
client: YunkeClient,
store: ObjectStore,
session: AsyncSession,
*,
batch_size: int = 50,
) -> tuple[int, int]:
"""Download voice_clip audio (``voice_clip.oss_file_name`` OSS URLs) into
the local object store and link them via ``voice_clip.media_hash``.
Mirrors :func:`download_call_recordings` exactly: iterates ``voice_clip``
rows that carry an http(s) ``oss_file_name`` but no ``media_hash`` yet. For
each: fetch bytes (:func:`download_url`), store content-addressed
(:meth:`ObjectStore.put_by_hash`, sha256 dedup), upsert a ``Media`` row
(``media_type='audio'``), and write ``media_hash`` back onto the clip.
Returns ``(attempted, localized)``. Resilient: a per-row download/store
failure is logged and counted as attempted-but-not-localized; no exception
escapes. Idempotent within a run: a per-run ``tried`` set excludes already-
attempted voice_clip_ids from later batches so a persistently-failing row
is NOT re-fetched infinitely (it stays ``media_hash IS NULL`` and is
retried on the NEXT run). Rows already carrying ``media_hash`` are skipped;
identical bytes are stored once.
"""
attempted = 0
localized = 0
tried: set[int] = set()
while True:
stmt = (
sa.select(
VoiceClip.voice_clip_id,
VoiceClip.oss_file_name,
VoiceClip.channel,
)
.where(
VoiceClip.oss_file_name.is_not(None),
VoiceClip.oss_file_name.like("http%"),
VoiceClip.media_hash.is_(None),
)
.limit(batch_size)
)
if tried:
stmt = stmt.where(VoiceClip.voice_clip_id.not_in(tried))
rows = (await session.execute(stmt)).all()
if not rows:
break
for voice_clip_id, oss_file_name, channel in rows:
attempted += 1
tried.add(voice_clip_id)
try:
data = await download_url(client, oss_file_name)
except (httpx.HTTPError, OSError) as exc:
logger.warning(
"voice_clip audio download failed for %s: %s",
voice_clip_id,
exc,
)
continue
ext = _ext_from_url(oss_file_name)
sha256_hex, path = await store.put_by_hash(data, ext=ext)
await upsert_media(
session,
media_hash=sha256_hex,
channel=channel or "wechat",
media_type="audio",
original_url=oss_file_name,
local_path=str(path),
size_bytes=len(data),
mime_type=guess_mime_type(ext),
)
await session.execute(
sa.update(VoiceClip)
.where(VoiceClip.voice_clip_id == voice_clip_id)
.values(media_hash=sha256_hex)
)
localized += 1
await session.commit()
return attempted, localized