feat(scraper): download_call_recordings 下载通话录音到本地 Media + 回写 record_media_hash

复用 download_url/put_by_hash/upsert_media;幂等+resilient(tried 集合防死循环);media_type='audio'。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-07 16:30:29 +08:00
parent 8f70ac5220
commit 1e399ff8c0
2 changed files with 284 additions and 1 deletions
+74 -1
View File
@@ -37,12 +37,13 @@ import httpx
import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Media
from app.db.models import Media, PhoneCall
from app.session.yunke_client import YunkeClient
from app.storage.object_store import ObjectStore
from app.storage.repos import upsert_media
__all__ = [
"download_call_recordings",
"download_message_media",
"download_url",
"guess_mime_type",
@@ -220,3 +221,75 @@ async def download_message_media(
raw_json=msg.get("raw_json"),
)
return media_hash
async def download_call_recordings(
client: YunkeClient,
store: ObjectStore,
session: AsyncSession,
*,
batch_size: int = 50,
) -> tuple[int, int]:
"""Download call recordings (``phone_call.record_file`` OSS URLs) into the
local object store and link them via ``phone_call.record_media_hash``.
Iterates ``phone_call`` rows that carry an http(s) ``record_file`` but no
``record_media_hash`` yet. For each: fetch bytes (:func:`download_url`),
store content-addressed (:meth:`ObjectStore.put_by_hash`, sha256 dedup),
upsert a ``Media`` row (``media_type='audio'``), and write
``record_media_hash`` back onto the call.
Returns ``(attempted, localized)``. Resilient: a per-row download/store
failure is logged and counted as attempted-but-not-localized; no exception
escapes. Idempotent within a run: a per-run ``tried`` set excludes already-
attempted call_ids from later batches so a persistently-failing row is NOT
re-fetched infinitely (it stays ``record_media_hash IS NULL`` and is retried
on the NEXT run, which is desirable). Rows already carrying
``record_media_hash`` are skipped; identical bytes are stored once.
"""
attempted = 0
localized = 0
tried: set[str] = set()
while True:
stmt = (
sa.select(PhoneCall.call_id, PhoneCall.record_file, PhoneCall.channel)
.where(
PhoneCall.record_file.is_not(None),
PhoneCall.record_file.like("http%"),
PhoneCall.record_media_hash.is_(None),
)
.limit(batch_size)
)
if tried:
stmt = stmt.where(PhoneCall.call_id.not_in(tried))
rows = (await session.execute(stmt)).all()
if not rows:
break
for call_id, record_file, channel in rows:
attempted += 1
tried.add(call_id)
try:
data = await download_url(client, record_file)
except (httpx.HTTPError, OSError) as exc:
logger.warning("recording download failed for %s: %s", call_id, exc)
continue
ext = _ext_from_url(record_file)
sha256_hex, path = await store.put_by_hash(data, ext=ext)
await upsert_media(
session,
media_hash=sha256_hex,
channel=channel or "wechat",
media_type="audio",
original_url=record_file,
local_path=str(path),
size_bytes=len(data),
mime_type=guess_mime_type(ext),
)
await session.execute(
sa.update(PhoneCall)
.where(PhoneCall.call_id == call_id)
.values(record_media_hash=sha256_hex)
)
localized += 1
await session.commit()
return attempted, localized