feat(scraper): download_call_recordings 下载通话录音到本地 Media + 回写 record_media_hash
复用 download_url/put_by_hash/upsert_media;幂等+resilient(tried 集合防死循环);media_type='audio'。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -37,12 +37,13 @@ import httpx
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Media
|
||||
from app.db.models import Media, PhoneCall
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from app.storage.object_store import ObjectStore
|
||||
from app.storage.repos import upsert_media
|
||||
|
||||
__all__ = [
|
||||
"download_call_recordings",
|
||||
"download_message_media",
|
||||
"download_url",
|
||||
"guess_mime_type",
|
||||
@@ -220,3 +221,75 @@ async def download_message_media(
|
||||
raw_json=msg.get("raw_json"),
|
||||
)
|
||||
return media_hash
|
||||
|
||||
|
||||
async def download_call_recordings(
|
||||
client: YunkeClient,
|
||||
store: ObjectStore,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
batch_size: int = 50,
|
||||
) -> tuple[int, int]:
|
||||
"""Download call recordings (``phone_call.record_file`` OSS URLs) into the
|
||||
local object store and link them via ``phone_call.record_media_hash``.
|
||||
|
||||
Iterates ``phone_call`` rows that carry an http(s) ``record_file`` but no
|
||||
``record_media_hash`` yet. For each: fetch bytes (:func:`download_url`),
|
||||
store content-addressed (:meth:`ObjectStore.put_by_hash`, sha256 dedup),
|
||||
upsert a ``Media`` row (``media_type='audio'``), and write
|
||||
``record_media_hash`` back onto the call.
|
||||
|
||||
Returns ``(attempted, localized)``. Resilient: a per-row download/store
|
||||
failure is logged and counted as attempted-but-not-localized; no exception
|
||||
escapes. Idempotent within a run: a per-run ``tried`` set excludes already-
|
||||
attempted call_ids from later batches so a persistently-failing row is NOT
|
||||
re-fetched infinitely (it stays ``record_media_hash IS NULL`` and is retried
|
||||
on the NEXT run, which is desirable). Rows already carrying
|
||||
``record_media_hash`` are skipped; identical bytes are stored once.
|
||||
"""
|
||||
attempted = 0
|
||||
localized = 0
|
||||
tried: set[str] = set()
|
||||
while True:
|
||||
stmt = (
|
||||
sa.select(PhoneCall.call_id, PhoneCall.record_file, PhoneCall.channel)
|
||||
.where(
|
||||
PhoneCall.record_file.is_not(None),
|
||||
PhoneCall.record_file.like("http%"),
|
||||
PhoneCall.record_media_hash.is_(None),
|
||||
)
|
||||
.limit(batch_size)
|
||||
)
|
||||
if tried:
|
||||
stmt = stmt.where(PhoneCall.call_id.not_in(tried))
|
||||
rows = (await session.execute(stmt)).all()
|
||||
if not rows:
|
||||
break
|
||||
for call_id, record_file, channel in rows:
|
||||
attempted += 1
|
||||
tried.add(call_id)
|
||||
try:
|
||||
data = await download_url(client, record_file)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
logger.warning("recording download failed for %s: %s", call_id, exc)
|
||||
continue
|
||||
ext = _ext_from_url(record_file)
|
||||
sha256_hex, path = await store.put_by_hash(data, ext=ext)
|
||||
await upsert_media(
|
||||
session,
|
||||
media_hash=sha256_hex,
|
||||
channel=channel or "wechat",
|
||||
media_type="audio",
|
||||
original_url=record_file,
|
||||
local_path=str(path),
|
||||
size_bytes=len(data),
|
||||
mime_type=guess_mime_type(ext),
|
||||
)
|
||||
await session.execute(
|
||||
sa.update(PhoneCall)
|
||||
.where(PhoneCall.call_id == call_id)
|
||||
.values(record_media_hash=sha256_hex)
|
||||
)
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
Reference in New Issue
Block a user