feat(scraper): download_call_recordings 下载通话录音到本地 Media + 回写 record_media_hash

复用 download_url/put_by_hash/upsert_media；幂等+resilient（tried 集合防死循环）；media_type='audio'。 Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-07 16:30:29 +08:00
parent 8f70ac5220
commit 1e399ff8c0
2 changed files with 284 additions and 1 deletions
@@ -0,0 +1,210 @@
+"""Tests for :func:`app.scraper.media.download_call_recordings`.
+
+Covers the call-recording batch downloader that localizes
+``phone_call.record_file`` OSS URLs into the local object store, writes a
+``Media`` row per blob, and links the call back via ``record_media_hash``.
+
+* Happy path: one call -> downloaded, stored, upserted, linked.
+* Idempotent: a call already carrying ``record_media_hash`` is excluded from
+  the candidate set (``attempted == 0``).
+* Resilient: a persistent 500 does NOT hang the loop (the per-run ``tried``
+  set excludes the failing call_id from later batches) and returns
+  ``(attempted=1, localized=0)`` with no exception escaping.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+import httpx
+import respx
+import sqlalchemy as sa
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.db.base import Base
+from app.db.models import Media, PhoneCall
+from app.scraper.media import download_call_recordings
+from app.session.yunke_client import YunkeClient
+from app.storage.object_store import ObjectStore
+
+BASE = "https://crm2.yunkecn.com"
+# A realistic OSS recording URL (same shape as the yunke-pcfile call
+# recordings): public GET, .mp3 suffix.
+OSS_MP3 = (
+    "https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/8m0atd/"
+    "x/ossCallback/a_b.mp3"
+)
+
+USERID = "a" * 32
+COMPANY = "8m0atd"
+
+
+# ---------------------------------------------------------------------------
+# Shared helpers (mirror tests/test_media_downloader.py)
+# ---------------------------------------------------------------------------
+
+
+def _primed_client() -> YunkeClient:
+    """A YunkeClient already in a logged-in state (login flow skipped)."""
+    return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
+
+
+async def _make_engine():
+    """Create an in-memory async SQLite engine with all tables created."""
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    return engine
+
+
+# ---------------------------------------------------------------------------
+# download_call_recordings
+# ---------------------------------------------------------------------------
+
+
+@respx.mock
+async def test_download_call_recordings_localizes_and_links(tmp_path):
+    """One call with an OSS mp3 record_file: downloaded, stored, linked.
+
+    Returns ``(1, 1)``; the call's ``record_media_hash`` is the sha256 of the
+    fetched bytes; a ``Media`` row exists with ``media_type='audio'``,
+    ``mime_type='audio/mpeg'``, ``original_url == OSS_MP3``; the file exists
+    under ``tmp_path`` at the sha256-content-addressed ``.mp3`` path.
+    """
+    store = ObjectStore(tmp_path)
+    engine = await _make_engine()
+    SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+
+    blob = b"ID3\x04\x00\x00\x00\x00\x00\x00mp3-payload"
+    respx.get(OSS_MP3).mock(return_value=httpx.Response(200, content=blob))
+
+    client = _primed_client()
+    try:
+        async with SessionLocal() as session:
+            session.add(
+                PhoneCall(
+                    call_id="c1", channel="voice_call", record_file=OSS_MP3
+                )
+            )
+            await session.commit()
+
+        async with SessionLocal() as session:
+            attempted, localized = await download_call_recordings(
+                client, store, session
+            )
+            await session.commit()
+
+        assert (attempted, localized) == (1, 1)
+
+        # File landed on disk at the sha256-content-addressed path with .mp3.
+        expected_sha = hashlib.sha256(blob).hexdigest()
+        expected_path = tmp_path / expected_sha[:2] / f"{expected_sha}.mp3"
+        assert expected_path.is_file()
+        assert expected_path.read_bytes() == blob
+
+        async with SessionLocal() as session:
+            call = (
+                await session.execute(
+                    sa.select(PhoneCall).where(PhoneCall.call_id == "c1")
+                )
+            ).scalar_one()
+            assert call.record_media_hash == expected_sha
+
+            media = (
+                await session.execute(
+                    sa.select(Media).where(Media.media_hash == expected_sha)
+                )
+            ).scalar_one()
+            assert media.media_type == "audio"
+            assert media.mime_type == "audio/mpeg"
+            assert media.original_url == OSS_MP3
+            assert media.local_path == str(expected_path)
+            assert media.size_bytes == len(blob)
+    finally:
+        await client.aclose()
+        await engine.dispose()
+
+
+@respx.mock
+async def test_idempotent_skip(tmp_path):
+    """A call already carrying ``record_media_hash`` is excluded -> attempted 0.
+
+    The WHERE clause filters ``record_media_hash IS NULL``, so a previously
+    localized call is never re-fetched.
+    """
+    store = ObjectStore(tmp_path)
+    engine = await _make_engine()
+    SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+
+    client = _primed_client()
+    try:
+        async with SessionLocal() as session:
+            session.add(
+                PhoneCall(
+                    call_id="c2",
+                    channel="voice_call",
+                    record_file=OSS_MP3,
+                    record_media_hash="already",
+                )
+            )
+            await session.commit()
+
+        async with SessionLocal() as session:
+            attempted, localized = await download_call_recordings(
+                client, store, session
+            )
+            await session.commit()
+
+        assert (attempted, localized) == (0, 0)
+        # No network was touched: the candidate set was empty.
+        assert respx.calls.call_count == 0
+    finally:
+        await client.aclose()
+        await engine.dispose()
+
+
+@respx.mock
+async def test_resilient_on_failure_and_no_infinite_loop(tmp_path):
+    """A persistent 500: no exception, loop TERMINATES, ``(1, 0)`` returned.
+
+    This is the key guard for the per-run ``tried`` set: without it, the
+    failing row would stay ``record_media_hash IS NULL`` and be re-selected
+    by every subsequent batch forever, hanging the run. The ``tried`` set
+    excludes the attempted call_id from later batches, so the next SELECT
+    returns an empty page and the loop breaks.
+    """
+    store = ObjectStore(tmp_path)
+    engine = await _make_engine()
+    SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+
+    respx.get(OSS_MP3).mock(return_value=httpx.Response(500))
+
+    client = _primed_client()
+    try:
+        async with SessionLocal() as session:
+            session.add(
+                PhoneCall(
+                    call_id="c3", channel="voice_call", record_file=OSS_MP3
+                )
+            )
+            await session.commit()
+
+        async with SessionLocal() as session:
+            attempted, localized = await download_call_recordings(
+                client, store, session
+            )
+            await session.commit()
+
+        assert (attempted, localized) == (1, 0)
+
+        # The failing call is left un-localized (retried on the NEXT run).
+        async with SessionLocal() as session:
+            call = (
+                await session.execute(
+                    sa.select(PhoneCall).where(PhoneCall.call_id == "c3")
+                )
+            ).scalar_one()
+            assert call.record_media_hash is None
+    finally:
+        await client.aclose()
+        await engine.dispose()