feat: media downloader + serve api

M2 Task 2.6 (media downloader) + Task 2.8 (media serve API).

Downloader (app/scraper/media.py):
- download_url(client, url): fetches bytes via the YunkeClient's underlying
  httpx client (carries session cookies; respx-mockable; follows redirects).
- download_message_media(client, msg, store, session): URL-driven fetcher that
  picks the best http(s) URL (file_url -> file_hd_url -> file_th_url), dedups
  by md5 (skips the network when a Media row with local file already exists),
  falls back to sha256(bytes) when no md5, stores via ObjectStore.put_by_hash,
  and upserts a Media row. Skips type-10 system messages (non-media per
  phase0-findings §2) and type-3 opaque-id / text (no http url). Resilient:
  download errors are logged and return None (no raise).
- media_type_from_message / guess_mime_type helpers.

Repo (app/storage/repos.py): upsert_media — idempotent Core upsert into Media by
media_hash. Volatile download artifacts (local_path/size_bytes/mime_type) use
COALESCE(incoming, existing) on conflict so a re-scrape that did not re-download
does not clobber previously stored values. Exported from app.storage.

Serve API (app/api/routers/media.py): GET /api/media/{media_hash} streams the
local file via FileResponse (handles HTTP Range / 206). 404 when the row is
absent, has no local file, or the file is missing on disk. Content-type from the
row's mime_type, falling back to ext-derived guess.

Tests (+19, suite 130 -> 149, all green; ruff clean):
- test_media_downloader.py: image/video download + upsert; md5 dedup skips
  second network hit; sha256 fallback when no md5; type-3/text/system -> None;
  download failure -> None (no raise); upsert_media idempotency +
  null-local_path preservation + 3x-same-key-one-row.
- test_api_media.py: 200 + content-type + bytes; 404 unknown hash; 404 missing
  file on disk; Range -> 206 partial; ext-derived content-type when mime NULL.

Co-Authored-By: Claude <noreply@anthropic.com>

yunke-server/app/api/main.py

@@ -7,6 +7,7 @@ Run locally with::
 Includes:
 * ``GET /api/health``  -> ``{"status": "ok"}`` liveness probe.
 * ``GET /api/customers`` (see :mod:`app.api.routers.customer`).
+* ``GET /api/media/{media_hash}`` (see :mod:`app.api.routers.media`).
 
 CORS is configured permissively (wildcard origin, no credentials) so the local
 frontend can call the API during development. Auth / finer CORS are LATER tasks.
@@ -17,7 +18,7 @@ from __future__ import annotations
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
-from app.api.routers import customer
+from app.api.routers import customer, media
 
 app = FastAPI(title="yunke data mirror")
 
@@ -29,6 +30,7 @@ app.add_middleware(
 )
 
 app.include_router(customer.router)
+app.include_router(media.router)
 
 
 @app.get("/api/health")

yunke-server/app/api/routers/media.py

@@ -0,0 +1,54 @@
+"""GET /api/media/{media_hash} — stream a cached media blob (Task 2.8).
+
+Looks up the :class:`app.db.models.Media` row keyed by ``media_hash`` and
+streams the local file with the correct ``Content-Type``. HTTP Range requests
+are honored (Starlette's :class:`fastapi.responses.FileResponse` handles
+``Range`` / ``206 Partial Content`` natively). Returns ``404`` when the row is
+absent, has no local file recorded, or the underlying file is missing on disk.
+
+The endpoint reads the absolute ``local_path`` straight off the row (the
+downloader stores it absolute), so it does not need to know the
+:class:`ObjectStore` layout — only that the file exists where the row points.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import sqlalchemy as sa
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import FileResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_session
+from app.db.models import Media
+from app.scraper.media import guess_mime_type
+
+router = APIRouter(prefix="/api", tags=["media"])
+
+SessionDep = Annotated[AsyncSession, Depends(get_session)]
+
+
+@router.get("/media/{media_hash}")
+async def get_media(media_hash: str, session: SessionDep) -> FileResponse:
+    """Stream the cached media blob for ``media_hash``."""
+    row = (
+        await session.execute(
+            sa.select(Media).where(Media.media_hash == media_hash)
+        )
+    ).scalar_one_or_none()
+    if row is None or not row.local_path:
+        raise HTTPException(status_code=404, detail="media not found")
+
+    path = Path(row.local_path)
+    if not path.is_file():
+        # Row exists but the blob was removed out-of-band — treat as not-found.
+        raise HTTPException(status_code=404, detail="media file missing")
+
+    media_type = (
+        row.mime_type
+        or guess_mime_type(path.suffix)
+        or "application/octet-stream"
+    )
+    return FileResponse(path, media_type=media_type, filename=path.name)

yunke-server/app/scraper/media.py

@@ -0,0 +1,222 @@
+"""Media downloader (M2, Task 2.6).
+
+Fetches WeChat / OSS media blobs referenced by message records into the local
+:class:`app.storage.object_store.ObjectStore` and records them in the
+:class:`app.db.models.Media` table, deduplicated by the source ``md5``.
+
+Phase-0 contract (see ``docs/superpowers/specs/phase0-findings.md`` §2/§3):
+
+* WeChat type ``2`` (image / emoji, ``.webp``) and ``4`` (video, ``.mp4``) carry
+  a DIRECT http(s) URL on ``file`` / ``fileTh`` / ``fileHd`` plus ``md5`` ->
+  downloadable, deduped by ``md5``.
+* WeChat type ``3`` (legacy image) carries an OPAQUE id on ``file`` (not a URL)
+  -> cannot be resolved yet; :func:`download_message_media` returns ``None``.
+* WeChat type ``10`` (system / link / card / invite) carries a
+  ``support.weixin.qq.com`` URL that is a documentation link, NOT a media blob
+  -> explicitly skipped before any network attempt.
+
+The downloader is URL-driven and generic: any http(s) media URL works (the
+recording URLs on ``yunke-pcfile`` are the same shape), so call recordings and
+voice-call recordings can be fetched by the same path once their owning records
+are wired in. That wiring is OUT of scope here (this module only consumes
+normalized message dicts); see ``app.scraper.wechat`` for those.
+
+Resilient by design: a download failure is logged and the function returns
+``None`` rather than raising. A retry queue is a LATER milestone.
+"""
+
+from __future__ import annotations
+
+import logging
+import mimetypes
+from pathlib import PurePosixPath
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+import sqlalchemy as sa
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Media
+from app.session.yunke_client import YunkeClient
+from app.storage.object_store import ObjectStore
+from app.storage.repos import upsert_media
+
+__all__ = [
+    "download_message_media",
+    "download_url",
+    "guess_mime_type",
+    "media_type_from_message",
+]
+
+logger = logging.getLogger(__name__)
+
+# Normalized message ``type`` label -> :attr:`Media.media_type` value. Only the
+# Phase-0-confirmed image / video labels map today; voice / file codes
+# (``6`` / ``9`` / ...) remain ``"unknown"`` at the scraper layer and yield
+# ``None`` here until they are confirmed (phase0-findings.md §2).
+_MEDIA_TYPE_BY_LABEL: dict[str, str] = {
+    "image": "image",
+    "image_legacy": "image",
+    "video": "video",
+}
+
+# Extension -> MIME fallback map for the OSS / call-recording media types we
+# observe. Used when the stdlib ``mimetypes`` database does not recognize the
+# extension (e.g. ``.webp`` / ``.amr`` on some platforms) so content-type
+# derivation does not depend on the system MIME database.
+_EXT_MIME: dict[str, str] = {
+    ".webp": "image/webp",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".gif": "image/gif",
+    ".bmp": "image/bmp",
+    ".mp4": "video/mp4",
+    ".mp3": "audio/mpeg",
+    ".m4a": "audio/mp4",
+    ".wav": "audio/wav",
+    ".amr": "audio/amr",
+}
+
+
+def media_type_from_message(msg: dict[str, Any]) -> str | None:
+    """Map a normalized message dict to a :attr:`Media.media_type` label.
+
+    Returns ``"image"`` for ``image`` / ``image_legacy`` messages and ``"video"``
+    for ``video`` messages; ``None`` for everything else (text / system /
+    unknown / not-yet-confirmed voice-or-file codes).
+    """
+    return _MEDIA_TYPE_BY_LABEL.get(msg.get("type"))  # type: ignore[arg-type]
+
+
+def _is_http_url(value: Any) -> bool:
+    """True iff ``value`` is an ``http://`` / ``https://`` string."""
+    return isinstance(value, str) and value.startswith(("http://", "https://"))
+
+
+def _pick_media_url(msg: dict[str, Any]) -> str | None:
+    """Pick the best downloadable http(s) URL from a message dict.
+
+    Preference order: ``file_url`` (original) -> ``file_hd_url`` (high-def) ->
+    ``file_th_url`` (thumbnail). Returns ``None`` when no candidate is an
+    http(s) URL (covers type-``3`` opaque ids and text messages).
+    """
+    for key in ("file_url", "file_hd_url", "file_th_url"):
+        url = msg.get(key)
+        if _is_http_url(url):
+            return url  # type: ignore[return-value]
+    return None
+
+
+def _ext_from_url(url: str) -> str:
+    """Extract the lowercase extension (with leading ``.``) from a URL path.
+
+    Query / fragment are stripped first. Returns ``""`` when the path has no
+    suffix (e.g. an OSS callback URL without an extension).
+    """
+    suffix = PurePosixPath(urlparse(url).path).suffix
+    return suffix.lower()
+
+
+def guess_mime_type(ext: str) -> str | None:
+    """Guess a MIME type from a file extension.
+
+    The built-in :data:`_EXT_MIME` map is consulted first (covers ``.webp`` /
+    ``.mp4`` / ``.amr`` reliably regardless of platform), then the stdlib
+    ``mimetypes`` database. Returns ``None`` when nothing matches.
+    """
+    if not ext:
+        return None
+    key = ext.lower()
+    if key in _EXT_MIME:
+        return _EXT_MIME[key]
+    guessed, _ = mimetypes.guess_type(f"x{key}")
+    return guessed
+
+
+async def download_url(client: YunkeClient, url: str) -> bytes:
+    """Fetch the raw bytes at ``url`` via the client's underlying httpx session.
+
+    The :class:`YunkeClient` httpx client carries the session cookies (harmless
+    for the public OSS media URLs, useful for any same-origin authenticated
+    URLs) and is the transport ``respx`` mocks in the test suite, so media
+    downloads go through the same mocked transport as every other scraper
+    request. Redirects are followed (OSS / avatar URLs commonly redirect).
+    """
+    resp = await client.client.get(url, follow_redirects=True)
+    resp.raise_for_status()
+    return resp.content
+
+
+async def download_message_media(
+    client: YunkeClient,
+    msg: dict[str, Any],
+    store: ObjectStore,
+    session: AsyncSession,
+) -> str | None:
+    """Download one message's media blob and record it in the ``Media`` table.
+
+    Returns the dedup hash (the message ``md5`` when present, else the sha256 of
+    the downloaded bytes), or ``None`` when:
+
+    * the message is a type-``10`` system / link message (non-media; skipped
+      before any network attempt), or
+    * the message carries no downloadable http(s) URL (text, type-``3``
+      opaque-id images, or any message whose ``file``/``fileHd``/``fileTh`` are
+      absent or non-URL), or
+    * the download fails (the error is logged; no exception escapes).
+
+    Dedup: when the message carries an ``md5`` and a :class:`Media` row for that
+    hash already exists with a local file, the download is SKIPPED and the hash
+    is returned without touching the network.
+
+    The blob is written via :meth:`ObjectStore.put_by_hash` (sha256
+    content-addressed on disk), so identical bytes are stored once regardless of
+    how many messages reference them.
+    """
+    # Type-10 system / link messages carry a support URL that is NOT media
+    # (phase0-findings.md §2): skip them explicitly before URL selection so the
+    # downloader never fetches a documentation page as if it were a media blob.
+    if msg.get("type") == "system":
+        return None
+
+    url = _pick_media_url(msg)
+    if url is None:
+        return None
+
+    md5 = msg.get("md5")
+    if md5:
+        existing = (
+            await session.execute(
+                sa.select(Media.local_path).where(Media.media_hash == md5)
+            )
+        ).scalar_one_or_none()
+        if existing:
+            # Already downloaded (local file present); short-circuit the network.
+            return md5  # type: ignore[return-value]
+
+    try:
+        data = await download_url(client, url)
+    except (httpx.HTTPError, OSError) as exc:
+        logger.warning("media download failed for %s: %s", url, exc)
+        return None
+
+    ext = _ext_from_url(url)
+    sha256_hex, path = await store.put_by_hash(data, ext=ext)
+    media_hash: str = md5 if md5 else sha256_hex
+
+    await upsert_media(
+        session,
+        media_hash=media_hash,
+        channel=msg.get("channel") or "wechat",
+        media_type=media_type_from_message(msg),
+        original_url=msg.get("file_url"),
+        original_th_url=msg.get("file_th_url"),
+        original_hd_url=msg.get("file_hd_url"),
+        local_path=str(path),
+        size_bytes=len(data),
+        mime_type=guess_mime_type(ext),
+        raw_json=msg.get("raw_json"),
+    )
+    return media_hash

yunke-server/app/storage/__init__.py

@@ -19,6 +19,7 @@ from app.storage.object_store import ObjectStore
 from app.storage.repos import (
     upsert_customer,
     upsert_customers,
+    upsert_media,
     upsert_messages,
     upsert_staff_from_account,
     upsert_wechat_account,
@@ -28,6 +29,7 @@ __all__ = [
     "ObjectStore",
     "upsert_customer",
     "upsert_customers",
+    "upsert_media",
     "upsert_messages",
     "upsert_staff_from_account",
     "upsert_wechat_account",

yunke-server/app/storage/repos.py

@@ -58,6 +58,7 @@ from app.db.models import (
     Customer,
     Department,
     GroupMember,
+    Media,
     Message,
     PhoneCall,
     Staff,
@@ -69,6 +70,7 @@ __all__ = [
     "upsert_customer",
     "upsert_customers",
     "upsert_department",
+    "upsert_media",
     "upsert_messages",
     "upsert_staff",
     "upsert_staff_from_account",
@@ -474,3 +476,101 @@ async def upsert_voice_call(session: AsyncSession, rec: dict[str, Any]) -> None:
     per-row ``channel`` value disambiguates them.
     """
     await _upsert_one(session, PhoneCall, rec)
+
+
+# ---------------------------------------------------------------------------
+# Media upsert (M2, Task 2.6)
+# ---------------------------------------------------------------------------
+
+# Download artifacts that must NOT be clobbered with NULL when a later upsert
+# (e.g. a re-scrape that did not re-download) omits them. COALESCE'd against the
+# existing row so a genuine new value still wins.
+_MEDIA_PRESERVE_IF_NULL: frozenset[str] = frozenset(
+    {"local_path", "size_bytes", "mime_type"}
+)
+
+
+async def upsert_media(
+    session: AsyncSession,
+    *,
+    media_hash: str,
+    channel: str,
+    media_type: str | None = None,
+    original_url: str | None = None,
+    original_th_url: str | None = None,
+    original_hd_url: str | None = None,
+    local_path: str | None = None,
+    size_bytes: int | None = None,
+    mime_type: str | None = None,
+    duration: int | None = None,
+    raw_json: dict[str, Any] | None = None,
+) -> None:
+    """Idempotently upsert one :class:`Media` row (PK = ``media_hash``).
+
+    Same ``media_hash`` twice -> one row. Non-PK columns are refreshed from the
+    incoming record on conflict, EXCEPT the volatile download artifacts
+    (``local_path`` / ``size_bytes`` / ``mime_type``): those are written with
+    ``COALESCE(incoming, existing)`` so that a re-scrape which did NOT re-download
+    (and therefore passes ``None``) leaves the previously downloaded values
+    intact. A genuine new value (non-``None``) still overrides the old one.
+
+    The ``media_hash`` is the cross-message dedup key: the source ``md5`` when
+    the message carried one, else the sha256 of the downloaded bytes (see
+    :mod:`app.scraper.media`).
+    """
+    values: dict[str, Any] = {
+        "media_hash": media_hash,
+        "channel": channel,
+        "media_type": media_type,
+        "original_url": original_url,
+        "original_th_url": original_th_url,
+        "original_hd_url": original_hd_url,
+        "local_path": local_path,
+        "size_bytes": size_bytes,
+        "mime_type": mime_type,
+        "duration": duration,
+        "raw_json": raw_json,
+    }
+
+    dialect = await _dialect_name(session)
+    pk_cols = _pk_col_names(Media)
+    update_cols = [
+        c.name for c in Media.__table__.columns if c.name not in pk_cols
+    ]
+    media_cols = Media.__table__.c
+
+    def _build_set(other_cols: Any) -> dict[str, Any]:
+        """Map each updated column to its conflict-set expression.
+
+        Volatile download artifacts use ``COALESCE(incoming, existing)``; every
+        other column is refreshed directly from the incoming row.
+        """
+        set_: dict[str, Any] = {}
+        for name in update_cols:
+            if name in _MEDIA_PRESERVE_IF_NULL:
+                set_[name] = sa.func.coalesce(other_cols[name], media_cols[name])
+            else:
+                set_[name] = other_cols[name]
+        return set_
+
+    if dialect == "sqlite":
+        from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+
+        bare = sqlite_insert(Media)
+        set_ = _build_set(bare.excluded)
+        stmt = bare.values(**values).on_conflict_do_update(
+            index_elements=pk_cols, set_=set_
+        )
+    elif dialect == "mysql":
+        from sqlalchemy.dialects.mysql import insert as mysql_insert
+
+        bare = mysql_insert(Media)
+        set_ = _build_set(bare.inserted)
+        stmt = bare.values(**values).on_duplicate_key_update(**set_)
+    else:
+        raise ValueError(
+            f"Unsupported dialect for upsert_media: {dialect!r}. "
+            "Only 'sqlite' (tests) and 'mysql' (prod) are supported."
+        )
+
+    await session.execute(stmt)