feat: media downloader + serve api
M2 Task 2.6 (media downloader) + Task 2.8 (media serve API).
Downloader (app/scraper/media.py):
- download_url(client, url): fetches bytes via the YunkeClient's underlying
httpx client (carries session cookies; respx-mockable; follows redirects).
- download_message_media(client, msg, store, session): URL-driven fetcher that
picks the best http(s) URL (file_url -> file_hd_url -> file_th_url), dedups
by md5 (skips the network when a Media row with local file already exists),
falls back to sha256(bytes) when no md5, stores via ObjectStore.put_by_hash,
and upserts a Media row. Skips type-10 system messages (non-media per
phase0-findings §2) and type-3 opaque-id / text (no http url). Resilient:
download errors are logged and return None (no raise).
- media_type_from_message / guess_mime_type helpers.
Repo (app/storage/repos.py): upsert_media — idempotent Core upsert into Media by
media_hash. Volatile download artifacts (local_path/size_bytes/mime_type) use
COALESCE(incoming, existing) on conflict so a re-scrape that did not re-download
does not clobber previously stored values. Exported from app.storage.
Serve API (app/api/routers/media.py): GET /api/media/{media_hash} streams the
local file via FileResponse (handles HTTP Range / 206). 404 when the row is
absent, has no local file, or the file is missing on disk. Content-type from the
row's mime_type, falling back to ext-derived guess.
Tests (+19, suite 130 -> 149, all green; ruff clean):
- test_media_downloader.py: image/video download + upsert; md5 dedup skips
second network hit; sha256 fallback when no md5; type-3/text/system -> None;
download failure -> None (no raise); upsert_media idempotency +
null-local_path preservation + 3x-same-key-one-row.
- test_api_media.py: 200 + content-type + bytes; 404 unknown hash; 404 missing
file on disk; Range -> 206 partial; ext-derived content-type when mime NULL.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -58,6 +58,7 @@ from app.db.models import (
|
||||
Customer,
|
||||
Department,
|
||||
GroupMember,
|
||||
Media,
|
||||
Message,
|
||||
PhoneCall,
|
||||
Staff,
|
||||
@@ -69,6 +70,7 @@ __all__ = [
|
||||
"upsert_customer",
|
||||
"upsert_customers",
|
||||
"upsert_department",
|
||||
"upsert_media",
|
||||
"upsert_messages",
|
||||
"upsert_staff",
|
||||
"upsert_staff_from_account",
|
||||
@@ -474,3 +476,101 @@ async def upsert_voice_call(session: AsyncSession, rec: dict[str, Any]) -> None:
|
||||
per-row ``channel`` value disambiguates them.
|
||||
"""
|
||||
await _upsert_one(session, PhoneCall, rec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Media upsert (M2, Task 2.6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Download artifacts that must NOT be clobbered with NULL when a later upsert
|
||||
# (e.g. a re-scrape that did not re-download) omits them. COALESCE'd against the
|
||||
# existing row so a genuine new value still wins.
|
||||
_MEDIA_PRESERVE_IF_NULL: frozenset[str] = frozenset(
|
||||
{"local_path", "size_bytes", "mime_type"}
|
||||
)
|
||||
|
||||
|
||||
async def upsert_media(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
media_hash: str,
|
||||
channel: str,
|
||||
media_type: str | None = None,
|
||||
original_url: str | None = None,
|
||||
original_th_url: str | None = None,
|
||||
original_hd_url: str | None = None,
|
||||
local_path: str | None = None,
|
||||
size_bytes: int | None = None,
|
||||
mime_type: str | None = None,
|
||||
duration: int | None = None,
|
||||
raw_json: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Idempotently upsert one :class:`Media` row (PK = ``media_hash``).
|
||||
|
||||
Same ``media_hash`` twice -> one row. Non-PK columns are refreshed from the
|
||||
incoming record on conflict, EXCEPT the volatile download artifacts
|
||||
(``local_path`` / ``size_bytes`` / ``mime_type``): those are written with
|
||||
``COALESCE(incoming, existing)`` so that a re-scrape which did NOT re-download
|
||||
(and therefore passes ``None``) leaves the previously downloaded values
|
||||
intact. A genuine new value (non-``None``) still overrides the old one.
|
||||
|
||||
The ``media_hash`` is the cross-message dedup key: the source ``md5`` when
|
||||
the message carried one, else the sha256 of the downloaded bytes (see
|
||||
:mod:`app.scraper.media`).
|
||||
"""
|
||||
values: dict[str, Any] = {
|
||||
"media_hash": media_hash,
|
||||
"channel": channel,
|
||||
"media_type": media_type,
|
||||
"original_url": original_url,
|
||||
"original_th_url": original_th_url,
|
||||
"original_hd_url": original_hd_url,
|
||||
"local_path": local_path,
|
||||
"size_bytes": size_bytes,
|
||||
"mime_type": mime_type,
|
||||
"duration": duration,
|
||||
"raw_json": raw_json,
|
||||
}
|
||||
|
||||
dialect = await _dialect_name(session)
|
||||
pk_cols = _pk_col_names(Media)
|
||||
update_cols = [
|
||||
c.name for c in Media.__table__.columns if c.name not in pk_cols
|
||||
]
|
||||
media_cols = Media.__table__.c
|
||||
|
||||
def _build_set(other_cols: Any) -> dict[str, Any]:
|
||||
"""Map each updated column to its conflict-set expression.
|
||||
|
||||
Volatile download artifacts use ``COALESCE(incoming, existing)``; every
|
||||
other column is refreshed directly from the incoming row.
|
||||
"""
|
||||
set_: dict[str, Any] = {}
|
||||
for name in update_cols:
|
||||
if name in _MEDIA_PRESERVE_IF_NULL:
|
||||
set_[name] = sa.func.coalesce(other_cols[name], media_cols[name])
|
||||
else:
|
||||
set_[name] = other_cols[name]
|
||||
return set_
|
||||
|
||||
if dialect == "sqlite":
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
|
||||
bare = sqlite_insert(Media)
|
||||
set_ = _build_set(bare.excluded)
|
||||
stmt = bare.values(**values).on_conflict_do_update(
|
||||
index_elements=pk_cols, set_=set_
|
||||
)
|
||||
elif dialect == "mysql":
|
||||
from sqlalchemy.dialects.mysql import insert as mysql_insert
|
||||
|
||||
bare = mysql_insert(Media)
|
||||
set_ = _build_set(bare.inserted)
|
||||
stmt = bare.values(**values).on_duplicate_key_update(**set_)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported dialect for upsert_media: {dialect!r}. "
|
||||
"Only 'sqlite' (tests) and 'mysql' (prod) are supported."
|
||||
)
|
||||
|
||||
await session.execute(stmt)
|
||||
|
||||
Reference in New Issue
Block a user