xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(media): image_legacy渲染+下载覆盖+WA raw_json URL回退

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-13 14:02:42 +08:00
parent 9e4a5ee536
commit 2154d050e5
3 changed files with 164 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/app/scraper/media.py
+20 -11
View File
@@ -394,11 +394,13 @@ def _pick_message_image_url(
) -> str | None:
"""Pick the best downloadable http(s) URL for a message media row.
For ``image`` messages, prefer the thumbnail (``file_th_url``): thumbnails
are small, fast to fetch, and exactly what the chat timeline renders, so
localizing the thumbnail is sufficient for display and easy on disk + bandwidth.
Falls back to ``file_url`` then ``file_hd_url`` when no http(s) thumbnail is
available.
For ``image`` / ``image_legacy`` messages, prefer the thumbnail
(``file_th_url``): thumbnails are small, fast to fetch, and exactly what the
chat timeline renders, so localizing the thumbnail is sufficient for display
and easy on disk + bandwidth. Falls back to ``file_url`` then ``file_hd_url``
when no http(s) thumbnail is available. ``image_legacy`` (smoke.db 26 rows
with valid ``wx.oss.yunkecn.com`` URLs) is treated exactly like ``image``
here so the legacy WeChat type-3-via-OSS rows are localized.
For every other media type (``video`` / ``file``), prefer ``file_url`` (the
original), then ``file_hd_url``, then ``file_th_url``.
@@ -406,7 +408,7 @@ def _pick_message_image_url(
Returns ``None`` when no candidate is an http(s) URL (e.g. type-3 opaque ids
or relative filenames). Only ``http://`` / ``https://`` URLs are downloadable.
"""
if type_ == "image":
if type_ in ("image", "image_legacy"):
order = (file_th_url, file_url, file_hd_url)
else:
order = (file_url, file_hd_url, file_th_url)
@@ -428,12 +430,15 @@ async def download_message_images(
Iterates ``message`` rows that carry an http(s) URL on any of
``file_url`` / ``file_th_url`` / ``file_hd_url``, have ``type`` in
('image', 'video', 'file'), are on the wechat / whatsapp channel, and do
NOT yet carry a ``media_hash``. For each: pick the best available URL
(image: thumbnail-first; other: original-first — see
('image', 'image_legacy', 'video', 'file'), are on the wechat / whatsapp
channel, and do NOT yet carry a ``media_hash``. For each: pick the best
available URL (image: thumbnail-first; other: original-first — see
:func:`_pick_message_image_url`), fetch bytes (:func:`download_url`),
store content-addressed (:meth:`ObjectStore.put_by_hash`, sha256 dedup),
upsert a ``Media`` row, and write ``media_hash`` back onto the message.
``image_legacy`` rows are stored as ``media_type='image'`` (normalized via
:func:`media_type_from_message`) so the legacy WeChat type-3-via-OSS rows
(smoke.db 26 rows with valid ``wx.oss.yunkecn.com`` URLs) are covered.
Returns ``(attempted, localized)``. Resilient: a per-row download/store
failure is logged and counted as attempted-but-not-localized; no exception
@@ -462,7 +467,7 @@ async def download_message_images(
)
.where(Message.media_hash.is_(None))
.where(Message.channel.in_(("wechat", "whatsapp")))
.where(Message.type.in_(("image", "video", "file")))
.where(Message.type.in_(("image", "image_legacy", "video", "file")))
.where(
sa.or_(
Message.file_url.like("http%"),
@@ -500,11 +505,15 @@ async def download_message_images(
continue
ext = _ext_from_url(url)
sha256_hex, path = await store.put_by_hash(data, ext=ext)
# Normalize ``image_legacy`` to ``image`` on the stored Media row so
# the legacy WeChat type-3-via-OSS rows land in the same bucket as
# type-2 images (matches ``media_type_from_message``).
media_type = "image" if type_ == "image_legacy" else type_
await upsert_media(
session,
media_hash=sha256_hex,
channel=channel or "wechat",
media_type=type_,
media_type=media_type,
original_url=file_url,
original_th_url=file_th_url,
original_hd_url=file_hd_url,
yunke-server/tests/test_message_images_downloader.py
+77 -3
View File
@@ -17,7 +17,10 @@ Mirrors ``test_recording_downloader.py``:
set excludes the failing msg_id from later batches) and returns
``(attempted=1, localized=0)`` with no exception escaping.
* Filter: a text message with an http URL (e.g. a documentation link) is
NOT selected (only ``type IN ('image','video','file')`` rows are eligible).
NOT selected (only ``type IN ('image','image_legacy','video','file')`` rows
are eligible).
* ``image_legacy`` coverage: legacy WeChat type-3 rows whose ``file_url``
carries a valid OSS URL ARE selected and stored as ``media_type='image'``.
"""
from __future__ import annotations
@@ -340,8 +343,9 @@ async def test_resilient_on_failure_and_no_infinite_loop(tmp_path):
async def test_text_message_with_url_is_not_selected(tmp_path):
"""A text message whose text happens to be a URL is NOT selected.
The candidate filter requires ``type IN ('image','video','file')``; a
text message is skipped even if its columns somehow carry an http URL.
The candidate filter requires ``type IN ('image','image_legacy','video',
'file')``; a text message is skipped even if its columns somehow carry an
http URL.
"""
store = ObjectStore(tmp_path)
engine = await _make_engine()
@@ -373,3 +377,73 @@ async def test_text_message_with_url_is_not_selected(tmp_path):
finally:
await client.aclose()
await engine.dispose()
@respx.mock
async def test_image_legacy_is_selected_and_normalized_to_image(tmp_path):
"""``type='image_legacy'`` rows ARE selected (smoke.db 26 rows).
The candidate filter includes ``image_legacy`` so the legacy WeChat type-3
rows whose ``file_url`` carries a valid ``wx.oss.yunkecn.com`` OSS URL get
localized. The thumbnail-first preference applies (same as ``image``), and
the stored ``Media.media_type`` is NORMALIZED to ``'image'`` so the legacy
bucket lands in the same bin as type-2 images.
"""
store = ObjectStore(tmp_path)
engine = await _make_engine()
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
thumb_blob = b"\xff\xd8\xff\xe0legacy-thumb-jpg"
thumb_route = respx.get(OSS_THUMB).mock(
return_value=httpx.Response(200, content=thumb_blob)
)
full_route = respx.get(OSS_FULL).mock(return_value=httpx.Response(200, content=b"full"))
client = _primed_client()
try:
async with SessionLocal() as session:
session.add(
Message(
msg_id="m-legacy",
channel="wechat",
type="image_legacy",
type_raw=3,
file_url=OSS_FULL,
file_th_url=OSS_THUMB,
file_hd_url=OSS_HD,
)
)
await session.commit()
async with SessionLocal() as session:
attempted, localized = await download_message_images(
client, store, session
)
await session.commit()
assert (attempted, localized) == (1, 1)
# Thumbnail-first preference applies to image_legacy exactly as for image.
assert thumb_route.called
assert not full_route.called
expected_sha = hashlib.sha256(thumb_blob).hexdigest()
async with SessionLocal() as session:
msg = (
await session.execute(
sa.select(Message).where(Message.msg_id == "m-legacy")
)
).scalar_one()
assert msg.media_hash == expected_sha
media = (
await session.execute(
sa.select(Media).where(Media.media_hash == expected_sha)
)
).scalar_one()
# Normalized: image_legacy -> image on the stored Media row.
assert media.media_type == "image"
assert media.mime_type == "image/jpeg"
assert media.original_url == OSS_FULL
finally:
await client.aclose()
await engine.dispose()