feat(scraper): moments 实体(浏览器fetch getFriendsterInfo,秒→ms,hashNext翻页,接入full_sync+4h调度+图片media)
- scrape_via_page:_normalize_moments(createTime秒×1000,缺id丢弃)+ _iter_moments_pages 翻页内核(fetch回调注入,sleep可注入,hashNext假/ 信封异常/max_pages 上限即止)+ scrape_moments(逐员工遍历wechat_account, form小驼峰,首页code≠10000告警cookie失效,换页重试惯例)+CLI注册moments - media:download_moments_images(image_urls JSON数组展开去重,MomentsPost无 回链列→用Media.original_url做已下载标记,分片IN防SQLite变量上限,批间commit) 接入scrape_media第4源 - full_sync_all.sh:9c步moments(media前,同轮图片即可本地化)+汇总加moments计数 - scheduler:4h messages job 末尾追加 moments --max-pages 3 浅扫(全量翻页 由每日full承担);测试断言7条命令序列 - tests/test_scrape_moments.py:归一化(秒→ms/空安全/坏createTime)+翻页内核 (两页累积停止/hashNext假单页/max_pages防死循环/丢id行/坏信封)+图片本地化 (一帖两图/幂等跳过/非http跳过+500容错);全套440测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -37,7 +37,7 @@ import httpx
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Media, Message, PhoneCall, VoiceClip
|
||||
from app.db.models import Media, Message, MomentsPost, PhoneCall, VoiceClip
|
||||
from app.session.yunke_client import YunkeClient
|
||||
from app.storage.object_store import ObjectStore
|
||||
from app.storage.repos import upsert_media
|
||||
@@ -46,6 +46,7 @@ __all__ = [
|
||||
"download_call_recordings",
|
||||
"download_message_images",
|
||||
"download_message_media",
|
||||
"download_moments_images",
|
||||
"download_url",
|
||||
"download_voice_clip_audio",
|
||||
"guess_mime_type",
|
||||
@@ -535,3 +536,88 @@ async def download_message_images(
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
|
||||
async def download_moments_images(
|
||||
client: YunkeClient,
|
||||
store: ObjectStore,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
batch_size: int = 50,
|
||||
) -> tuple[int, int]:
|
||||
"""Download 员工朋友圈 images (``moments_post.image_urls`` OSS 直链) into
|
||||
the local object store, one ``Media`` row per blob.
|
||||
|
||||
朋友圈与消息图片的关键差异(2026-08-25 接入时确认):``image_urls`` 是
|
||||
JSON **数组**(一帖多图),且 :class:`MomentsPost` 没有 ``media_hash``
|
||||
回链列 —— 已下载标记改用 ``Media.original_url``:URL 已有对应 Media 行
|
||||
即视为已本地化并跳过(不重下载)。无回链也就无法"标记"帖子行,故候选
|
||||
集一次物化(全表展开 + 去重),再按 ``original_url`` 剔除已下载的,分批
|
||||
下载(批间 commit,部分进度可保留)。
|
||||
|
||||
Returns ``(attempted, localized)``. Resilient: a per-URL download failure is
|
||||
logged and counted as attempted-but-not-localized; no exception escapes.
|
||||
Failed URLs keep no Media row → retried on the NEXT run. Identical bytes are
|
||||
stored once (:meth:`ObjectStore.put_by_hash`, sha256 dedup).
|
||||
|
||||
Known edge: two DISTINCT URLs hosting byte-identical content share one
|
||||
``media_hash`` (content-addressed PK), and the upsert refreshes
|
||||
``original_url`` to the latest — so the loser URL is re-fetched once per run
|
||||
(bounded, never grows; identical bytes at two OSS paths is rare).
|
||||
"""
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(MomentsPost.id, MomentsPost.image_urls).where(
|
||||
MomentsPost.image_urls.is_not(None)
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
# 一帖多图 → 展开去重(同图多帖只下载一次)
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for _post_id, image_urls in rows:
|
||||
for url in image_urls or []:
|
||||
if _is_http_url(url) and url not in seen:
|
||||
seen.add(url)
|
||||
urls.append(url)
|
||||
if not urls:
|
||||
return 0, 0
|
||||
|
||||
# 剔除已本地化的 URL。分片 IN —— 单次塞全部 URL 会超 SQLite 变量上限
|
||||
# (同 download_message_images 的 tried 分片惯例,2026-08-20)。
|
||||
existing: set[str] = set()
|
||||
for i in range(0, len(urls), 500):
|
||||
res = await session.execute(
|
||||
sa.select(Media.original_url).where(
|
||||
Media.original_url.in_(urls[i : i + 500])
|
||||
)
|
||||
)
|
||||
existing.update(u for (u,) in res.fetchall() if u)
|
||||
pending = [u for u in urls if u not in existing]
|
||||
|
||||
attempted = 0
|
||||
localized = 0
|
||||
for i in range(0, len(pending), batch_size):
|
||||
for url in pending[i : i + batch_size]:
|
||||
attempted += 1
|
||||
try:
|
||||
data = await download_url(client, url)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
logger.warning("moments image download failed for %s: %s", url, exc)
|
||||
continue
|
||||
ext = _ext_from_url(url)
|
||||
sha256_hex, path = await store.put_by_hash(data, ext=ext)
|
||||
await upsert_media(
|
||||
session,
|
||||
media_hash=sha256_hex,
|
||||
channel="wechat", # 朋友圈是微信独有实体(无 WA 朋友圈)
|
||||
media_type="image",
|
||||
original_url=url,
|
||||
local_path=str(path),
|
||||
size_bytes=len(data),
|
||||
mime_type=guess_mime_type(ext),
|
||||
)
|
||||
localized += 1
|
||||
await session.commit()
|
||||
return attempted, localized
|
||||
|
||||
Reference in New Issue
Block a user