feat(scraper): moments 实体(浏览器fetch getFriendsterInfo,秒→ms,hashNext翻页,接入full_sync+4h调度+图片media)
- scrape_via_page:_normalize_moments(createTime秒×1000,缺id丢弃)+ _iter_moments_pages 翻页内核(fetch回调注入,sleep可注入,hashNext假/ 信封异常/max_pages 上限即止)+ scrape_moments(逐员工遍历wechat_account, form小驼峰,首页code≠10000告警cookie失效,换页重试惯例)+CLI注册moments - media:download_moments_images(image_urls JSON数组展开去重,MomentsPost无 回链列→用Media.original_url做已下载标记,分片IN防SQLite变量上限,批间commit) 接入scrape_media第4源 - full_sync_all.sh:9c步moments(media前,同轮图片即可本地化)+汇总加moments计数 - scheduler:4h messages job 末尾追加 moments --max-pages 3 浅扫(全量翻页 由每日full承担);测试断言7条命令序列 - tests/test_scrape_moments.py:归一化(秒→ms/空安全/坏createTime)+翻页内核 (两页累积停止/hashNext假单页/max_pages防死循环/丢id行/坏信封)+图片本地化 (一帖两图/幂等跳过/非http跳过+500容错);全套440测试通过 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
@@ -41,6 +41,8 @@ echo "$(date) --- 9/10 WA messages ---"
|
||||
$PY scripts/scrape_via_page.py --entity wa-messages --max-conversations 99999
|
||||
echo "$(date) --- 9b/10 WA group messages ---"
|
||||
$PY scripts/scrape_via_page.py --entity wa-group-messages --max-conversations 99999
|
||||
echo "$(date) --- 9c/10 Moments (员工朋友圈) ---"
|
||||
$PY scripts/scrape_via_page.py --entity moments
|
||||
echo "$(date) --- 10/10 Media download (audio+images) ---"
|
||||
$PY scripts/scrape_via_page.py --entity media
|
||||
|
||||
@@ -65,6 +67,7 @@ for label, sql in [
|
||||
('whatsapp messages', \"SELECT COUNT(*) FROM message WHERE channel='whatsapp'\"),
|
||||
('voice calls', \"SELECT COUNT(*) FROM phone_call WHERE channel IN ('voice_call','wa_voice_call')\"),
|
||||
('voice clips', 'SELECT COUNT(*) FROM voice_clip'),
|
||||
('moments posts', 'SELECT COUNT(*) FROM moments_post'),
|
||||
('media files', 'SELECT COUNT(*) FROM media'),
|
||||
]:
|
||||
print(f' {label:22s} {c.execute(sql).fetchone()[0]:>6}')
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity voice-calls
|
||||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity voice-clips
|
||||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity wa-messages
|
||||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity moments
|
||||
DB_URL="sqlite+aiosqlite:///./smoke.db" .venv/Scripts/python.exe scripts/scrape_via_page.py --entity media
|
||||
|
||||
Cookie 从 .env 的 YK_COOKIE 读取。
|
||||
@@ -27,6 +28,7 @@ import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger("scrape_via_page")
|
||||
@@ -44,6 +46,7 @@ INTERCEPT_PATHS = {
|
||||
"wa_friend": "/qqweb/WhatsAppUser/businessFriendList", # WA friends + groups
|
||||
"wa_user": "/qqweb/WhatsAppUser/businessSalesList", # WA accounts
|
||||
"msg_chats": "/wechatweb/msg/chats", # chat messages
|
||||
"friendster": "/wechatweb/wechatFriendster/getFriendsterInfo", # 员工朋友圈
|
||||
}
|
||||
|
||||
|
||||
@@ -1377,6 +1380,153 @@ async def scrape_wa_groups(cookie_str: str, max_pages: int = 200):
|
||||
return total_fetched, total_upserted
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 员工朋友圈 Moments (browser fetch, per employee)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _normalize_moments(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"""云客朋友圈行(getFriendsterInfo data.data[]) → moments_post 记录。
|
||||
|
||||
真机抓包(2026-08-25):行的 ``createTime`` 是**秒**字符串(与消息接口的
|
||||
ms 不同),统一 ×1000 成 ms epoch;缺 id 行返回 id=None 由调用方丢弃。
|
||||
"""
|
||||
ct = record.get("createTime")
|
||||
try:
|
||||
ct_ms = int(ct) * 1000 if ct is not None else None
|
||||
except (TypeError, ValueError):
|
||||
ct_ms = None
|
||||
return {
|
||||
"id": record.get("id"),
|
||||
"wechat_id": record.get("wechatId"),
|
||||
"sns_id": record.get("snsId"),
|
||||
"author_name": record.get("friendsterName"),
|
||||
"author_wxid": record.get("friendsterWXId"),
|
||||
"author_avatar": record.get("friendsterWXIdImg"),
|
||||
"content": record.get("content"),
|
||||
"type_raw": str(record.get("type")) if record.get("type") is not None else None,
|
||||
"create_time": ct_ms,
|
||||
"praise_json": record.get("praise") or [],
|
||||
"comment_json": record.get("comment") or [],
|
||||
"image_urls": record.get("imgUrlsList") or [],
|
||||
"raw_json": record,
|
||||
}
|
||||
|
||||
|
||||
async def _iter_moments_pages(
|
||||
fetch,
|
||||
wechat_id: str,
|
||||
page_size: int = 100,
|
||||
max_pages: int = 50,
|
||||
sleep=None,
|
||||
):
|
||||
"""getFriendsterInfo 翻页内核(fetch 回调注入,测试不启浏览器)。
|
||||
|
||||
``fetch(body) -> 响应 dict``;body 由本内核按真机抓包构造(form 小驼峰:
|
||||
``startTime``/``endTime`` 空串、``pageNum`` 从 1、``pageSize``、
|
||||
``wechatId``)。逐页 ``data.data[]`` → :func:`_normalize_moments`(丢
|
||||
id=None 行)累积;``data.hashNext`` 假 / 信封异常 / 达 ``max_pages`` 页
|
||||
上限即止(4h 调度浅扫 max-pages 3 靠此上限)。页间 sleep 1s。
|
||||
"""
|
||||
if sleep is None:
|
||||
sleep = asyncio.sleep
|
||||
records: list[dict] = []
|
||||
page_num = 1
|
||||
while page_num <= max_pages:
|
||||
body = {
|
||||
"startTime": "",
|
||||
"endTime": "",
|
||||
"pageNum": page_num,
|
||||
"pageSize": page_size,
|
||||
"wechatId": wechat_id,
|
||||
}
|
||||
resp = await fetch(body)
|
||||
data = resp.get("data") if isinstance(resp, dict) else None
|
||||
rows = data.get("data") if isinstance(data, dict) else None
|
||||
if not isinstance(rows, list):
|
||||
break
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
rec = _normalize_moments(row)
|
||||
if rec.get("id") is not None:
|
||||
records.append(rec)
|
||||
if not data.get("hashNext"):
|
||||
break
|
||||
page_num += 1
|
||||
await sleep(1)
|
||||
return records
|
||||
|
||||
|
||||
async def scrape_moments(cookie_str: str, max_pages: int = 50, page_size: int = 100):
|
||||
"""员工朋友圈:浏览器直发 getFriendsterInfo(form-encoded),逐员工翻页。
|
||||
|
||||
结构镜像 scrape_wa_groups;差异在**逐员工**遍历(端点按 wechatId 过滤,
|
||||
微信渠道独有实体,wechat_account 表全量)。翻页开关是 ``data.hashNext``
|
||||
(不是短页判断);批量 upsert 走 :func:`upsert_moments_posts`(收整列表)。
|
||||
返回总计 dict。
|
||||
"""
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
from app.db.base import async_engine, async_session_maker
|
||||
from app.storage.repos import upsert_moments_posts
|
||||
|
||||
async with async_engine.connect() as conn:
|
||||
result = await conn.execute(sa_text(
|
||||
"SELECT wx_id FROM wechat_account WHERE wx_id IS NOT NULL AND wx_id != ''"
|
||||
))
|
||||
employees = [row[0] for row in result.fetchall()]
|
||||
|
||||
total_fetched = 0
|
||||
total_upserted = 0
|
||||
async with PageScraper(cookie_str) as scraper:
|
||||
page = await _open_fetch_page(scraper)
|
||||
for i, wx_id in enumerate(employees):
|
||||
records: list[dict] = []
|
||||
|
||||
async def _fetch(body: dict):
|
||||
resp = await _browser_fetch(
|
||||
page, INTERCEPT_PATHS["friendster"], form_body=body
|
||||
)
|
||||
# cookie 失效告警惯例(同 _fetch_pages 首页 code 校验):
|
||||
# 健康 list 端点 code=10000
|
||||
if body["pageNum"] == 1 and isinstance(resp, dict):
|
||||
code = resp.get("code")
|
||||
if code is not None and str(code) not in ("10000",):
|
||||
logger.warning(
|
||||
f"⚠️ {INTERCEPT_PATHS['friendster']} code={code} "
|
||||
f"({resp.get('message')}) — cookie 可能已过期/失效,"
|
||||
"请从浏览器重新复制 YK_COOKIE 到 .env!"
|
||||
)
|
||||
return resp
|
||||
|
||||
try:
|
||||
records = await _iter_moments_pages(
|
||||
_fetch, wx_id, page_size=page_size, max_pages=max_pages
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f" [{i+1}/{len(employees)}] {wx_id[:16]}.. failed: {e}")
|
||||
# 换页重试惯例(同 scrape_voice_clips):该员工放弃,继续下一个
|
||||
try:
|
||||
await page.close()
|
||||
except Exception:
|
||||
pass
|
||||
page = await _open_fetch_page(scraper)
|
||||
continue
|
||||
|
||||
total_fetched += len(records)
|
||||
if records:
|
||||
async with async_session_maker() as session:
|
||||
await upsert_moments_posts(session, records)
|
||||
await session.commit()
|
||||
total_upserted += len(records)
|
||||
logger.info(f"DONE moments {wx_id} +{len(records)}")
|
||||
await asyncio.sleep(2 + (i % 3))
|
||||
|
||||
logger.info(f"DONE: moments fetched={total_fetched}, upserted={total_upserted}")
|
||||
return {"fetched": total_fetched, "upserted": total_upserted}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Voice call records (navigation + interception, both WeChat + WA pages)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1841,6 +1991,7 @@ async def scrape_media(cookie_str: str, batch_size: int = 50):
|
||||
* call recordings (phone_call.record_file OSS URLs)
|
||||
* voice clip audio (voice_clip.oss_file_name OSS URLs)
|
||||
* message images/videos (message.file_url / file_th_url / file_hd_url)
|
||||
* moments images (moments_post.image_urls OSS URLs, 一帖多图)
|
||||
|
||||
These are plain OSS CDN downloads (not CRM API calls), so httpx direct
|
||||
fetching is safe. The YunkeClient is only used as the httpx transport.
|
||||
@@ -1850,6 +2001,7 @@ async def scrape_media(cookie_str: str, batch_size: int = 50):
|
||||
from app.scraper.media import (
|
||||
download_call_recordings,
|
||||
download_message_images,
|
||||
download_moments_images,
|
||||
download_voice_clip_audio,
|
||||
)
|
||||
from app.session.yunke_client import YunkeClient
|
||||
@@ -1872,6 +2024,7 @@ async def scrape_media(cookie_str: str, batch_size: int = 50):
|
||||
totals["call_recordings"] = await download_call_recordings(client, store, session, batch_size=batch_size)
|
||||
totals["voice_clip_audio"] = await download_voice_clip_audio(client, store, session, batch_size=batch_size)
|
||||
totals["message_images"] = await download_message_images(client, store, session, batch_size=batch_size)
|
||||
totals["moments_images"] = await download_moments_images(client, store, session, batch_size=batch_size)
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
@@ -1899,13 +2052,14 @@ async def main():
|
||||
"voice-clips", # 长语音 (browser fetch call/list per employee)
|
||||
"wa-messages", # WA消息 (browser fetch chatRecords per conversation)
|
||||
"wa-group-messages", # WA群消息 (chatRecords with friendUid=@g.us)
|
||||
"moments", # 员工朋友圈 (browser fetch getFriendsterInfo)
|
||||
"media", # OSS媒体下载 (录音/长语音音频/消息图片)
|
||||
],
|
||||
help="抓取实体",
|
||||
)
|
||||
parser.add_argument("--max-employees", type=int, default=100, help="最多遍历多少员工 (friends/groups/voice-clips)")
|
||||
parser.add_argument("--max-conversations", type=int, default=100, help="消息模式：最多遍历多少对话 (messages/wa-messages/group-messages)")
|
||||
parser.add_argument("--max-pages", type=int, default=200, help="列表翻页上限 (accounts/wa-accounts/wa-friends/voice-calls)")
|
||||
parser.add_argument("--max-pages", type=int, default=200, help="列表翻页上限 (accounts/wa-accounts/wa-friends/voice-calls/moments)")
|
||||
parser.add_argument("--batch-size", type=int, default=50, help="媒体下载批次大小 (media)")
|
||||
parser.add_argument("--full-history", action="store_true", help="消息类实体强制全量翻历史(默认增量水位)")
|
||||
args = parser.parse_args()
|
||||
@@ -1945,6 +2099,8 @@ async def main():
|
||||
await scrape_wa_messages(cookie, args.max_conversations, args.full_history)
|
||||
elif args.entity == "wa-group-messages":
|
||||
await scrape_wa_group_messages(cookie, args.max_conversations, args.full_history)
|
||||
elif args.entity == "moments":
|
||||
await scrape_moments(cookie, args.max_pages)
|
||||
elif args.entity == "media":
|
||||
await scrape_media(cookie, args.batch_size)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user