xinyouhe/yunke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api): 长语音 voice_clip 后端(model+migration+scraper+sync+endpoint)

Browse Source 
- VoiceClip model (BigInteger PK voice_clip_id, structured cols + raw_json)
- Alembic migration a1b2c3d4e5f6 (chains from 8c4f1d2a9e6b)
- scrape_voice_clips(client, user_id, *, channel, page_size) per-employee scraper
  over POST /wechatweb/call/list {pageNum, pageSize, userId}; channel stamped
  locally (wechat/whatsapp), startTime/endTime ms-epoch string -> int
- upsert_voice_clip repo + start_time/end_time/mp3_duration int coercion
- sync_voice_clips orchestrator: iterates distinct user_id across
  WechatAccount + WhatsappAccount, stamps channel per account table
- scheduler ENTITY_FUNCS['voice_clips'] + added to daily 'all' job
- GET /api/voice-clips (user_id exact beats user_real_name fuzzy; channel;
  company_code; start/end window on end_time; ordered end_time desc)
- VoiceClipOut schema + router registered in main.py
- tests: scraper (5) + API (11) = 16 new; full suite 370 green

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
浅念
2026-08-10 18:12:28 +08:00
parent a0db4535f1
commit d9060b6920
12 changed files with 1271 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yunke-server/tests/cassettes/voiceClip_fixture.py
+152
View File
@@ -0,0 +1,152 @@
"""Fixture payloads for ``POST /wechatweb/call/list`` (长语音 voice_clip).
These records mirror the real field names confirmed by the 2026-08-07
Playwright capture (see spec §2): ``id`` (bigint PK) / ``userId`` /
``userRealName`` / ``wxId`` (WA=null) / ``talker`` (WA=null) / ``status`` /
``isSend`` / ``companyCode`` (企业串码) / ``startTime`` / ``endTime`` (both
ms-epoch STRINGS) / ``ossFileName`` (recording URL) / ``mp3Duration`` (秒) /
``durationFormat`` (HH:MM:SS).
The envelope is the SAME shape as ``queryWeChatVoiceList`` (rows under
``data.data``, declared total on ``data.total``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "pageNum":.., "pageSize":..}}
The cassette ships four records across two pages so a test can drive the
page-number paginator across the empty-page stop signal:
* page 1 -> 2 of the 4 fixture records, ``total=4``
* page 2 -> the other 2 records, ``total=4``
* (page 3 -> empty ``data`` array, served inline by the test)
Records deliberately exercise the documented mapping edges:
* ``vc_66001`` — wechat clip with start/end as ms-epoch strings, non-null
``ossFileName``, ``isSend=null`` (the WA/wechat long-clip payload carries
``isSend`` as null in the capture).
* ``vc_66002`` — clip with ``ossFileName=null`` (no recording URL).
* ``vc_66003`` — clip with ``mp3Duration=0`` + ``durationFormat="00:00:00"``.
* ``vc_66004`` — clip with a very long ``ossFileName`` URL.
"""
# Module name mirrors the upstream feature (长语音 voice_clip list).
# ruff: noqa: N999
from __future__ import annotations
from typing import Any
# ---------------------------------------------------------------------------
# Capture-shaped voice-clip records (4 total across 2 pages).
# ---------------------------------------------------------------------------
VOICE_CLIP_RECORDS: list[dict[str, Any]] = [
{
# Standard wechat clip: start/end as ms-epoch strings, recording URL.
"id": 66001,
"userId": "832EC001_wechat_staff",
"userRealName": "聂玉婷",
"wxId": "wxid_staff_alice01",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072733378", # ms-epoch string -> int
"endTime": "1786072926741",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/imei001/1786072733378_1786072926741.mp3"
),
"mp3Duration": 193,
"durationFormat": "00:03:13",
},
{
# Clip with no recording URL.
"id": 66002,
"userId": "832EC001_wechat_staff",
"userRealName": "聂玉婷",
"wxId": "wxid_staff_alice01",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072926742",
"endTime": "1786072926742",
"ossFileName": None,
"mp3Duration": 0,
"durationFormat": "00:00:00",
},
{
# Clip with zero duration.
"id": 66003,
"userId": "832EC002_wechat_staff",
"userRealName": "李四",
"wxId": "wxid_staff_bob02",
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786000000000",
"endTime": "1786000000000",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/imei002/1786000000000_1786000000000.mp3"
),
"mp3Duration": 0,
"durationFormat": "00:00:00",
},
{
# WA employee (userId range F33DA..., wxId=null) with a long URL.
"id": 66004,
"userId": "F33DA003_wa_staff",
"userRealName": "王五",
"wxId": None,
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786100000000",
"endTime": "1786100123000",
"ossFileName": (
"https://yunke-pcfile.oss-cn-beijing.aliyuncs.com/wechat-call/"
"8m0atd/wa/F33DA003/1786100000000_1786100123000.mp3"
),
"mp3Duration": 123,
"durationFormat": "00:02:03",
},
]
# Default page size for the fixture (and the scraper). total=4 means the
# fixture fills exactly 2 pages of size 2 plus a terminating empty page.
TOTAL = len(VOICE_CLIP_RECORDS)
DEFAULT_PAGE_SIZE = 2
def make_voice_clip_list_response(
records: list[dict[str, Any]],
*,
total: int,
page_num: int = 1,
page_size: int = DEFAULT_PAGE_SIZE,
trace_id: str | None = None,
) -> dict[str, Any]:
"""Build a capture-shaped ``call/list`` response envelope.
Shape (rows + pagination fields directly under ``data``)::
{"message":"调用成功","code":"10000","success":true,"traceId":"...",
"data":{"data":[...], "total":.., "pageNum":.., "pageSize":..}}
"""
return {
"message": "调用成功",
"code": "10000",
"success": True,
"traceId": trace_id or f"trace-{page_num}",
"data": {
"data": list(records),
"total": total,
"pageNum": page_num,
"pageSize": page_size,
},
}
yunke-server/tests/test_api_voice_clips.py
+239
View File
@@ -0,0 +1,239 @@
"""Tests for GET /api/voice-clips — 长语音 (voice_clip) list.
Seeds 3 clips (2 wechat for user u1 at end_time 1000/3000, 1 whatsapp for u2
at end_time 2000) and asserts filters + ordering (``end_time desc``) +
pagination + the ``user_id``-beats-``user_real_name`` precedence rule.
"""
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import VoiceClip
async def _seed(session: AsyncSession) -> None:
session.add_all(
[
VoiceClip(
voice_clip_id=1,
user_id="u1",
channel="wechat",
user_real_name="Alice",
company_code="8m0atd",
start_time=900,
end_time=1000,
oss_file_name="https://oss/clip1.mp3",
mp3_duration=100,
duration_format="00:01:40",
),
VoiceClip(
voice_clip_id=2,
user_id="u2",
channel="whatsapp",
user_real_name="Bob",
company_code="8m0atd",
start_time=1900,
end_time=2000,
oss_file_name="https://oss/clip2.mp3",
mp3_duration=60,
duration_format="00:01:00",
),
VoiceClip(
voice_clip_id=3,
user_id="u1",
channel="wechat",
user_real_name="Alice",
company_code="other",
start_time=2900,
end_time=3000,
oss_file_name=None,
mp3_duration=0,
duration_format="00:00:00",
),
]
)
await session.commit()
async def test_default_list_returns_all_ordered_desc(make_client):
"""No filters -> 3 rows, ordered by ``end_time desc``."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips")
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert [it["voice_clip_id"] for it in body["items"]] == [3, 2, 1]
for key in (
"voice_clip_id",
"user_id",
"channel",
"user_real_name",
"wx_id",
"talker",
"status",
"is_send",
"company_code",
"start_time",
"end_time",
"oss_file_name",
"mp3_duration",
"duration_format",
"raw_json",
):
assert key in body["items"][0]
async def test_filter_by_user_id(make_client):
"""``?user_id=u1`` -> 2 clips for Alice."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"user_id": "u1"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 3}
async def test_user_id_takes_precedence_over_user_real_name(make_client):
"""When both ``user_id`` and ``user_real_name`` are given, user_id wins.
The 云客 modal switches between the two based on whether the user edited
the staff-name box; the id is the more precise filter and wins on conflict.
Here ``user_id=u1`` (Alice) + ``user_real_name=Bob`` should still return
Alice's clips (user_id path), NOT Bob's.
"""
client = await make_client(_seed)
r = await client.get(
"/api/voice-clips", params={"user_id": "u1", "user_real_name": "Bob"}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 3}
async def test_filter_by_user_real_name_fuzzy(make_client):
"""``?user_real_name=ob`` -> Bob's clip only (ILIKE contains)."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"user_real_name": "ob"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filter_by_channel(make_client):
"""``?channel=whatsapp`` -> the single WA clip."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"channel": "whatsapp"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filter_by_company_code(make_client):
"""``?company_code=8m0atd`` -> 2 clips (clip 3 has 'other')."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"company_code": "8m0atd"})
assert r.status_code == 200
body = r.json()
assert body["total"] == 2
assert {it["voice_clip_id"] for it in body["items"]} == {1, 2}
async def test_time_window_filter(make_client):
"""``?start=1500&end=2500`` -> only clip 2 (end_time 2000)."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"start": 1500, "end": 2500})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 2
async def test_filters_compose(make_client):
"""``user_id=u1 & start=1500`` -> only clip 3 (u1's later clip)."""
client = await make_client(_seed)
r = await client.get(
"/api/voice-clips", params={"user_id": "u1", "start": 1500}
)
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 3
async def test_pagination(make_client):
"""``page_size=1`` -> 1 item, total 3."""
client = await make_client(_seed)
r = await client.get("/api/voice-clips", params={"page": 1, "page_size": 1})
assert r.status_code == 200
body = r.json()
assert body["total"] == 3
assert len(body["items"]) == 1
assert body["items"][0]["voice_clip_id"] == 3
async def test_oss_file_name_serializes(make_client):
"""``oss_file_name`` (OSS recording URL) round-trips so the modal can
render an inline ``<audio>`` player — present when set, None when unset.
"""
client = await make_client(_seed)
r = await client.get("/api/voice-clips")
assert r.status_code == 200
by_id = {it["voice_clip_id"]: it for it in r.json()["items"]}
assert by_id[1]["oss_file_name"] == "https://oss/clip1.mp3"
assert by_id[3]["oss_file_name"] is None
async def test_raw_json_serializes_verbatim(make_client):
"""``raw_json`` round-trips the verbatim upstream record when set."""
async def seed(session: AsyncSession) -> None:
await _seed(session)
session.add(
VoiceClip(
voice_clip_id=99,
user_id="u1",
channel="wechat",
user_real_name="Alice",
start_time=5000,
end_time=5100,
raw_json={
"id": 99,
"userId": "u1",
"userRealName": "Alice",
"startTime": "5000",
"endTime": "5100",
"ossFileName": "https://oss/clip99.mp3",
"mp3Duration": 100,
"durationFormat": "00:01:40",
"companyCode": "8m0atd",
},
)
)
await session.commit()
client = await make_client(seed)
r = await client.get("/api/voice-clips", params={"user_id": "u1", "start": 5000})
assert r.status_code == 200
body = r.json()
assert body["total"] == 1
assert body["items"][0]["voice_clip_id"] == 99
assert body["items"][0]["raw_json"] == {
"id": 99,
"userId": "u1",
"userRealName": "Alice",
"startTime": "5000",
"endTime": "5100",
"ossFileName": "https://oss/clip99.mp3",
"mp3Duration": 100,
"durationFormat": "00:01:40",
"companyCode": "8m0atd",
}
# Seeded without an explicit raw_json -> JSON null.
r_all = await client.get("/api/voice-clips")
by_id = {it["voice_clip_id"]: it for it in r_all.json()["items"]}
assert by_id[1]["raw_json"] is None
yunke-server/tests/test_scrape_voice_clips.py
+297
View File
@@ -0,0 +1,297 @@
"""Tests for the voice-clip (长语音) scraper.
Drives the real page-number pagination + normalization logic through
``respx`` by wiring a real ``YunkeClient`` (primed with a fake session so the
login flow is skipped) against a mocked transport.
The fixture ships 4 records across 2 pages of size 2, plus a terminating
empty page (the page-number paginator's primary stop signal):
* page 1 -> 2 records, ``total=4``
* page 2 -> 2 records, ``total=4``
* page 3 -> empty ``data`` array (no more rows)
The scraper must yield exactly the 4 records normalized to the ``VoiceClip``
column shape, make exactly 3 calls, and stop without a 4th. Field mapping
asserted end-to-end incl.:
* ``id`` -> ``voice_clip_id`` (PK, int)
* ``userId`` -> ``user_id``
* ``userRealName`` -> ``user_real_name``
* ``startTime``/``endTime`` -> ``start_time``/``end_time`` (ms-epoch STRING
coerced to int)
* ``ossFileName`` -> ``oss_file_name`` (may be null)
* ``mp3Duration`` -> ``mp3_duration`` (int seconds)
* ``channel`` -> stamped locally from the ``channel`` arg (``wechat`` /
``whatsapp``), NOT from the API response
"""
from __future__ import annotations
import json as _json
import httpx
import respx
from app.scraper.voice_clip import scrape_voice_clips
from app.session.yunke_client import YunkeClient
from tests.cassettes.voiceClip_fixture import (
DEFAULT_PAGE_SIZE,
TOTAL,
VOICE_CLIP_RECORDS,
make_voice_clip_list_response,
)
BASE = "https://crm2.yunkecn.com"
VOICE_CLIP_LIST_URL = f"{BASE}/wechatweb/call/list"
USERID = "a" * 32
COMPANY = "8m0atd"
def _primed_client() -> YunkeClient:
"""A YunkeClient already in a logged-in state (login flow skipped)."""
return YunkeClient(base_url=BASE, userid=USERID, company=COMPANY)
# ---------------------------------------------------------------------------
# End-to-end: pagination + field mapping + raw_json
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_yields_records_and_paginates():
"""Two full pages + a terminating empty page -> 4 records, 3 calls."""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:2], total=TOTAL, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[2:], total=TOTAL, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
page3 = make_voice_clip_list_response(
[], total=TOTAL, page_num=3, page_size=DEFAULT_PAGE_SIZE
)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
httpx.Response(200, json=page3),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client,
"832EC001_wechat_staff",
channel="wechat",
page_size=DEFAULT_PAGE_SIZE,
)
]
finally:
await client.aclose()
# Pagination: exactly 3 calls (page1, page2, empty-page stop).
assert route.call_count == 3
# Exactly the 4 fixture records (not duplicated, not dropped).
assert len(results) == 4
assert {r["voice_clip_id"] for r in results} == {
rec["id"] for rec in VOICE_CLIP_RECORDS
}
# Field mapping for each record (compare against the source record).
for result, record in zip(results, VOICE_CLIP_RECORDS, strict=True):
# Identity + locally-stamped channel.
assert result["voice_clip_id"] == record["id"]
assert result["channel"] == "wechat"
# Staff side.
assert result["user_id"] == record["userId"]
assert result["user_real_name"] == record["userRealName"]
assert result["company_code"] == record["companyCode"]
# ms-epoch strings coerced to int.
assert result["start_time"] == int(record["startTime"])
assert result["end_time"] == int(record["endTime"])
# Recording URL (may be None).
assert result["oss_file_name"] == record["ossFileName"]
# Duration fields.
assert result["mp3_duration"] == record["mp3Duration"]
assert result["duration_format"] == record["durationFormat"]
# raw_json preserves the original record verbatim.
assert result["raw_json"] == record
# ---------------------------------------------------------------------------
# Channel is stamped locally (NOT from the API response)
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_stamps_whatsapp_channel():
"""``channel="whatsapp"`` is stamped on every record regardless of payload."""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:1], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "F33DA003_wa_staff", channel="whatsapp"
)
]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["channel"] == "whatsapp"
# ---------------------------------------------------------------------------
# Null ossFileName -> oss_file_name None
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_null_oss_file_name_yields_none():
"""A record with ``ossFileName=null`` -> ``oss_file_name`` None."""
rec = {
"id": 66010,
"userId": "staff_x",
"userRealName": "X",
"wxId": None,
"talker": None,
"status": None,
"isSend": None,
"companyCode": "8m0atd",
"startTime": "1786072733000",
"endTime": "1786072733000",
"ossFileName": None,
"mp3Duration": 0,
"durationFormat": "00:00:00",
}
page1 = make_voice_clip_list_response(
[rec], total=1, page_num=1, page_size=DEFAULT_PAGE_SIZE
)
page2 = make_voice_clip_list_response(
[], total=1, page_num=2, page_size=DEFAULT_PAGE_SIZE
)
respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "staff_x", channel="wechat", page_size=DEFAULT_PAGE_SIZE
)
]
finally:
await client.aclose()
assert len(results) == 1
assert results[0]["oss_file_name"] is None
assert results[0]["voice_clip_id"] == 66010
# ---------------------------------------------------------------------------
# Pagination body: pageNum/pageSize/userId
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_sends_pagination_body():
"""Body carries ``userId`` + advancing ``pageNum``/``pageSize`` only.
The ``call/list`` endpoint takes ``{pageNum, pageSize, userId}``. The
optional ``startTime``/``endTime`` filters are omitted for full-history
sync (the scraper pulls the employee's whole voice-clip set in one pass).
"""
page1 = make_voice_clip_list_response(
VOICE_CLIP_RECORDS[:1], total=TOTAL, page_num=1, page_size=1
)
page2 = make_voice_clip_list_response(
[], total=TOTAL, page_num=2, page_size=1
)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
side_effect=[
httpx.Response(200, json=page1),
httpx.Response(200, json=page2),
]
)
client = _primed_client()
try:
results = [
r
async for r in scrape_voice_clips(
client, "EMP_USER_001", channel="wechat", page_size=1
)
]
finally:
await client.aclose()
# One record on page 1, empty page 2 stops the loop.
assert len(results) == 1
assert route.call_count == 2
# Page 1 body: userId + standard pagination pair, no time window.
body1 = _json.loads(route.calls[0].request.content.decode())
assert body1["userId"] == "EMP_USER_001"
assert body1["pageNum"] == 1
assert body1["pageSize"] == 1
# No startTime/endTime (full-history scrape).
assert "startTime" not in body1
assert "endTime" not in body1
# Page 2 advances pageNum and keeps userId verbatim.
body2 = _json.loads(route.calls[1].request.content.decode())
assert body2["pageNum"] == 2
assert body2["pageSize"] == 1
assert body2["userId"] == "EMP_USER_001"
# ---------------------------------------------------------------------------
# Empty first page -> no yields, single call
# ---------------------------------------------------------------------------
@respx.mock
async def test_scrape_voice_clips_stops_on_empty_first_page():
"""If the very first page is empty, scraper stops without over-fetching."""
empty = make_voice_clip_list_response([], total=0, page_num=1)
route = respx.post(VOICE_CLIP_LIST_URL).mock(
return_value=httpx.Response(200, json=empty)
)
client = _primed_client()
try:
results = [r async for r in scrape_voice_clips(client, "any_user")]
finally:
await client.aclose()
assert results == []
assert route.call_count == 1