diff --git a/frontend/src/components/file/inline-file-preview-utils.test.ts b/frontend/src/components/file/inline-file-preview-utils.test.ts index b440d5bb6..b42c62459 100644 --- a/frontend/src/components/file/inline-file-preview-utils.test.ts +++ b/frontend/src/components/file/inline-file-preview-utils.test.ts @@ -31,6 +31,8 @@ describe('inline-file-preview-utils', () => { ).toBe('document') expect(getInlineFilePreviewKind({ filename: 'data.xlsx' })).toBe('spreadsheet') expect(getInlineFilePreviewKind({ filename: 'chart.png' })).toBe('image') + expect(getInlineFilePreviewKind({ filename: 'podcast.mp3' })).toBe('audio') + expect(getInlineFilePreviewKind({ mimeType: 'audio/mpeg' })).toBe('audio') }) it('classifies .pptx (OOXML) as inline-previewable presentation', () => { @@ -236,6 +238,7 @@ describe('inline-file-preview-utils', () => { 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) expect(getInlineFilePreviewMimeType('image')).toBeUndefined() + expect(getInlineFilePreviewMimeType('audio')).toBeUndefined() expect(getInlineFilePreviewMimeType('file')).toBeUndefined() }) }) diff --git a/frontend/src/components/file/inline-file-preview-utils.ts b/frontend/src/components/file/inline-file-preview-utils.ts index 62d389ebf..173e713b2 100644 --- a/frontend/src/components/file/inline-file-preview-utils.ts +++ b/frontend/src/components/file/inline-file-preview-utils.ts @@ -2,6 +2,7 @@ import { getFilePublicDownloadUrl, getFilePublicPreviewUrl } from '@/lib/utils' export type InlineFilePreviewKind = | 'image' + | 'audio' | 'presentation' | 'document' | 'spreadsheet' @@ -25,6 +26,7 @@ export type PreviewUrlTrust = { const PREVIEWABLE_KINDS = new Set([ 'image', + 'audio', 'presentation', 'document', 'spreadsheet', @@ -58,7 +60,7 @@ export const INLINE_FILE_PREVIEW_MIME_BY_KIND: Partial< export const getInlineFilePreviewMimeType = ( kind: InlineFilePreviewKind ): string | undefined => { - if (kind === 'file' || kind === 'image') return undefined + if (kind === 'file' || kind === 'image' || kind === 'audio') return undefined return INLINE_FILE_PREVIEW_MIME_BY_KIND[kind] } @@ -75,6 +77,7 @@ export const getInlineFilePreviewKind = ( const mimeType = source.mimeType?.toLowerCase() || '' if (type === 'image') return 'image' + if (type === 'audio') return 'audio' if (type === 'presentation') { // An explicit ``type: 'presentation'`` artifact must still be // cross-checked: pptxviewjs only supports OOXML .pptx, so a @@ -104,6 +107,7 @@ export const getInlineFilePreviewKind = ( if (type === 'spreadsheet') return 'spreadsheet' if (mimeType.startsWith('image/')) return 'image' + if (mimeType.startsWith('audio/')) return 'audio' if (PRESENTATION_MIME_TYPES.has(mimeType) || mimeType.includes('presentationml')) { return 'presentation' } @@ -113,6 +117,7 @@ export const getInlineFilePreviewKind = ( } if (/\.(jpg|jpeg|png|gif|webp|svg)$/.test(filename)) return 'image' + if (/\.(mp3|wav|ogg|opus|flac|m4a|aac)$/.test(filename)) return 'audio' // Only OOXML .pptx is previewable inline — see PRESENTATION_MIME_TYPES // comment. Legacy .ppt falls through to the generic 'file' kind. if (filename.endsWith('.pptx')) return 'presentation' diff --git a/frontend/src/components/file/inline-file-preview.test.tsx b/frontend/src/components/file/inline-file-preview.test.tsx index ae6c82544..c25e1d46a 100644 --- a/frontend/src/components/file/inline-file-preview.test.tsx +++ b/frontend/src/components/file/inline-file-preview.test.tsx @@ -193,6 +193,85 @@ describe('InlineFilePreview', () => { }) }) + it('loads managed audio files through authenticated preview', async () => { + apiRequestMock.mockResolvedValue({ + ok: true, + blob: async () => new Blob(['audio-bytes'], { type: 'audio/mpeg' }), + }) + + render( + + ) + + await waitFor(() => { + expect(apiRequestMock).toHaveBeenCalledWith( + 'http://api.local/api/files/preview/audio-file-id', + expect.objectContaining({ cache: 'no-cache' }) + ) + }) + + const audio = await screen.findByLabelText('podcast.mp3') + expect(audio.tagName.toLowerCase()).toBe('audio') + expect(audio.getAttribute('src')).toMatch(/^blob:/) + expect(screen.getByRole('link', { name: 'Open' }).getAttribute('href')).toMatch( + /^blob:/ + ) + }) + + it('falls back to the public audio preview when authenticated loading fails', async () => { + apiRequestMock.mockResolvedValue({ ok: false, status: 401 }) + + render( + + ) + + const audio = await screen.findByLabelText('podcast.mp3') + expect(audio).toHaveAttribute( + 'src', + 'http://api.local/api/files/public/preview/audio-file-id' + ) + expect(screen.getByRole('link', { name: 'Open' })).toHaveAttribute( + 'href', + 'http://api.local/api/files/public/preview/audio-file-id' + ) + }) + + it('loads legacy workspace audio paths through authenticated preview', async () => { + apiRequestMock.mockResolvedValue({ + ok: true, + blob: async () => new Blob(['audio-bytes'], { type: 'audio/mpeg' }), + }) + + render( + + ) + + await waitFor(() => { + expect(apiRequestMock).toHaveBeenCalledWith( + 'http://api.local/api/files/preview/output%2Fxagent_061_podcast.mp3', + expect.objectContaining({ cache: 'no-cache' }) + ) + }) + + const audio = await screen.findByLabelText('xagent_061_podcast.mp3') + await waitFor(() => { + expect(audio.getAttribute('src')).toMatch(/^blob:/) + }) + expect(screen.getByRole('link', { name: 'Open' }).getAttribute('href')).toMatch( + /^blob:/ + ) + }) + it('mounts PptxPreviewRenderer immediately with fileId without eager byte fetch', () => { // PDF-first path: when a managed fileId is available, InlineFilePreview // skips the eager /api/files/public/preview bytes download and mounts diff --git a/frontend/src/components/file/inline-file-preview.tsx b/frontend/src/components/file/inline-file-preview.tsx index 4b39636aa..b66c2ff5a 100644 --- a/frontend/src/components/file/inline-file-preview.tsx +++ b/frontend/src/components/file/inline-file-preview.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react' -import { FileText, Loader2 } from 'lucide-react' +import { FileText, Loader2, Volume2 } from 'lucide-react' import { DocxPreviewRenderer } from '@/components/file/docx-preview-renderer' import { ExcelPreviewRenderer } from '@/components/file/excel-preview-renderer' @@ -130,6 +130,108 @@ function InlineImagePreview({ ) } +function InlineAudioPreview({ + source, + previewUrl, + filename, + openLabel, + className, +}: { + source: InlineFilePreviewSource + previewUrl: string + filename: string + openLabel: string + className?: string +}) { + const apiUrl = getApiUrl() + const shouldFallback = Boolean(source.fileId) + const [resolvedUrl, setResolvedUrl] = useState(shouldFallback ? '' : previewUrl) + + useEffect(() => { + let objectUrl: string | null = null + let isCancelled = false + + setResolvedUrl(shouldFallback ? '' : previewUrl) + + const loadAuthenticatedAudio = async () => { + if (!shouldFallback || !source.fileId) return + try { + const response = await apiRequest( + `${apiUrl}/api/files/preview/${encodeURIComponent(source.fileId)}`, + { + cache: 'no-cache', + headers: { + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + }, + } + ) + if (isCancelled) return + if (!response.ok) { + setResolvedUrl(previewUrl) + return + } + const blob = await response.blob() + if (isCancelled) return + objectUrl = URL.createObjectURL(blob) + setResolvedUrl(objectUrl) + } catch { + if (!isCancelled) { + setResolvedUrl(previewUrl) + } + } + } + + void loadAuthenticatedAudio() + + return () => { + isCancelled = true + if (objectUrl) URL.revokeObjectURL(objectUrl) + } + }, [apiUrl, previewUrl, shouldFallback, source.fileId]) + + return ( +
+
+ + {filename} + {resolvedUrl ? ( + + {openLabel} + + ) : null} +
+
+ {resolvedUrl ? ( +
+
+ ) +} + function InlineOfficeContent({ kind, previewUrl, @@ -303,6 +405,18 @@ export function InlineFilePreview({ ) } + if (kind === 'audio') { + return ( + + ) + } + if (!isPreviewableInlineFileKind(kind)) { return ( { + it("keeps sound effect and music distinct on the audio tab", () => { + expect( + getProviderDisplayCapabilities( + [ + { category: "speech", abilities: ["tts"] }, + { category: "speech", abilities: ["asr"] }, + { category: "sound_effect", abilities: ["generate"] }, + { category: "music", abilities: ["generate"] }, + ], + "audio", + ), + ).toEqual(["tts", "asr", "sound_effect", "music"]) + }) + + it("keeps generic generate for non-audio provider cards", () => { + expect( + getProviderDisplayCapabilities( + [{ category: "image", abilities: ["generate"] }], + "image", + ), + ).toEqual(["generate"]) + }) + + it("keeps non-generate abilities on sound effect and music models", () => { + expect( + getProviderDisplayCapabilities( + [ + { category: "sound_effect", abilities: ["generate", "edit"] }, + { category: "music", abilities: ["generate", "tts"] }, + ], + "audio", + ), + ).toEqual(["sound_effect", "edit", "music", "tts"]) + }) + + it("deduplicates repeated provider capabilities", () => { + expect( + getProviderDisplayCapabilities( + [ + { category: "speech", abilities: ["tts"] }, + { category: "speech", abilities: ["tts"] }, + ], + "audio", + ), + ).toEqual(["tts"]) + }) +}) diff --git a/frontend/src/components/pages/model-display-capabilities.ts b/frontend/src/components/pages/model-display-capabilities.ts new file mode 100644 index 000000000..9e1f73257 --- /dev/null +++ b/frontend/src/components/pages/model-display-capabilities.ts @@ -0,0 +1,35 @@ +export interface DisplayCapabilityModel { + category: string + abilities?: string[] +} + +/** + * Build provider-card capability badges without collapsing distinct audio + * model categories that share the generic backend `generate` ability. + */ +export function getProviderDisplayCapabilities( + models: DisplayCapabilityModel[], + activeTab: string, +): string[] { + const capabilities = new Set() + + models.forEach((model) => { + const isSoundEffect = + activeTab === "audio" && model.category === "sound_effect" + const isMusic = activeTab === "audio" && model.category === "music" + + if (isSoundEffect) { + capabilities.add("sound_effect") + } else if (isMusic) { + capabilities.add("music") + } + + model.abilities?.forEach((ability) => { + if (ability !== "generate" || (!isSoundEffect && !isMusic)) { + capabilities.add(ability) + } + }) + }) + + return Array.from(capabilities) +} diff --git a/frontend/src/components/pages/models.tsx b/frontend/src/components/pages/models.tsx index b9ac5af27..a09525b59 100644 --- a/frontend/src/components/pages/models.tsx +++ b/frontend/src/components/pages/models.tsx @@ -21,10 +21,12 @@ import { CheckCircle2, Video, Volume2, + Music, } from "lucide-react" import { useI18n } from "@/contexts/i18n-context" import { ModelManagementDialog } from "./model-management-dialog" import { toast } from "@/components/ui/sonner" +import { getProviderDisplayCapabilities } from "./model-display-capabilities" const MODEL_TABS = ["llm", "embedding", "rerank", "image", "video", "audio"] as const const DEFAULT_MODEL_TAB = "llm" @@ -533,14 +535,7 @@ export function ModelsPage() { ? providerModels[0].model_name : providerConfig.name - // Gather unique abilities across all models in this provider - const allAbilities = new Set() - providerModels.forEach(m => { - if (m.abilities) { - m.abilities.forEach(a => allAbilities.add(a)) - } - }) - const capabilities = Array.from(allAbilities) + const capabilities = getProviderDisplayCapabilities(providerModels, activeTab) const labels: Record = { chat: t('models.abilities.chat'), vision: t('models.abilities.vision'), @@ -550,7 +545,9 @@ export function ModelsPage() { generate: t('models.abilities.generate'), edit: t('models.abilities.edit'), asr: t('models.abilities.asr'), - tts: t('models.abilities.tts') + tts: t('models.abilities.tts'), + sound_effect: t('models.abilities.sound_effect'), + music: t('models.abilities.music') } const icons: Record = { chat: , @@ -565,7 +562,9 @@ export function ModelsPage() { : , edit: , asr: , - tts: + tts: , + sound_effect: , + music: } // Check defaults diff --git a/frontend/src/components/ui/__tests__/markdown-renderer.test.tsx b/frontend/src/components/ui/__tests__/markdown-renderer.test.tsx index 79e3c09c2..cb63c076b 100644 --- a/frontend/src/components/ui/__tests__/markdown-renderer.test.tsx +++ b/frontend/src/components/ui/__tests__/markdown-renderer.test.tsx @@ -208,6 +208,29 @@ describe('MarkdownRenderer', () => { }) }) + it('corrects legacy-path image markdown for mp3 files to a playable audio blob', async () => { + apiRequestMock.mockResolvedValue({ + ok: true, + blob: async () => new Blob(['audio-bytes'], { type: 'audio/mpeg' }), + }) + const content = + '![xagent_061_podcast.mp3](file:output/xagent_061_podcast.mp3)' + render() + + await waitFor(() => { + expect(apiRequestMock).toHaveBeenCalledWith( + 'http://api.local/api/files/preview/output%2Fxagent_061_podcast.mp3', + expect.objectContaining({ cache: 'no-cache' }) + ) + }) + const audio = await screen.findByLabelText('xagent_061_podcast.mp3') + expect(audio.getAttribute('src')).toMatch(/^blob:/) + expect(screen.getByRole('link', { name: 'Open' }).getAttribute('href')).toMatch( + /^blob:/ + ) + expect(screen.queryByRole('img')).not.toBeInTheDocument() + }) + it('prefers link label over generic file id when determining preview kind', async () => { apiRequestMock.mockResolvedValue({ ok: true, diff --git a/frontend/src/components/ui/markdown-renderer.tsx b/frontend/src/components/ui/markdown-renderer.tsx index 7e405fe2b..bf2ad3ad2 100644 --- a/frontend/src/components/ui/markdown-renderer.tsx +++ b/frontend/src/components/ui/markdown-renderer.tsx @@ -341,12 +341,15 @@ export function MarkdownRenderer({ content, className = '', onFileClick, onAgent const filePath = src.replace(/^file:/, '') const fileNameFromPath = filePath.split('/').pop() || filePath const fileName = title || alt || fileNameFromPath + const preview = resolvePreviewableFileLink({ fileNameFromPath, fileName }) + const previewKind = preview?.previewKind ?? 'image' return ( bool: """Check if model supports voice cloning.""" return "voice_cloning" in self.abilities + @property + def supports_persistent_voice_cloning(self) -> bool: + """Check if model can create reusable provider-side voice IDs.""" + return "persistent_voice_cloning" in self.abilities + @property def supports_voice_listing(self) -> bool: """Check if model can list available voices dynamically.""" @@ -103,3 +109,17 @@ async def list_available_voices(self) -> list[dict[str, Any]]: raise NotImplementedError( f"{self.provider_name} TTS does not support dynamic voice listing" ) + + async def clone_voice( + self, + *, + name: str, + reference_audio_files: list[str], + description: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + remove_background_noise: bool = False, + ) -> dict[str, Any]: + """Create a persistent provider-side voice clone.""" + raise NotImplementedError( + f"{self.provider_name} TTS does not support persistent voice cloning" + ) diff --git a/src/xagent/core/model/tts/elevenlabs.py b/src/xagent/core/model/tts/elevenlabs.py index ec10e3983..8fc6e77da 100644 --- a/src/xagent/core/model/tts/elevenlabs.py +++ b/src/xagent/core/model/tts/elevenlabs.py @@ -2,10 +2,15 @@ from __future__ import annotations +import asyncio +import json import logging +import mimetypes import os +import uuid from collections.abc import Iterable from inspect import isawaitable +from pathlib import Path from typing import Any, Optional, Union from ...utils.security import redact_sensitive_text @@ -101,6 +106,8 @@ def __init__( self.sample_rate = _sample_rate_from_output_format(self.output_format) self._client: Any = None self._async_client: Any = None + self._cloned_voice_ids: dict[str, str] = {} + self._voice_clone_lock = asyncio.Lock() @staticmethod def _create_client(api_key: Optional[str], base_url: Optional[str] = None) -> Any: @@ -225,13 +232,48 @@ def _close_sync_client(client: Any) -> None: close() async def aclose(self) -> None: - """Close any cached ElevenLabs SDK clients.""" - client = self._client - async_client = self._async_client - self._client = None - self._async_client = None - await self._close_client(async_client) - await self._close_client(client) + """Delete temporary voice clones and close cached ElevenLabs SDK clients.""" + async with self._voice_clone_lock: + client = self._client + async_client = self._async_client + if ( + async_client is None + and self._cloned_voice_ids + and self.api_key is not None + ): + try: + async_client = self._create_async_client( + self.api_key, self.base_url + ) + except Exception as exc: + logger.warning( + "Failed to create ElevenLabs client for temporary voice cleanup: %s", + redact_sensitive_text(str(exc)), + ) + self._client = None + self._async_client = None + + if async_client is not None and self._cloned_voice_ids: + deleted_voice_ids: set[str] = set() + for voice_id in set(self._cloned_voice_ids.values()): + try: + await async_client.voices.delete(voice_id) + deleted_voice_ids.add(voice_id) + except Exception as exc: + logger.warning( + "Failed to delete temporary ElevenLabs voice clone %s: %s", + voice_id, + redact_sensitive_text(str(exc)), + ) + if deleted_voice_ids: + self._cloned_voice_ids = { + cache_key: voice_id + for cache_key, voice_id in self._cloned_voice_ids.items() + if voice_id not in deleted_voice_ids + } + + await self._close_client(async_client) + await self._close_client(client) async def __aenter__(self) -> "ElevenLabsTTS": return self @@ -340,6 +382,129 @@ def _normalize_verified_languages(languages: Any) -> Optional[list[dict[str, Any return normalized_languages or None + @staticmethod + def _reference_audio_metadata(reference_audio: Any) -> tuple[Path, str, str]: + if not isinstance(reference_audio, (str, os.PathLike)): + raise ValueError( + "ElevenLabs reference_audio must be a local audio file path" + ) + + audio_path = Path(reference_audio).expanduser() + if not audio_path.is_file(): + raise ValueError( + f"ElevenLabs reference audio file does not exist: {audio_path}" + ) + + stat = audio_path.stat() + if stat.st_size == 0: + raise ValueError("ElevenLabs reference audio file must not be empty") + + mime_type = ( + mimetypes.guess_type(audio_path.name)[0] or "application/octet-stream" + ) + resolved_path = str(audio_path.resolve()) + cache_key = f"{resolved_path}:{stat.st_size}:{stat.st_mtime_ns}" + return audio_path, cache_key, mime_type + + @classmethod + def _reference_audio_file(cls, reference_audio: Any) -> tuple[str, bytes, str]: + audio_path, cache_key, mime_type = cls._reference_audio_metadata( + reference_audio + ) + audio = audio_path.read_bytes() + if not audio: + raise ValueError("ElevenLabs reference audio file must not be empty") + return cache_key, audio, mime_type + + async def _get_or_create_cloned_voice(self, reference_audio: Any) -> str: + audio_path, cache_key, mime_type = self._reference_audio_metadata( + reference_audio + ) + cached_voice_id = self._cloned_voice_ids.get(cache_key) + if cached_voice_id: + return cached_voice_id + + async with self._voice_clone_lock: + cached_voice_id = self._cloned_voice_ids.get(cache_key) + if cached_voice_id: + return cached_voice_id + + audio = audio_path.read_bytes() + if not audio: + raise ValueError("ElevenLabs reference audio file must not be empty") + clone_name = f"xagent-{audio_path.stem[:40]}-{uuid.uuid4().hex[:8]}" + client = self._ensure_async_client() + response = await client.voices.ivc.create( + name=clone_name, + files=[(audio_path.name, audio, mime_type)], + ) + voice_id = _get_field(response, "voice_id", "id") + if not voice_id: + raise RuntimeError("ElevenLabs voice cloning returned no voice ID") + + normalized_voice_id = str(voice_id) + self._cloned_voice_ids[cache_key] = normalized_voice_id + return normalized_voice_id + + async def clone_voice( + self, + *, + name: str, + reference_audio_files: list[str], + description: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + remove_background_noise: bool = False, + ) -> dict[str, Any]: + """Create a persistent ElevenLabs Instant Voice Clone.""" + normalized_name = name.strip() + if not normalized_name: + raise ValueError("ElevenLabs voice clone name must not be empty") + if not reference_audio_files: + raise ValueError("At least one reference audio file is required") + + files: list[tuple[str, bytes, str]] = [] + for reference_audio in reference_audio_files: + _, audio, mime_type = self._reference_audio_file(reference_audio) + files.append((Path(reference_audio).name, audio, mime_type)) + + request_kwargs: dict[str, Any] = { + "name": normalized_name, + "files": files, + } + if description is not None: + request_kwargs["description"] = description + if labels is not None: + # SDK 2.0 accepted only a serialized labels object, while newer + # releases also accept a mapping. A JSON string works across both. + request_kwargs["labels"] = json.dumps(labels) + if remove_background_noise: + request_kwargs["remove_background_noise"] = True + + client = self._ensure_async_client() + try: + response = await client.voices.ivc.create(**request_kwargs) + except Exception as exc: + redacted_error = redact_sensitive_text(str(exc)) + logger.error("ElevenLabs voice cloning failed: %s", redacted_error) + raise RuntimeError( + f"ElevenLabs voice cloning failed: {redacted_error}" + ) from exc + + voice_id = _get_field(response, "voice_id", "id") + if not voice_id: + raise RuntimeError("ElevenLabs voice cloning returned no voice ID") + + result: dict[str, Any] = { + "voice_id": str(voice_id), + "name": normalized_name, + "provider": self.provider_name, + "persistent": True, + } + requires_verification = _get_field(response, "requires_verification") + if requires_verification is not None: + result["requires_verification"] = bool(requires_verification) + return result + async def synthesize( self, text: str, @@ -350,6 +515,7 @@ async def synthesize( verbose: bool = False, **kwargs: Any, ) -> Union[bytes, TTSResult]: + reference_audio = kwargs.pop("reference_audio", None) voice_id = voice or self.voice final_output_format = self._resolve_output_format( format or self.output_format, sample_rate or self.sample_rate @@ -374,6 +540,8 @@ async def synthesize( client = self._ensure_async_client() try: + if reference_audio: + voice_id = await self._get_or_create_cloned_voice(reference_audio) response = client.text_to_speech.convert( text=text, voice_id=voice_id, @@ -392,6 +560,19 @@ async def synthesize( chunks.append(self._coerce_audio_bytes(chunk)) audio = b"".join(chunks) except Exception as exc: + error_text = str(exc) + if ( + "invalid_uid" in error_text + or "invalid ID has been received" in error_text + ): + message = ( + "ElevenLabs rejected the provided voice ID. ElevenLabs voice IDs " + "are account-specific opaque values: call list_tts_voices or " + "clone_tts_voice and use an exact returned voice_id, or omit voice " + "to use the configured default." + ) + logger.error("ElevenLabs TTS failed: %s", message) + raise RuntimeError(message) from exc redacted_error = redact_sensitive_text(str(exc)) logger.error( "ElevenLabs TTS failed: %s", @@ -426,6 +607,8 @@ def abilities(self) -> list[str]: "audio_generation", "multilingual", "multiple_voices", + "voice_cloning", + "persistent_voice_cloning", "voice_listing", "voice_settings", "real_time", @@ -486,9 +669,12 @@ def _normalize_voice_response(response: Any) -> list[dict[str, Any]]: "voice_id": str(voice_id), "provider": "elevenlabs", } + category = _get_field(raw_voice, "category") + if category: + voice_info["category"] = str(category).lower() + for field_name in ( "name", - "category", "description", "preview_url", "labels", diff --git a/src/xagent/core/tools/adapters/vibe/audio_tool.py b/src/xagent/core/tools/adapters/vibe/audio_tool.py index a039d3096..c0b58d0ea 100644 --- a/src/xagent/core/tools/adapters/vibe/audio_tool.py +++ b/src/xagent/core/tools/adapters/vibe/audio_tool.py @@ -135,6 +135,23 @@ def get_tools(self) -> list: ) ) + if any( + self._get_tts_provider_name(model) == "elevenlabs" + and getattr(model, "supports_persistent_voice_cloning", False) + for model in voice_listing_models + ): + clone_tts_voice_description = self.CLONE_TTS_VOICE_DESCRIPTION.format( + "elevenlabs" + ) + tools.append( + AudioFunctionTool( + self.clone_tts_voice, + name="clone_tts_voice", + description=clone_tts_voice_description, + owner=self, + ) + ) + return tools diff --git a/src/xagent/core/tools/core/audio_tool.py b/src/xagent/core/tools/core/audio_tool.py index 4204b1a54..f2104b3fe 100644 --- a/src/xagent/core/tools/core/audio_tool.py +++ b/src/xagent/core/tools/core/audio_tool.py @@ -13,13 +13,14 @@ import uuid from inspect import isawaitable from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional from ...file_ref import build_workspace_file_ref from ...model.asr.base import ASRResult, BaseASR from ...model.tts.base import BaseTTS, TTSResult from ...workspace import TaskWorkspace from .audio_tool_descriptions import ( + CLONE_TTS_VOICE_DESCRIPTION, LIST_TTS_VOICES_DESCRIPTION, SYNTHESIZE_SPEECH_DESCRIPTION, SYNTHESIZE_SPEECH_JSON_DESCRIPTION, @@ -41,6 +42,7 @@ class AudioToolCore: SYNTHESIZE_SPEECH_DESCRIPTION = SYNTHESIZE_SPEECH_DESCRIPTION SYNTHESIZE_SPEECH_JSON_DESCRIPTION = SYNTHESIZE_SPEECH_JSON_DESCRIPTION LIST_TTS_VOICES_DESCRIPTION = LIST_TTS_VOICES_DESCRIPTION + CLONE_TTS_VOICE_DESCRIPTION = CLONE_TTS_VOICE_DESCRIPTION def __init__( self, @@ -496,15 +498,22 @@ def _get_tts_model_id(self, tts_model: BaseTTS) -> str: return model_id return "default" - def _get_voice_listing_tts_model( - self, model_id: Optional[str] = None + def _get_provider_tts_model( + self, + *, + provider: str, + capability: str, + model_id: Optional[str] = None, ) -> tuple[Optional[BaseTTS], str]: if model_id: - tts_model = self._get_tts_model(model_id) - actual_model_id = ( - model_id if model_id and model_id in self._tts_models else "default" - ) - return tts_model, actual_model_id + tts_model = self._tts_models.get(model_id) + if ( + tts_model is None + and self._default_tts_model is not None + and getattr(self._default_tts_model, "model_name", None) == model_id + ): + tts_model = self._default_tts_model + return tts_model, model_id candidates: list[BaseTTS] = [] if self._default_tts_model is not None: @@ -512,12 +521,12 @@ def _get_voice_listing_tts_model( candidates.extend(self._tts_models.values()) for candidate in candidates: - if getattr(candidate, "supports_voice_listing", False): + if self._get_tts_provider_name(candidate) == provider and getattr( + candidate, capability, False + ): return candidate, self._get_tts_model_id(candidate) - tts_model = self._get_tts_model() - actual_model_id = self._get_tts_model_id(tts_model) if tts_model else "default" - return tts_model, actual_model_id + return None, "default" def _resolve_audio_path(self, audio_input: str) -> str: """ @@ -763,7 +772,7 @@ async def synthesize_speech( Args: text: Input text to synthesize - voice: Voice ID or name (optional) + voice: Provider-specific voice identifier (optional). Never invent it. language: Language code (optional) audio_format: Output audio format (default: 'mp3') sample_rate: Sample rate in Hz (optional) @@ -809,7 +818,11 @@ async def synthesize_speech( if sample_rate is not None: synthesis_kwargs["sample_rate"] = sample_rate if reference_audio: - synthesis_kwargs["reference_audio"] = reference_audio + synthesis_kwargs["reference_audio"] = ( + self._resolve_audio_path(reference_audio) + if self._workspace is not None + else reference_audio + ) # Synthesize the speech (async) result = await tts_model.synthesize( @@ -946,6 +959,9 @@ def list_available_models(self) -> Dict[str, Any]: "supports_voice_cloning": bool( getattr(tts_model, "supports_voice_cloning", False) ), + "supports_persistent_voice_cloning": bool( + getattr(tts_model, "supports_persistent_voice_cloning", False) + ), "supported_voice_settings": list( getattr(tts_model, "supported_voice_settings", []) ), @@ -978,13 +994,15 @@ def list_available_models(self) -> Dict[str, Any]: async def list_tts_voices( self, + provider: Literal["elevenlabs"] = "elevenlabs", model_id: Optional[str] = None, ) -> Dict[str, Any]: """ List available TTS voices for providers that support dynamic voice lookup. Args: - model_id: Specific TTS model to use. Omit to use the default model. + provider: TTS voice provider. Currently only "elevenlabs". + model_id: Provider model configuration to use. Returns: Dictionary containing: @@ -997,13 +1015,17 @@ async def list_tts_voices( """ supported_providers = self._get_voice_listing_supported_providers() try: - tts_model, actual_model_id = self._get_voice_listing_tts_model(model_id) + tts_model, actual_model_id = self._get_provider_tts_model( + provider=provider, + capability="supports_voice_listing", + model_id=model_id, + ) if not tts_model: return { "success": False, "supported": False, - "error": "No available TTS models configured", + "error": f"No {provider} TTS model is configured", "voices": [], "count": 0, "model_used": actual_model_id, @@ -1011,6 +1033,20 @@ async def list_tts_voices( } provider_name = self._get_tts_provider_name(tts_model) + if provider_name != provider: + return { + "success": False, + "supported": False, + "error": ( + f"list_tts_voices provider is '{provider}', but model " + f"'{actual_model_id}' uses provider '{provider_name}'." + ), + "voices": [], + "count": 0, + "provider": provider_name, + "model_used": actual_model_id, + "supported_providers": supported_providers, + } if not getattr(tts_model, "supports_voice_listing", False): return { "success": False, @@ -1055,6 +1091,85 @@ async def list_tts_voices( "supported_providers": supported_providers, } + async def clone_tts_voice( + self, + name: str, + reference_audio_files: List[str], + provider: Literal["elevenlabs"] = "elevenlabs", + description: Optional[str] = None, + labels: Optional[Dict[str, str]] = None, + remove_background_noise: bool = False, + model_id: Optional[str] = None, + ) -> Dict[str, Any]: + """Create a persistent voice clone with a configured provider account.""" + try: + tts_model, actual_model_id = self._get_provider_tts_model( + provider=provider, + capability="supports_persistent_voice_cloning", + model_id=model_id, + ) + if not tts_model: + return { + "success": False, + "supported": False, + "error": f"No {provider} TTS model is configured", + "provider": provider, + "model_used": actual_model_id, + } + + provider_name = self._get_tts_provider_name(tts_model) + if provider_name != provider: + return { + "success": False, + "supported": False, + "error": ( + f"clone_tts_voice provider is '{provider}', but model " + f"'{actual_model_id}' uses provider '{provider_name}'." + ), + "provider": provider_name, + "model_used": actual_model_id, + } + if not getattr(tts_model, "supports_persistent_voice_cloning", False): + return { + "success": False, + "supported": False, + "error": f"The configured {provider} client does not support persistent voice cloning", + "provider": provider_name, + "model_used": actual_model_id, + } + + resolved_audio_files = [ + self._resolve_audio_path(reference_audio) + if self._workspace is not None + else reference_audio + for reference_audio in reference_audio_files + ] + clone = await tts_model.clone_voice( + name=name, + reference_audio_files=resolved_audio_files, + description=description, + labels=labels, + remove_background_noise=remove_background_noise, + ) + return { + "success": True, + "supported": True, + **clone, + "model_used": actual_model_id, + } + except Exception as e: + logger.error(f"Failed to clone TTS voice: {e}") + actual_model_id = ( + model_id if model_id and model_id in self._tts_models else "default" + ) + return { + "success": False, + "supported": True, + "error": str(e), + "provider": provider, + "model_used": actual_model_id, + } + async def synthesize_speech_json( self, json_data: Optional[str | Dict[str, Any]] = None, @@ -1088,7 +1203,7 @@ async def synthesize_speech_json( reference_field: Field name containing reference audio ID (default: "reference_audio_id") voice_settings_field: Field name containing provider voice settings provider_options_field: Field name containing provider synthesis options - default_voice: Default voice for segments without voice specified + default_voice: Provider-specific voice identifier for segments without one default_language: Default language code (auto-detect if None) default_voice_settings: Default provider voice settings for all segments default_provider_options: Default provider synthesis options for all segments @@ -1112,16 +1227,13 @@ async def synthesize_speech_json( "segments": [ { "text": "你好世界", - "voice": "zh-female", "reference_audio_id": "ref_voice_1" }, { "text": "这是一个测试", - "voice": "zh-male", "reference_audio_id": "ref_voice_2" } ], - "default_voice": "zh-female", "output_format": "mp3", "sample_rate": 24000 } @@ -1130,8 +1242,8 @@ async def synthesize_speech_json( >>> # Batch synthesis with voice cloning >>> data = { ... "segments": [ - ... {"text": "你好", "voice": "zh-female", "reference_audio_id": "ref1"}, - ... {"text": "世界", "voice": "zh-male"} + ... {"text": "你好", "reference_audio_id": "ref1"}, + ... {"text": "世界", "reference_audio_id": "ref2"} ... ] ... } >>> result = await synthesize_speech_json(json_data=data) diff --git a/src/xagent/core/tools/core/audio_tool_descriptions.py b/src/xagent/core/tools/core/audio_tool_descriptions.py index c4bdf0c12..a6c808eb5 100644 --- a/src/xagent/core/tools/core/audio_tool_descriptions.py +++ b/src/xagent/core/tools/core/audio_tool_descriptions.py @@ -93,20 +93,21 @@ Parameters: - text (required): text content to synthesize into speech -- voice (optional): voice ID or name (e.g., 'zh-android', 'zh-female', 'en-male'). Omit for default voice. +- voice (optional): provider-specific voice identifier. Never invent or derive this value from language, gender, accent, or style. For ElevenLabs, use only an exact voice_id returned by list_tts_voices or clone_tts_voice. Omit for the configured default voice. - language (optional): language code (e.g., 'zh', 'en', 'yue'). Auto-detected from text if not specified. - audio_format (optional): audio output format (e.g., 'mp3', 'wav', 'pcm'). Default: 'mp3' - sample_rate (optional): sample rate in Hz when the provider/model supports it. -- reference_audio (optional): reference audio file path for voice cloning (if supported by model) +- reference_audio (optional): reference audio file path or workspace file ID for voice cloning (if supported by model). When provided, it takes precedence over voice. ElevenLabs creates a temporary Instant Voice Clone, reuses it within the task, and deletes it during task cleanup. - voice_settings (optional): provider-specific voice shaping object. Use only with models that advertise supports_voice_settings in list_audio_models. - provider_options (optional): provider-specific synthesis options. Use list_audio_models to inspect supported_provider_options. - model_id (optional): specific TTS model to use. Omit to use the default model marked with ⭐[DEFAULT]. Voice options depend on the model: -- Most models support standard voices: male, female, neutral +- Voice identifiers are provider-specific opaque values, not generic labels such as language plus gender. - Some models support voice cloning using reference_audio - Multilingual models can auto-detect language from text -- For providers that support dynamic voice lookup, call list_tts_voices first and pass the returned voice_id as voice. +- For providers that support dynamic voice lookup, call list_tts_voices first and pass an exact returned voice_id as voice. +- If the user requests voice attributes but provides no exact voice ID, inspect list_tts_voices metadata and select a matching returned voice_id. If no selection is needed, omit voice to use the configured default. Provider-specific options: - ElevenLabs supports voice_settings keys: stability, similarity_boost, style, speed, use_speaker_boost. @@ -155,24 +156,27 @@ ```json {{ "segments": [ - {{"text": "你好世界", "voice": "zh-female", "reference_audio": "ref_voice_1"}}, + {{"text": "你好世界", "reference_audio": "ref_voice_1"}}, {{ "text": "这是一个测试", - "voice": "zh-male", "reference_audio": "ref_voice_2", "voice_settings": {{"stability": 0.45, "style": 0.2}} }} ], - "default_voice": "zh-female", "default_provider_options": {{"seed": 1234}}, "output_format": "mp3", "sample_rate": 24000 }} ``` +Voice identifiers: +- Never invent voice or default_voice values from language, gender, accent, or style. +- For ElevenLabs, call list_tts_voices or clone_tts_voice first and use an exact returned voice_id. Omit voice/default_voice to use the configured default. + Voice Cloning: - Use reference_audio in each segment to clone voices from reference audio files - Supports both workspace file IDs and direct file paths (absolute or relative) +- ElevenLabs creates an Instant Voice Clone for each distinct reference audio file, reuses it within the task, and deletes it during task cleanup - Voice cloning quality depends on the reference audio quality - Not all models support voice cloning @@ -203,6 +207,7 @@ Currently supported providers: {}. Parameters: +- provider (optional): voice provider. Currently the only supported value is "elevenlabs". - model_id (optional): specific TTS model to inspect. Omit to use the default TTS model. Output: @@ -215,6 +220,36 @@ - supported_providers (list): Providers that currently support dynamic voice listing Notes: -- ElevenLabs voices may include name, category, description, labels, preview_url, available_for_tiers, settings, and verified_languages. +- ElevenLabs category identifies the voice type: "cloned" is an Instant Voice Clone and "professional" is a Professional Voice Clone; other documented values include "premade" and "generated". Voices may also include name, description, labels, preview_url, available_for_tiers, settings, and verified_languages. - Providers without dynamic listing may still accept provider-specific voice IDs in synthesize_speech. """.strip() + +# Description for clone_tts_voice tool +CLONE_TTS_VOICE_DESCRIPTION = """ +Create a persistent provider voice clone and return its voice ID. + +Currently supported providers: {}. + +The returned voice ID belongs to the provider account/API key selected by provider and model_id. It is not portable to another provider or account. + +Parameters: +- name (required): Human-readable name for the cloned voice. +- reference_audio_files (required): One or more workspace file IDs or local audio file paths containing the same speaker. +- provider (optional): Voice cloning provider. Currently the only supported value is "elevenlabs". +- description (optional): Description stored with the provider voice. +- labels (optional): Provider metadata such as language, accent, gender, age, or use_case. +- remove_background_noise (optional): Ask the provider to isolate background noise before cloning. Default: false. Do not enable for already-clean recordings because it can reduce quality. +- model_id (optional): Provider TTS configuration whose API key and base URL should be used. Omit to select the first configured model for provider. + +Output: +- voice_id: Persistent provider voice ID. Pass this value as voice to synthesize_speech or synthesize_speech_json. +- name: Stored voice name. +- provider: Provider that owns the returned voice ID. +- persistent: Always true for successful clones created by this tool. +- requires_verification: Whether the provider requires additional voice verification. + +Important: +- Only clone a voice when the user has the necessary rights and consent. +- The voice remains in the selected ElevenLabs account after the task ends. It is not deleted during tool teardown. +- For ElevenLabs Instant Voice Cloning, prefer approximately 1-2 minutes of clear, consistent, single-speaker audio without background noise or reverb. +""".strip() diff --git a/tests/core/model/tts/test_elevenlabs.py b/tests/core/model/tts/test_elevenlabs.py index 5e3ea06ee..3232a68d5 100644 --- a/tests/core/model/tts/test_elevenlabs.py +++ b/tests/core/model/tts/test_elevenlabs.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -95,6 +97,35 @@ async def convert(**kwargs: object): assert result == b"direct stream" +async def test_synthesize_turns_invalid_voice_id_into_retry_guidance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def convert(**kwargs: object): + raise RuntimeError( + "headers: {'x-trace-id': 'trace'}, status_code: 400, " + "body: {'detail': {'status': 'invalid_uid', " + "'message': 'An invalid ID has been received: en-male'}}" + ) + + fake_client = SimpleNamespace(text_to_speech=SimpleNamespace(convert=convert)) + monkeypatch.setattr( + ElevenLabsTTS, + "_create_async_client", + staticmethod(lambda api_key, base_url=None: fake_client), + ) + + tts = ElevenLabsTTS(api_key="test-key") + + with pytest.raises(RuntimeError) as exc_info: + await tts.synthesize("Hello", voice="en-male") + + error = str(exc_info.value) + assert "call list_tts_voices" in error + assert "exact returned voice_id" in error + assert "omit voice" in error + assert "headers" not in error + + async def test_synthesize_maps_pcm_sample_rate( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -178,15 +209,204 @@ async def test_synthesize_rejects_mismatched_sample_rate_for_fixed_format() -> N assert "Use format='pcm' for custom sample rates" in str(exc_info.value) -async def test_synthesize_rejects_unsupported_provider_options() -> None: +async def test_synthesize_clones_reference_audio_and_reuses_voice( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + reference_audio = tmp_path / "reference.wav" + reference_audio.write_bytes(b"reference-audio") + original_read_bytes = Path.read_bytes + read_paths: list[Path] = [] + + def read_bytes(path: Path) -> bytes: + read_paths.append(path) + return original_read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + create = AsyncMock(return_value=SimpleNamespace(voice_id="cloned-voice")) + + async def convert(**kwargs: object): + async def gen(): + yield b"cloned-audio" + + return gen() + + fake_client = SimpleNamespace( + voices=SimpleNamespace( + ivc=SimpleNamespace(create=create), + delete=AsyncMock(), + ), + text_to_speech=SimpleNamespace(convert=convert), + ) + monkeypatch.setattr( + ElevenLabsTTS, + "_create_async_client", + staticmethod(lambda api_key, base_url=None: fake_client), + ) + tts = ElevenLabsTTS(api_key="test-key") + first = await tts.synthesize("Hello", reference_audio=str(reference_audio)) + second = await tts.synthesize("Again", reference_audio=str(reference_audio)) - with pytest.raises(ValueError) as exc_info: - await tts.synthesize("Hello", reference_audio="ref.wav") + assert first == b"cloned-audio" + assert second == b"cloned-audio" + create.assert_awaited_once() + call = create.await_args.kwargs + assert call["name"].startswith("xagent-reference-") + assert call["files"] == [("reference.wav", b"reference-audio", "audio/x-wav")] + assert read_paths == [reference_audio] + + +async def test_synthesize_rejects_missing_reference_audio() -> None: + tts = ElevenLabsTTS(api_key="test-key") + + with pytest.raises(RuntimeError, match="reference audio file does not exist"): + await tts.synthesize("Hello", reference_audio="missing.wav") + + +async def test_aclose_deletes_temporary_voice_clones() -> None: + delete = AsyncMock() + async_client = SimpleNamespace(voices=SimpleNamespace(delete=delete)) + tts = ElevenLabsTTS(api_key="test-key") + tts._async_client = async_client + tts._cloned_voice_ids = { + "first": "voice-1", + "same-voice": "voice-1", + "second": "voice-2", + } + + await tts.aclose() + + assert delete.await_count == 2 + assert {call.args[0] for call in delete.await_args_list} == { + "voice-1", + "voice-2", + } + assert tts._cloned_voice_ids == {} + + +async def test_aclose_waits_for_inflight_temporary_voice_clone( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + reference_audio = tmp_path / "reference.wav" + reference_audio.write_bytes(b"reference-audio") + create_started = asyncio.Event() + allow_create = asyncio.Event() - assert "Unsupported ElevenLabs provider_options keys: reference_audio" in str( - exc_info.value + async def create(**kwargs: object) -> SimpleNamespace: + create_started.set() + await allow_create.wait() + return SimpleNamespace(voice_id="inflight-voice") + + delete = AsyncMock() + fake_client = SimpleNamespace( + voices=SimpleNamespace( + ivc=SimpleNamespace(create=create), + delete=delete, + ) + ) + monkeypatch.setattr( + ElevenLabsTTS, + "_create_async_client", + staticmethod(lambda api_key, base_url=None: fake_client), + ) + + tts = ElevenLabsTTS(api_key="test-key") + clone_task = asyncio.create_task( + tts._get_or_create_cloned_voice(str(reference_audio)) ) + await create_started.wait() + + close_task = asyncio.create_task(tts.aclose()) + await asyncio.sleep(0) + assert close_task.done() is False + + allow_create.set() + assert await clone_task == "inflight-voice" + await close_task + + delete.assert_awaited_once_with("inflight-voice") + assert tts._cloned_voice_ids == {} + + +async def test_aclose_retries_failed_temporary_voice_cleanup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + first_delete = AsyncMock(side_effect=RuntimeError("temporary network error")) + first_client = SimpleNamespace(voices=SimpleNamespace(delete=first_delete)) + retry_delete = AsyncMock() + retry_client = SimpleNamespace(voices=SimpleNamespace(delete=retry_delete)) + monkeypatch.setattr( + ElevenLabsTTS, + "_create_async_client", + staticmethod(lambda api_key, base_url=None: retry_client), + ) + + tts = ElevenLabsTTS(api_key="test-key") + tts._async_client = first_client + tts._cloned_voice_ids = {"reference": "voice-to-retry"} + + await tts.aclose() + assert tts._cloned_voice_ids == {"reference": "voice-to-retry"} + + await tts.aclose() + retry_delete.assert_awaited_once_with("voice-to-retry") + assert tts._cloned_voice_ids == {} + + +async def test_clone_voice_creates_persistent_voice_without_cleanup( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + first_audio = tmp_path / "first.mp3" + second_audio = tmp_path / "second.wav" + first_audio.write_bytes(b"first-audio") + second_audio.write_bytes(b"second-audio") + create = AsyncMock( + return_value=SimpleNamespace( + voice_id="persistent-voice", + requires_verification=False, + ) + ) + delete = AsyncMock() + fake_client = SimpleNamespace( + voices=SimpleNamespace( + ivc=SimpleNamespace(create=create), + delete=delete, + ) + ) + monkeypatch.setattr( + ElevenLabsTTS, + "_create_async_client", + staticmethod(lambda api_key, base_url=None: fake_client), + ) + + tts = ElevenLabsTTS(api_key="test-key") + result = await tts.clone_voice( + name="Product narrator", + reference_audio_files=[str(first_audio), str(second_audio)], + description="Consistent narration voice", + labels={"language": "en", "accent": "american"}, + remove_background_noise=True, + ) + await tts.aclose() + + assert result == { + "voice_id": "persistent-voice", + "name": "Product narrator", + "provider": "elevenlabs", + "persistent": True, + "requires_verification": False, + } + assert create.await_args.kwargs == { + "name": "Product narrator", + "files": [ + ("first.mp3", b"first-audio", "audio/mpeg"), + ("second.wav", b"second-audio", "audio/x-wav"), + ], + "description": "Consistent narration voice", + "labels": '{"language": "en", "accent": "american"}', + "remove_background_noise": True, + } + delete.assert_not_awaited() async def test_synthesize_redacts_sensitive_sdk_errors( @@ -436,10 +656,21 @@ async def test_list_available_voices_normalizes_sdk_voices( get_all.assert_awaited_once_with() +@pytest.mark.parametrize("category", ["cloned", "professional", "generated", "premade"]) +def test_normalize_voice_response_preserves_category(category: str) -> None: + voices = ElevenLabsTTS._normalize_voice_response( + {"voices": [{"voice_id": "voice-123", "category": category}]} + ) + + assert voices[0]["category"] == category + + def test_elevenlabs_advertises_voice_capabilities() -> None: tts = ElevenLabsTTS(api_key="test-key") assert tts.provider_name == "elevenlabs" + assert tts.supports_voice_cloning is True + assert tts.supports_persistent_voice_cloning is True assert tts.supports_voice_listing is True assert tts.supports_voice_settings is True assert tts.supported_voice_settings == [ diff --git a/tests/core/tools/test_audio_tool.py b/tests/core/tools/test_audio_tool.py index 37cc661e4..aefdf01fe 100644 --- a/tests/core/tools/test_audio_tool.py +++ b/tests/core/tools/test_audio_tool.py @@ -21,6 +21,7 @@ def __init__( self._abilities = abilities or ["tts", "text_to_speech"] self._voices = voices or [] self.calls: list[dict[str, Any]] = [] + self.clone_calls: list[dict[str, Any]] = [] self.model_name = "fake-tts" async def synthesize( @@ -68,6 +69,32 @@ def supported_provider_options(self) -> list[str]: async def list_available_voices(self) -> list[dict[str, Any]]: return self._voices + async def clone_voice( + self, + *, + name: str, + reference_audio_files: list[str], + description: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + remove_background_noise: bool = False, + ) -> dict[str, Any]: + self.clone_calls.append( + { + "name": name, + "reference_audio_files": reference_audio_files, + "description": description, + "labels": labels, + "remove_background_noise": remove_background_noise, + } + ) + return { + "voice_id": "persistent-voice", + "name": name, + "provider": self.provider_name, + "persistent": True, + "requires_verification": False, + } + class ClosableTTS(FakeTTS): def __init__(self) -> None: @@ -229,6 +256,7 @@ def test_list_audio_models_includes_tts_voice_capabilities() -> None: "supports_voice_listing": True, "supports_voice_settings": True, "supports_voice_cloning": False, + "supports_persistent_voice_cloning": False, "supported_voice_settings": ["stability", "style"], "supported_provider_options": ["seed", "apply_text_normalization"], } @@ -361,7 +389,7 @@ def test_filter_voice_model_metadata_handles_empty_configured_set() -> None: assert voices == [{"voice_id": "voice-1"}] -async def test_list_tts_voices_reports_dynamic_supported_providers() -> None: +async def test_list_tts_voices_rejects_model_from_other_provider() -> None: tool = AudioToolCore( tts_models={ "voice-model": FakeTTS( @@ -374,8 +402,9 @@ async def test_list_tts_voices_reports_dynamic_supported_providers() -> None: result = await tool.list_tts_voices(model_id="voice-model") - assert result["success"] is True - assert result["supported_providers"] == ["customvoice"] + assert result["success"] is False + assert result["supported"] is False + assert "provider is 'elevenlabs'" in result["error"] async def test_list_tts_voices_reports_unsupported_provider() -> None: @@ -387,10 +416,20 @@ async def test_list_tts_voices_reports_unsupported_provider() -> None: assert result["success"] is False assert result["supported"] is False assert result["provider"] == "xinference" - assert "Currently supported providers:" in result["error"] + assert "provider is 'elevenlabs'" in result["error"] assert result["supported_providers"] == [] +async def test_list_tts_voices_reports_missing_elevenlabs_model() -> None: + result = await AudioToolCore().list_tts_voices() + + assert result["success"] is False + assert result["supported"] is False + assert result["error"] == "No elevenlabs TTS model is configured" + assert result["voices"] == [] + assert result["model_used"] == "default" + + def test_list_tts_voices_tool_visible_only_for_voice_listing_provider() -> None: unsupported_tool = AudioTool( tts_models={"chat-tts": FakeTTS(provider_name="xinference", abilities=["tts"])} @@ -426,6 +465,171 @@ def test_synthesize_speech_schema_exposes_structured_options() -> None: assert "voice_settings" in schema_properties assert "provider_options" in schema_properties + assert "Never invent" in synthesize_tool.description + assert "exact voice_id returned by list_tts_voices" in synthesize_tool.description + assert "en-male" not in synthesize_tool.description + assert "zh-female" not in synthesize_tool.description + + +async def test_clone_tts_voice_returns_persistent_provider_voice_id() -> None: + tts = FakeTTS( + provider_name="elevenlabs", + abilities=["tts", "voice_cloning", "persistent_voice_cloning"], + ) + tool = AudioToolCore(tts_models={"eleven_v3": tts}) + + result = await tool.clone_tts_voice( + name="Product narrator", + reference_audio_files=["first.mp3", "second.wav"], + provider="elevenlabs", + description="Narration voice", + labels={"language": "en"}, + remove_background_noise=True, + model_id="eleven_v3", + ) + + assert result == { + "success": True, + "supported": True, + "voice_id": "persistent-voice", + "name": "Product narrator", + "provider": "elevenlabs", + "persistent": True, + "requires_verification": False, + "model_used": "eleven_v3", + } + assert tts.clone_calls == [ + { + "name": "Product narrator", + "reference_audio_files": ["first.mp3", "second.wav"], + "description": "Narration voice", + "labels": {"language": "en"}, + "remove_background_noise": True, + } + ] + + +async def test_clone_tts_voice_rejects_other_provider_model() -> None: + elevenlabs = FakeTTS( + provider_name="elevenlabs", + abilities=["tts", "persistent_voice_cloning"], + ) + xinference = FakeTTS( + provider_name="xinference", + abilities=["tts", "voice_cloning"], + ) + tool = AudioToolCore(tts_models={"eleven_v3": elevenlabs, "index-tts": xinference}) + + result = await tool.clone_tts_voice( + name="Wrong provider", + reference_audio_files=["reference.wav"], + model_id="index-tts", + ) + + assert result["success"] is False + assert result["supported"] is False + assert "provider is 'elevenlabs'" in result["error"] + assert elevenlabs.clone_calls == [] + assert xinference.clone_calls == [] + + +async def test_clone_tts_voice_reports_missing_elevenlabs_model() -> None: + result = await AudioToolCore().clone_tts_voice( + name="Missing provider", + reference_audio_files=["reference.wav"], + ) + + assert result == { + "success": False, + "supported": False, + "error": "No elevenlabs TTS model is configured", + "provider": "elevenlabs", + "model_used": "default", + } + + +async def test_clone_tts_voice_selects_provider_not_default_tts() -> None: + elevenlabs = FakeTTS( + provider_name="elevenlabs", + abilities=["tts", "persistent_voice_cloning"], + ) + xinference = FakeTTS( + provider_name="xinference", + abilities=["tts", "voice_cloning"], + ) + tool = AudioToolCore( + tts_models={"index-tts": xinference, "eleven_v3": elevenlabs}, + default_tts_model=xinference, + ) + + result = await tool.clone_tts_voice( + name="ElevenLabs voice", + reference_audio_files=["reference.wav"], + ) + + assert result["success"] is True + assert result["provider"] == "elevenlabs" + assert result["model_used"] == "eleven_v3" + assert len(elevenlabs.clone_calls) == 1 + assert xinference.clone_calls == [] + + +def test_clone_tts_voice_tool_exposes_provider_enum() -> None: + elevenlabs_tool = AudioTool( + tts_models={ + "eleven_v3": FakeTTS( + provider_name="elevenlabs", + abilities=["tts", "persistent_voice_cloning"], + ) + } + ) + other_provider_tool = AudioTool( + tts_models={ + "other": FakeTTS( + provider_name="other", + abilities=["tts", "persistent_voice_cloning"], + ) + } + ) + + elevenlabs_tools = { + candidate.name: candidate for candidate in elevenlabs_tool.get_tools() + } + other_tool_names = {candidate.name for candidate in other_provider_tool.get_tools()} + + assert "clone_tts_voice" in elevenlabs_tools + assert "clone_tts_voice" not in other_tool_names + schema = elevenlabs_tools["clone_tts_voice"].args_type().model_json_schema() + assert set(schema["properties"]) == { + "name", + "reference_audio_files", + "provider", + "description", + "labels", + "remove_background_noise", + "model_id", + } + assert schema["properties"]["provider"]["const"] == "elevenlabs" + + +def test_list_tts_voices_tool_exposes_provider_enum() -> None: + audio_tool = AudioTool( + tts_models={ + "eleven_v3": FakeTTS( + provider_name="elevenlabs", + abilities=["tts", "voice_listing"], + ) + } + ) + list_tool = next( + candidate + for candidate in audio_tool.get_tools() + if candidate.name == "list_tts_voices" + ) + + schema = list_tool.args_type().model_json_schema() + assert schema["properties"]["provider"]["const"] == "elevenlabs" + async def test_synthesize_speech_json_merges_default_and_segment_options() -> None: tts = FakeTTS()