From 6ddcb647ece01b3eaa1b01c0aa47ae00b9b3e5ad Mon Sep 17 00:00:00 2001 From: MP Date: Tue, 7 Oct 2025 17:28:25 -0500 Subject: [PATCH] Add: new feature to enable Kokoro TTS voice playback in chat messages --- public/kokoro_openapi.json | 838 +++++++++++++++++++++++++++ src/components/ChatMessage.tsx | 47 +- src/components/TextToSpeech.tsx | 231 ++++---- src/components/VoiceSelector.tsx | 32 + src/components/useAvailableVoices.ts | 96 +++ src/config/config-default.json | 2 + src/i18n/en.json | 27 +- src/pages/Settings.tsx | 136 +++-- src/types/configuration.ts | 4 + 9 files changed, 1211 insertions(+), 202 deletions(-) create mode 100644 public/kokoro_openapi.json create mode 100644 src/components/VoiceSelector.tsx create mode 100644 src/components/useAvailableVoices.ts diff --git a/public/kokoro_openapi.json b/public/kokoro_openapi.json new file mode 100644 index 00000000..684fbdac --- /dev/null +++ b/public/kokoro_openapi.json @@ -0,0 +1,838 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Kokoro TTS API", + "description": "API for text-to-speech generation using Kokoro", + "version": "1.0.0" + }, + "paths": { + "/v1/audio/speech": { + "post": { + "tags": ["OpenAI Compatible TTS"], + "summary": "Create Speech", + "description": "OpenAI-compatible endpoint for text-to-speech", + "operationId": "create_speech_v1_audio_speech_post", + "parameters": [ + { + "name": "x-raw-response", + "in": "header", + "required": false, + "schema": { + "type": "string", + "title": "X-Raw-Response" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OpenAISpeechRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/download/{filename}": { + "get": { + "tags": ["OpenAI Compatible TTS"], + "summary": "Download Audio File", + "description": "Download a generated audio file from temp storage", + "operationId": "download_audio_file_v1_download__filename__get", + "parameters": [ + { + "name": "filename", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Filename" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/models": { + "get": { + "tags": ["OpenAI Compatible TTS"], + "summary": "List Models", + "description": "List all available models", + "operationId": "list_models_v1_models_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + } + } + } + }, + "/v1/models/{model}": { + "get": { + "tags": ["OpenAI Compatible TTS"], + "summary": "Retrieve Model", + "description": "Retrieve a specific model", + "operationId": "retrieve_model_v1_models__model__get", + "parameters": [ + { + "name": "model", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Model" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/audio/voices": { + "get": { + "tags": ["OpenAI Compatible TTS"], + "summary": "List Voices", + "description": "List all available voices for text-to-speech", + "operationId": "list_voices_v1_audio_voices_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + } + } + } + }, + "/v1/audio/voices/combine": { + "post": { + "tags": ["OpenAI Compatible TTS"], + "summary": "Combine Voices", + "description": "Combine multiple voices into a new voice and return the .pt file.\n\nArgs:\n request: Either a string with voices separated by + (e.g. \"voice1+voice2\")\n or a list of voice names to combine\n\nReturns:\n FileResponse with the combined voice .pt file\n\nRaises:\n HTTPException:\n - 400: Invalid request (wrong number of voices, voice not found)\n - 500: Server error (file system issues, combination failed)", + "operationId": "combine_voices_v1_audio_voices_combine_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "title": "Request" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/dev/phonemize": { + "post": { + "tags": ["text processing"], + "summary": "Phonemize Text", + "description": "Convert text to phonemes using Kokoro's quiet mode.\n\nArgs:\n request: Request containing text and language\n\nReturns:\n Phonemes and token IDs", + "operationId": "phonemize_text_dev_phonemize_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PhonemeRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PhonemeResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/dev/generate_from_phonemes": { + "post": { + "tags": ["text processing"], + "summary": "Generate From Phonemes", + "description": "Generate audio directly from phonemes using Kokoro's phoneme format", + "operationId": "generate_from_phonemes_dev_generate_from_phonemes_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateFromPhonemesRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/dev/captioned_speech": { + "post": { + "tags": ["text processing"], + "summary": "Create Captioned Speech", + "description": "Generate audio with word-level timestamps using streaming approach", + "operationId": "create_captioned_speech_dev_captioned_speech_post", + "parameters": [ + { + "name": "x-raw-response", + "in": "header", + "required": false, + "schema": { + "type": "string", + "title": "X-Raw-Response" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CaptionedSpeechRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/debug/threads": { + "get": { + "tags": ["debug"], + "summary": "Get Thread Info", + "operationId": "get_thread_info_debug_threads_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/debug/storage": { + "get": { + "tags": ["debug"], + "summary": "Get Storage Info", + "operationId": "get_storage_info_debug_storage_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/debug/system": { + "get": { + "tags": ["debug"], + "summary": "Get System Info", + "operationId": "get_system_info_debug_system_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/debug/session_pools": { + "get": { + "tags": ["debug"], + "summary": "Get Session Pool Info", + "description": "Get information about ONNX session pools.", + "operationId": "get_session_pool_info_debug_session_pools_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/web/{filename}": { + "get": { + "tags": ["Web Player"], + "summary": "Serve Web File", + "description": "Serve web player static files asynchronously.", + "operationId": "serve_web_file_web__filename__get", + "parameters": [ + { + "name": "filename", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Filename" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/health": { + "get": { + "summary": "Health Check", + "description": "Health check endpoint", + "operationId": "health_check_health_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v1/test": { + "get": { + "summary": "Test Endpoint", + "description": "Test endpoint to verify routing", + "operationId": "test_endpoint_v1_test_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + } + }, + "components": { + "schemas": { + "CaptionedSpeechRequest": { + "properties": { + "model": { + "type": "string", + "title": "Model", + "description": "The model to use for generation. Supported models: tts-1, tts-1-hd, kokoro", + "default": "kokoro" + }, + "input": { + "type": "string", + "title": "Input", + "description": "The text to generate audio for" + }, + "voice": { + "type": "string", + "title": "Voice", + "description": "The voice to use for generation. Can be a base voice or a combined voice name.", + "default": "af_heart" + }, + "response_format": { + "type": "string", + "enum": ["mp3", "opus", "aac", "flac", "wav", "pcm"], + "title": "Response Format", + "description": "The format to return audio in. Supported formats: mp3, opus, flac, wav, pcm. PCM format returns raw 16-bit samples without headers. AAC is not currently supported.", + "default": "mp3" + }, + "speed": { + "type": "number", + "maximum": 4, + "minimum": 0.25, + "title": "Speed", + "description": "The speed of the generated audio. Select a value from 0.25 to 4.0.", + "default": 1 + }, + "stream": { + "type": "boolean", + "title": "Stream", + "description": "If true (default), audio will be streamed as it's generated. Each chunk will be a complete sentence.", + "default": true + }, + "return_timestamps": { + "type": "boolean", + "title": "Return Timestamps", + "description": "If true (default), returns word-level timestamps in the response", + "default": true + }, + "return_download_link": { + "type": "boolean", + "title": "Return Download Link", + "description": "If true, returns a download link in X-Download-Path header after streaming completes", + "default": false + }, + "lang_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Lang Code", + "description": "Optional language code to use for text processing. If not provided, will use first letter of voice name." + }, + "normalization_options": { + "anyOf": [ + { + "$ref": "#/components/schemas/NormalizationOptions" + }, + { + "type": "null" + } + ], + "description": "Options for the normalization system", + "default": { + "normalize": true, + "unit_normalization": false, + "url_normalization": true, + "email_normalization": true, + "optional_pluralization_normalization": true, + "phone_normalization": true + } + } + }, + "type": "object", + "required": ["input"], + "title": "CaptionedSpeechRequest", + "description": "Request schema for captioned speech endpoint" + }, + "GenerateFromPhonemesRequest": { + "properties": { + "phonemes": { + "type": "string", + "title": "Phonemes", + "description": "Phoneme string to synthesize" + }, + "voice": { + "type": "string", + "title": "Voice", + "description": "Voice ID to use for generation" + } + }, + "type": "object", + "required": ["phonemes", "voice"], + "title": "GenerateFromPhonemesRequest", + "description": "Simple request for phoneme-to-speech generation" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "NormalizationOptions": { + "properties": { + "normalize": { + "type": "boolean", + "title": "Normalize", + "description": "Normalizes input text to make it easier for the model to say", + "default": true + }, + "unit_normalization": { + "type": "boolean", + "title": "Unit Normalization", + "description": "Transforms units like 10KB to 10 kilobytes", + "default": false + }, + "url_normalization": { + "type": "boolean", + "title": "Url Normalization", + "description": "Changes urls so they can be properly pronounced by kokoro", + "default": true + }, + "email_normalization": { + "type": "boolean", + "title": "Email Normalization", + "description": "Changes emails so they can be properly pronouced by kokoro", + "default": true + }, + "optional_pluralization_normalization": { + "type": "boolean", + "title": "Optional Pluralization Normalization", + "description": "Replaces (s) with s so some words get pronounced correctly", + "default": true + }, + "phone_normalization": { + "type": "boolean", + "title": "Phone Normalization", + "description": "Changes phone numbers so they can be properly pronouced by kokoro", + "default": true + } + }, + "type": "object", + "title": "NormalizationOptions", + "description": "Options for the normalization system" + }, + "OpenAISpeechRequest": { + "properties": { + "model": { + "type": "string", + "title": "Model", + "description": "The model to use for generation. Supported models: tts-1, tts-1-hd, kokoro", + "default": "kokoro" + }, + "input": { + "type": "string", + "title": "Input", + "description": "The text to generate audio for" + }, + "voice": { + "type": "string", + "title": "Voice", + "description": "The voice to use for generation. Can be a base voice or a combined voice name.", + "default": "af_heart" + }, + "response_format": { + "type": "string", + "enum": ["mp3", "opus", "aac", "flac", "wav", "pcm"], + "title": "Response Format", + "description": "The format to return audio in. Supported formats: mp3, opus, flac, wav, pcm. PCM format returns raw 16-bit samples without headers. AAC is not currently supported.", + "default": "mp3" + }, + "download_format": { + "anyOf": [ + { + "type": "string", + "enum": ["mp3", "opus", "aac", "flac", "wav", "pcm"] + }, + { + "type": "null" + } + ], + "title": "Download Format", + "description": "Optional different format for the final download. If not provided, uses response_format." + }, + "speed": { + "type": "number", + "maximum": 4, + "minimum": 0.25, + "title": "Speed", + "description": "The speed of the generated audio. Select a value from 0.25 to 4.0.", + "default": 1 + }, + "stream": { + "type": "boolean", + "title": "Stream", + "description": "If true (default), audio will be streamed as it's generated. Each chunk will be a complete sentence.", + "default": true + }, + "return_download_link": { + "type": "boolean", + "title": "Return Download Link", + "description": "If true, returns a download link in X-Download-Path header after streaming completes", + "default": false + }, + "lang_code": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Lang Code", + "description": "Optional language code to use for text processing. If not provided, will use first letter of voice name." + }, + "normalization_options": { + "anyOf": [ + { + "$ref": "#/components/schemas/NormalizationOptions" + }, + { + "type": "null" + } + ], + "description": "Options for the normalization system", + "default": { + "normalize": true, + "unit_normalization": false, + "url_normalization": true, + "email_normalization": true, + "optional_pluralization_normalization": true, + "phone_normalization": true + } + } + }, + "type": "object", + "required": ["input"], + "title": "OpenAISpeechRequest", + "description": "Request schema for OpenAI-compatible speech endpoint" + }, + "PhonemeRequest": { + "properties": { + "text": { + "type": "string", + "title": "Text" + }, + "language": { + "type": "string", + "title": "Language", + "default": "a" + } + }, + "type": "object", + "required": ["text"], + "title": "PhonemeRequest" + }, + "PhonemeResponse": { + "properties": { + "phonemes": { + "type": "string", + "title": "Phonemes" + }, + "tokens": { + "items": { + "type": "integer" + }, + "type": "array", + "title": "Tokens" + } + }, + "type": "object", + "required": ["phonemes", "tokens"], + "title": "PhonemeResponse" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + }, + "type": "object", + "required": ["loc", "msg", "type"], + "title": "ValidationError" + } + } + } +} diff --git a/src/components/ChatMessage.tsx b/src/components/ChatMessage.tsx index b4c91277..9cf67440 100644 --- a/src/components/ChatMessage.tsx +++ b/src/components/ChatMessage.tsx @@ -36,10 +36,12 @@ import ChatInputExtraContextItem from './ChatInputExtraContextItem'; import { IntlIconButton } from './common'; import { DropzoneArea } from './DropzoneArea'; import MarkdownDisplay from './MarkdownDisplay'; -import TextToSpeech, { - getSpeechSynthesisVoiceByName, - IS_SPEECH_SYNTHESIS_SUPPORTED, -} from './TextToSpeech'; +import TextToSpeech from '../components/TextToSpeech'; +import { + UnifiedVoice, + useAvailableVoices, +} from '../components/useAvailableVoices'; +const IS_SPEECH_SYNTHESIS_SUPPORTED = 'speechSynthesis' in window; interface SplitMessage { content: PendingMessage['content']; @@ -356,7 +358,6 @@ export default memo(function ChatMessage({ {/* play message */} @@ -538,25 +539,49 @@ const ThinkingSection = memo(function ThinkingSection({ interface PlayButtonProps { className?: string; - disabled?: boolean; text: string; } const PlayButton = memo(function PlayButton({ className, - disabled, text, }: PlayButtonProps) { const { t } = useTranslation(); const { - config: { ttsVoice, ttsPitch, ttsRate, ttsVolume }, + config: { + ttsVoice, + ttsPitch, + ttsRate, + ttsVolume, + ttsServerIp, + ttsServerPort, + }, } = useAppContext(); + const { voices } = useAvailableVoices(ttsServerIp, ttsServerPort); + const selectedVoice = useMemo( + () => voices.find((v: UnifiedVoice) => v.id === ttsVoice) || null, + [voices, ttsVoice] + ); + + const isDisabled = useMemo(() => { + if (!text) return true; + if (selectedVoice?.type === 'kokoro') { + return !ttsServerIp || !ttsServerPort; + } + return !IS_SPEECH_SYNTHESIS_SUPPORTED; + }, [text, selectedVoice?.type, ttsServerIp, ttsServerPort]); + return ( {({ isPlaying, play, stop }) => ( @@ -564,7 +589,7 @@ const PlayButton = memo(function PlayButton({ - speechSynthesis - ?.getVoices() - .filter((voice) => voice.localService) - .sort((a, b) => { - // Default voice first - if (a.default !== b.default) return a.default ? -1 : 1; - - // Popular languages on top - const aRank = popularLanguages.indexOf(a.lang.substring(0, 2)); - const bRank = popularLanguages.indexOf(b.lang.substring(0, 2)); - if (aRank !== bRank) { - const aEffectiveRank = aRank === -1 ? Infinity : aRank; - const bEffectiveRank = bRank === -1 ? Infinity : bRank; - return aEffectiveRank - bEffectiveRank; - } - - // Sort by language and name (alphabetically) - return a.lang.localeCompare(b.lang) || a.name.localeCompare(b.name); - }) || []; -export function getSpeechSynthesisVoiceByName(name: string) { - return getSpeechSynthesisVoices().find( - (voice) => `${voice.name} (${voice.lang})` === name - ); -} +import { UnifiedVoice } from './useAvailableVoices'; interface TextToSpeechProps { text: string; - voice?: SpeechSynthesisVoice; + selectedVoice: UnifiedVoice | null; pitch?: number; rate?: number; volume?: number; + serverConfig?: TTSServerConfig; } interface TextToSpeechState { @@ -66,117 +27,119 @@ interface TextToSpeechState { stop: () => void; } +interface TTSServerConfig { + serverIp: string; + serverPort: string; +} + const useTextToSpeech = ({ text, - voice = getSpeechSynthesisVoices()[0], + selectedVoice, pitch = 1, rate = 1, volume = 1, -}: TextToSpeechProps) => { + serverConfig, +}: TextToSpeechProps & { + serverConfig?: TTSServerConfig; +}): TextToSpeechState => { const [isPlaying, setIsPlaying] = useState(false); - const utteranceRef = useRef(null); - - useEffect(() => { - if (!IS_SPEECH_SYNTHESIS_SUPPORTED) { - console.warn('Speech synthesis not supported'); - return; - } - if (!text) { - console.warn('No text provided'); - return; - } + const audioRef = useRef(null); - // Clean up previous utterance - if (utteranceRef.current) { - utteranceRef.current.onstart = null; - utteranceRef.current.onend = null; - utteranceRef.current.onerror = null; - } - - const utterance = new window.SpeechSynthesisUtterance(text); - - utterance.voice = voice; - utterance.pitch = pitch; - utterance.rate = rate; - utterance.volume = volume; - - // Event handlers - utterance.onstart = () => { - setIsPlaying(true); - }; - - utterance.onend = () => { - setIsPlaying(false); - }; + const play = useCallback(async () => { + if (!selectedVoice) return; - utterance.onerror = (event) => { - console.error('Speech synthesis error: ', event.error); - setIsPlaying(false); - }; - - utteranceRef.current = utterance; - - return () => { - speechSynthesis.cancel(); - if (utteranceRef.current === utterance) { - utteranceRef.current.onstart = null; - utteranceRef.current.onend = null; - utteranceRef.current.onerror = null; - utteranceRef.current = null; + if (selectedVoice.type === 'kokoro') { + if (!serverConfig?.serverIp || !serverConfig?.serverPort) { + console.error('TTS server configuration is missing'); + return; } - }; - }, [pitch, rate, text, voice, volume]); - - const play = useCallback(() => { - if (!IS_SPEECH_SYNTHESIS_SUPPORTED) { - console.warn('Speech synthesis not supported'); - return; - } - speechSynthesis.cancel(); - if (utteranceRef.current) { try { - speechSynthesis.speak(utteranceRef.current); - } catch (error) { - console.error('Speech synthesis error:', error); - setIsPlaying(false); + const response = await fetch( + `http://${serverConfig.serverIp}:${serverConfig.serverPort}/v1/audio/speech`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: 'kokoro', + input: text, + voice: selectedVoice.raw, // ✅ string + response_format: 'mp3', + speed: rate, + }), + } + ); + + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + + const audio = new Audio(url); + audio.volume = volume; + audioRef.current = audio; + + audio.onplay = () => setIsPlaying(true); + audio.onended = () => { + setIsPlaying(false); + URL.revokeObjectURL(url); + }; + await audio.play(); + } catch (err) { + console.error('Kokoro TTS failed:', err); } + } else if (selectedVoice.type === 'browser') { + const utterance = new SpeechSynthesisUtterance(text); + utterance.voice = selectedVoice.raw; // ✅ SpeechSynthesisVoice + utterance.pitch = pitch; + utterance.rate = rate; + utterance.volume = volume; + + utterance.onstart = () => setIsPlaying(true); + utterance.onend = () => setIsPlaying(false); + utterance.onerror = () => setIsPlaying(false); + + speechSynthesis.speak(utterance); } - }, []); + }, [ + text, + selectedVoice, + pitch, + rate, + volume, + serverConfig?.serverIp, + serverConfig?.serverPort, + ]); const stop = useCallback(() => { - speechSynthesis.cancel(); setIsPlaying(false); - }, []); - return { - isPlaying, - play, - stop, - }; + // Stop browser speech synthesis if active + if (selectedVoice?.type === 'browser') { + speechSynthesis.cancel(); + } + + // Stop and cleanup Kokoro audio if active + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current.currentTime = 0; + audioRef.current.src = ''; + audioRef.current = null; + } + }, [selectedVoice?.type]); + + return { isPlaying, play, stop }; }; const TextToSpeech = forwardRef< TextToSpeechState, TextToSpeechProps & { children: (props: TextToSpeechState) => ReactNode } ->(({ children, text, voice, pitch, rate, volume }, ref) => { - const { isPlaying, play, stop } = useTextToSpeech({ - text, - voice, - pitch, - rate, - volume, - }); - - useImperativeHandle( - ref, - () => ({ - isPlaying, - play, - stop, - }), - [isPlaying, play, stop] - ); +>(({ children, ...props }, ref) => { + const { isPlaying, play, stop } = useTextToSpeech(props); + + useImperativeHandle(ref, () => ({ isPlaying, play, stop }), [ + isPlaying, + play, + stop, + ]); return {children({ isPlaying, play, stop })}; }); diff --git a/src/components/VoiceSelector.tsx b/src/components/VoiceSelector.tsx new file mode 100644 index 00000000..0adca99d --- /dev/null +++ b/src/components/VoiceSelector.tsx @@ -0,0 +1,32 @@ +// VoiceSelector.tsx + +import { UnifiedVoice } from './useAvailableVoices'; + +interface VoiceSelectorProps { + voices: UnifiedVoice[]; + selected: UnifiedVoice | null; + onChange: (voice: UnifiedVoice) => void; +} + +const VoiceSelector = ({ voices, selected, onChange }: VoiceSelectorProps) => { + return ( + + ); +}; + +export default VoiceSelector; diff --git a/src/components/useAvailableVoices.ts b/src/components/useAvailableVoices.ts new file mode 100644 index 00000000..e0fa85a4 --- /dev/null +++ b/src/components/useAvailableVoices.ts @@ -0,0 +1,96 @@ +// useAvailableVoices.ts + +import { useEffect, useState, useCallback } from 'react'; + +export type VoiceType = 'kokoro' | 'browser'; + +export type UnifiedVoice = + | { + type: 'browser'; + id: string; + label: string; + raw: SpeechSynthesisVoice; + } + | { + type: 'kokoro'; + id: string; + label: string; + raw: string; + }; + +export function getKokoroApiUrl(serverIp: string, serverPort: string): string { + if (!serverIp || !serverPort) return ''; + return `http://${serverIp}:${serverPort}/v1/audio/voices`; +} + +interface UseAvailableVoicesResult { + voices: UnifiedVoice[]; + refreshVoices: () => void; + isLoading: boolean; +} + +export function useAvailableVoices( + serverIp?: string, + serverPort?: string +): UseAvailableVoicesResult { + const [voices, setVoices] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [refreshTrigger, setRefreshTrigger] = useState(0); + + const loadVoices = useCallback(async () => { + setIsLoading(true); + try { + // Always start with browser voices + const synthVoices = window.speechSynthesis.getVoices(); + const browserVoices: UnifiedVoice[] = synthVoices.map((v) => ({ + type: 'browser', + id: `browser-${v.name}-${v.lang}`, + label: `🌐 ${v.name} (${v.lang})`, + raw: v, + })); + + // Only try to fetch Kokoro voices if server settings are provided + const kokoroVoices: UnifiedVoice[] = []; + const apiUrl = getKokoroApiUrl(serverIp || '', serverPort || ''); + + if (apiUrl) { + try { + const res = await fetch(apiUrl); + if (res.ok) { + const data = await res.json(); + kokoroVoices.push( + ...data.voices.map((v: string) => ({ + type: 'kokoro', + id: `kokoro-${v}`, + label: `🎵 Kokoro: ${v}`, + raw: v, + })) + ); + } + } catch (err) { + console.warn('Could not load Kokoro voices:', err); + } + } + + setVoices([...browserVoices, ...kokoroVoices]); + } catch (err) { + console.error('Error loading voices:', err); + setVoices([]); // Reset to empty array on error + } finally { + setIsLoading(false); + } + }, [serverIp, serverPort]); + + useEffect(() => { + if (window.speechSynthesis.onvoiceschanged !== undefined) { + window.speechSynthesis.onvoiceschanged = loadVoices; + } + loadVoices(); + }, [loadVoices, refreshTrigger]); + + const refreshVoices = useCallback(() => { + setRefreshTrigger((prev) => prev + 1); + }, []); + + return { voices, refreshVoices, isLoading }; +} diff --git a/src/config/config-default.json b/src/config/config-default.json index 74c4eeb0..b9e53cb7 100644 --- a/src/config/config-default.json +++ b/src/config/config-default.json @@ -8,6 +8,8 @@ "showRawUserMessage": false, "showRawAssistantMessage": false, "pasteLongTextToFileLen": 10000, + "ttsServerIp": "localhost", + "ttsServerPort": "8888", "pdfAsImage": false, "showTokensPerSecond": false, "showThoughtInProgress": false, diff --git a/src/i18n/en.json b/src/i18n/en.json index b68f3f44..ba0d9542 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -19,7 +19,9 @@ }, "footer": { "ariaDesc": "Disclaimer", - "disclaimer": "The content is generated by AI. It may be inaccurate." + "disclaimer": "This is an experimental prototype. Use with caution.", + "version": "Version: {{version}}", + "storageNote": "Settings are stored locally in your browser" }, "sidebar": { "groups": { @@ -183,10 +185,18 @@ "importExport": "Import/Export", "advanced": "Advanced" }, + "actionButtons": { + "fetchModels": "Fetch Models", + "refreshVoices": "Fetch Voice List", + "saveBtnLabel": "Save", + "cancelBtnLabel": "Close", + "resetBtnLabel": "Reset" + }, "sections": { "inferenceProvider": "Inference Provider", "userInterface": "User Interface", "textToSpeech": "Text to Speech", + "ttsServer": "TTS Server Configuration", "chat": "Chat", "performance": "Performance", "reasoning": "Reasoning", @@ -357,6 +367,14 @@ "label": "Rate", "note": "The speed at which the utterance will be spoken at." }, + "ttsServerIp": { + "label": "Server IP", + "note": "The IP address of the TTS server" + }, + "ttsServerPort": { + "label": "Server Port", + "note": "The port number of the TTS server" + }, "ttsVoice": { "label": "Voice", "note": "The voice that will be used to speak the utterance." @@ -421,12 +439,7 @@ "technicalDemoSectionTitle": "Technical Demo", "importDemoConversationBtnLabel": "Import demo conversation" }, - "actionButtons": { - "fetchModels": "Fetch Models", - "saveBtnLabel": "Save", - "cancelBtnLabel": "Close", - "resetBtnLabel": "Reset" - }, + "footer": { "version": "Application Version: {{version}}", "storageNote": "Settings are saved in browser's localStorage." diff --git a/src/pages/Settings.tsx b/src/pages/Settings.tsx index ef748b21..3d9aed84 100644 --- a/src/pages/Settings.tsx +++ b/src/pages/Settings.tsx @@ -22,6 +22,7 @@ import { LuMonitor, LuRefreshCw, LuRocket, + LuServer, LuSettings, LuSpeech, LuVolume2, @@ -41,11 +42,11 @@ import { import { ImportExportComponent } from '../components/settings/ImportExportComponent'; import { PresetManager } from '../components/settings/PresetManager'; import { ThemeController } from '../components/settings/ThemeController'; -import TextToSpeech, { - getSpeechSynthesisVoiceByName, - getSpeechSynthesisVoices, - IS_SPEECH_SYNTHESIS_SUPPORTED, -} from '../components/TextToSpeech'; +import TextToSpeech from '../components/TextToSpeech'; +import { + useAvailableVoices, + UnifiedVoice, +} from '../components/useAvailableVoices'; import { CONFIG_DEFAULT, INFERENCE_PROVIDERS } from '../config'; import { useAppContext } from '../context/app'; import { useChatContext } from '../context/chat'; @@ -130,8 +131,21 @@ function toDropdown( function getSettingTabsConfiguration( config: Configuration, models: InferenceApiModel[], - t: ReturnType['t'] + t: ReturnType['t'], + voices: UnifiedVoice[], + refreshVoices: () => void, + isLoading: boolean ): SettingTab[] { + const unifiedVoiceOptions: { value: string; label: string }[] = ( + voices || [] + ).map((voice: UnifiedVoice) => ({ + value: voice.id, + label: voice.label, + })); + + const selectedVoice: UnifiedVoice | null = + voices.find((v: UnifiedVoice) => v.id === config.ttsVoice) ?? null; + return [ /* General */ { @@ -217,67 +231,67 @@ function getSettingTabsConfiguration( ), fields: [ - /* Text to Speech */ toSection( t('settings.sections.textToSpeech'), ), - toDropdown( - 'ttsVoice', - !IS_SPEECH_SYNTHESIS_SUPPORTED - ? [] - : getSpeechSynthesisVoices().map((voice) => ({ - value: `${voice.name} (${voice.lang})`, - label: `${voice.name} (${voice.lang})`, - })), - true - ), - toInput( - SettingInputType.RANGE_INPUT, - 'ttsPitch', - !IS_SPEECH_SYNTHESIS_SUPPORTED, - { - min: 0, - max: 2, - step: 0.5, - } - ), - toInput( - SettingInputType.RANGE_INPUT, - 'ttsRate', - !IS_SPEECH_SYNTHESIS_SUPPORTED, - { - min: 0.5, - max: 2, - step: 0.5, - } - ), - toInput( - SettingInputType.RANGE_INPUT, - 'ttsVolume', - !IS_SPEECH_SYNTHESIS_SUPPORTED, - { - min: 0, - max: 1, - step: 0.25, - } + toSection( + t('settings.sections.ttsServer'), + ), + toInput(SettingInputType.SHORT_INPUT, 'ttsServerIp'), + toInput(SettingInputType.SHORT_INPUT, 'ttsServerPort'), + { + type: SettingInputType.CUSTOM, + key: 'custom', + component: () => ( + + ), + }, + toDropdown('ttsVoice', unifiedVoiceOptions, true), + toInput(SettingInputType.RANGE_INPUT, 'ttsPitch', false, { + min: 0, + max: 2, + step: 0.5, + }), + toInput(SettingInputType.RANGE_INPUT, 'ttsRate', false, { + min: 0.5, + max: 2, + step: 0.5, + }), + toInput(SettingInputType.RANGE_INPUT, 'ttsVolume', false, { + min: 0, + max: 1, + step: 0.25, + }), { type: SettingInputType.CUSTOM, key: 'custom', // dummy key, won't be used component: () => ( {({ isPlaying, play, stop }) => (