diff --git a/CMakeLists.txt b/CMakeLists.txt index f0da03d..35b8972 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,10 @@ cmake_minimum_required(VERSION 3.15) project(arbiterAI VERSION 0.1.13 LANGUAGES C CXX) option(ARBITERAI_ENABLE_LLAMA "Enable local llama.cpp model runtime support" ON) +option(ARBITERAI_ENABLE_WHISPER "Enable local whisper.cpp speech-to-text support" OFF) +option(ARBITERAI_ENABLE_STABLE_DIFFUSION "Enable local stable-diffusion.cpp image generation support" OFF) +option(ARBITERAI_ENABLE_VIBEVOICE "Enable local vibevoice.cpp text-to-speech support" OFF) +option(ARBITERAI_ENABLE_ORPHEUS "Enable local Orpheus TTS via a dlopen'd chatllm.cpp engine library" OFF) # Read llama-cpp build number from our custom vcpkg port if(ARBITERAI_ENABLE_LLAMA) @@ -51,6 +55,18 @@ find_package(cxxopts CONFIG REQUIRED) if(ARBITERAI_ENABLE_LLAMA) find_package(llama CONFIG REQUIRED) endif() +if(ARBITERAI_ENABLE_WHISPER) + find_package(whisper CONFIG REQUIRED) +endif() +if(ARBITERAI_ENABLE_STABLE_DIFFUSION) + find_package(stable-diffusion CONFIG REQUIRED) + find_package(ZLIB REQUIRED) +endif() +if(ARBITERAI_ENABLE_VIBEVOICE) + # vibevoice.cpp installs a lib + header but no package config. + find_path(VIBEVOICE_INCLUDE_DIR NAMES vibevoice_capi.h REQUIRED) + find_library(VIBEVOICE_LIBRARY NAMES vibevoice REQUIRED) +endif() find_package(libgit2 CONFIG REQUIRED) find_package(Threads REQUIRED) find_package(spdlog CONFIG REQUIRED) @@ -108,6 +124,35 @@ if(ARBITERAI_ENABLE_LLAMA) ) endif() +if(ARBITERAI_ENABLE_WHISPER) + list(APPEND arbiterai_src + ./src/arbiterAI/providers/whisper.h + ./src/arbiterAI/providers/whisper.cpp + ) +endif() + +if(ARBITERAI_ENABLE_STABLE_DIFFUSION) + list(APPEND arbiterai_src + ./src/arbiterAI/providers/stableDiffusion.h + ./src/arbiterAI/providers/stableDiffusion.cpp + ) +endif() + +if(ARBITERAI_ENABLE_VIBEVOICE) + list(APPEND arbiterai_src + ./src/arbiterAI/providers/vibevoice.h + ./src/arbiterAI/providers/vibevoice.cpp + ) +endif() + +if(ARBITERAI_ENABLE_ORPHEUS) + # dlopen-based: needs only libdl (already linked via CMAKE_DL_LIBS). + list(APPEND arbiterai_src + ./src/arbiterAI/providers/orpheus.h + ./src/arbiterAI/providers/orpheus.cpp + ) +endif() + # Add library add_library(arbiterai ${arbiterai_src} @@ -130,11 +175,32 @@ if(ARBITERAI_ENABLE_LLAMA) target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_LLAMA=1) endif() +if(ARBITERAI_ENABLE_WHISPER) + target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_WHISPER=1) +endif() + +if(ARBITERAI_ENABLE_STABLE_DIFFUSION) + target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_STABLE_DIFFUSION=1) +endif() + +if(ARBITERAI_ENABLE_VIBEVOICE) + target_include_directories(arbiterai PRIVATE ${VIBEVOICE_INCLUDE_DIR}) + target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_VIBEVOICE=1) +endif() + +if(ARBITERAI_ENABLE_ORPHEUS) + target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_ORPHEUS=1) +endif() + target_link_libraries(arbiterai PUBLIC cpr::cpr nlohmann_json::nlohmann_json $<$:llama> + $<$:whisper> + $<$:stable-diffusion> + $<$:ZLIB::ZLIB> + $<$:${VIBEVOICE_LIBRARY}> libgit2::libgit2package PRIVATE spdlog::spdlog @@ -170,6 +236,30 @@ target_link_libraries(arbiterai tests/inferenceSchedulerTests.cpp ) endif() + + if(ARBITERAI_ENABLE_WHISPER) + target_sources(arbiterai_tests PRIVATE + tests/whisperProviderTests.cpp + ) + endif() + + if(ARBITERAI_ENABLE_STABLE_DIFFUSION) + target_sources(arbiterai_tests PRIVATE + tests/stableDiffusionProviderTests.cpp + ) + endif() + + if(ARBITERAI_ENABLE_VIBEVOICE) + target_sources(arbiterai_tests PRIVATE + tests/vibevoiceProviderTests.cpp + ) + endif() + + if(ARBITERAI_ENABLE_ORPHEUS) + target_sources(arbiterai_tests PRIVATE + tests/orpheusProviderTests.cpp + ) + endif() target_link_libraries(arbiterai_tests PRIVATE diff --git a/docs/feature_request.md b/docs/feature_request.md index 0404a89..4cb1cdd 100644 --- a/docs/feature_request.md +++ b/docs/feature_request.md @@ -2,4 +2,16 @@ - Allow for configurable devices that can be used and well as vram limits - Add support to just unload a model or all models - Need configuration for listening host/port, default 9010 -- Allow model config to be inject through the api, add support to the server so a client can add a model config. \ No newline at end of file +- Allow model config to be inject through the api, add support to the server so a client can add a model config. +- [DONE] Wire ModelDownloader/StorageManager for whisper and stable-diffusion providers so weights auto-download from an injected config's variants[].download.url on load (same as llama GGUFs) — today they only resolve a local file_path. + → `BaseProvider::resolveDownloadableModelFile()` downloads every configured variant file (incl. multi-file bundles like vibevoice tts+tokenizer+voice) to the StorageManager models dir when absent; whisper/sd/vibevoice/orpheus delegate to it. +- [DONE] Record modality-aware speed telemetry (STT: audio-seconds/realtime-factor; image: images and steps/sec) into InferenceStats and expose on /api/stats/history keyed by (model, variant, hardware), so clients can benchmark/normalize non-LLM speed. + → InferenceStats gained modality/imagesGenerated/imageSteps/audioSeconds/audioCharacters/realtimeFactor/stepsPerSecond/cost, computed at dispatch and emitted per-entry on /api/stats/history (client groups by model+variant+hardware). +- [DONE] Include the computed audio duration in the /v1/audio/transcriptions response body. + → `duration` is included in the JSON response. +- [DONE] Extend model_config schema with optional whisper_options / sd_options blocks for engine-specific knobs (language, steps, sampler, vae, etc.) without loosening additionalProperties. + → schema + ModelInfo.whisperOptions/sdOptions parsing; whisper applies language/translate defaults, sd applies steps/width/height/cfg_scale defaults. +- [OPEN] See docs/tasks/multimodal_benchmark_integration.md for the full benchmark-integration contract. + → The speed-telemetry foundation (above) is in place; the full benchmark runner/endpoints per that contract remain. +- [DONE] BUG (server-wide crash): invalid UTF-8 in a model's generated output aborts the whole server. When a generation clips a multi-byte UTF-8 character at a token/length boundary (or emits a llama.cpp byte-fallback token), arbiterResponse.text is not valid UTF-8; serializing it with nlohmann::json's default (strict) handler throws json.exception.type_error.316 ("incomplete UTF-8 string; last byte: 0x9E"). The exception is uncaught → terminate() → SIGABRT (code=dumped, status=6/ABRT), taking down every in-flight request. Reproduced twice on ai-minisforum with model qwythos-9b-claude-mythos-5-1m during HumanEval (2026-07-18 10:32 and 11:26 EDT). The model text enters JSON at src/server/routes.cpp:2117 (messageJson["content"]=arbiterResponse.text) and dies at the response .dump(). Fix: (1) serialize every response body that carries model text with the replace error handler — j.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) — on both the non-streaming path and the streaming deltas (routes.cpp ~1804/1847), and wrap serialization in try/catch so no model output can ever abort the process; (2) root cause — buffer incomplete multi-byte UTF-8 sequences across token boundaries in the llama provider's detokenization so partial characters are never emitted. + → (1) `safeDump()` (error_handler_t::replace + try/catch) now serializes all model-text response bodies (streaming deltas + non-streaming + transcription/image). (2) `Utf8StreamBuffer` in the llama provider's two generation loops holds an incomplete multi-byte tail until the next token completes it (verified: café/emoji boundary cases). \ No newline at end of file diff --git a/examples/multimodal_models.json b/examples/multimodal_models.json new file mode 100644 index 0000000..cf53044 --- /dev/null +++ b/examples/multimodal_models.json @@ -0,0 +1,90 @@ +{ + "schema_version": "1.1.0", + "models": [ + { + "model": "mock-transcribe", + "provider": "mock", + "mode": "transcription", + "ranking": 1, + "pricing": { + "audio_input_cost_per_second": 0.0001 + } + }, + { + "model": "mock-speech", + "provider": "mock", + "mode": "speech", + "ranking": 1, + "pricing": { + "audio_output_cost_per_character": 0.00003 + } + }, + { + "model": "mock-image", + "provider": "mock", + "mode": "image", + "ranking": 1, + "pricing": { + "image_cost": 0.04 + } + }, + { + "model": "whisper-1", + "provider": "openai", + "mode": "transcription", + "api_base": "https://api.openai.com/v1", + "ranking": 50, + "pricing": { + "audio_input_cost_per_second": 0.0001 + } + }, + { + "model": "tts-1", + "provider": "openai", + "mode": "speech", + "api_base": "https://api.openai.com/v1", + "ranking": 50, + "pricing": { + "audio_output_cost_per_character": 0.000015 + } + }, + { + "model": "dall-e-3", + "provider": "openai", + "mode": "image", + "api_base": "https://api.openai.com/v1", + "ranking": 50, + "pricing": { + "image_cost": 0.04 + } + }, + { + "model": "whisper-base-en", + "provider": "whisper", + "mode": "transcription", + "ranking": 50, + "file_path": "/srv/models/whisper/ggml-base.en.bin" + }, + { + "model": "sd-v1-5", + "provider": "stable-diffusion", + "mode": "image", + "ranking": 50, + "file_path": "/srv/models/sd/v1-5-pruned-emaonly.safetensors" + }, + { + "model": "vibevoice-0.5b", + "provider": "vibevoice", + "mode": "speech", + "ranking": 50, + "file_path": "/srv/models/vibevoice/vibevoice-realtime-0.5B-q8_0.gguf" + }, + { + "model": "orpheus-3b", + "provider": "orpheus", + "mode": "speech", + "ranking": 40, + "file_path": "/srv/models/orpheus/orpheus-3b-q8_0.bin" + } + ] +} diff --git a/schemas/model_config.schema.json b/schemas/model_config.schema.json index 50d0914..d0e8af9 100644 --- a/schemas/model_config.schema.json +++ b/schemas/model_config.schema.json @@ -30,7 +30,7 @@ "provider": { "type": "string", "description": "Model provider/service", - "enum": ["anthropic", "llama", "openai", "deepseek", "openrouter", "mock"] + "enum": ["anthropic", "llama", "openai", "deepseek", "openrouter", "mock", "whisper", "stable-diffusion", "vibevoice", "orpheus"] }, "ranking": { "type": "integer", @@ -41,7 +41,8 @@ }, "mode": { "type": "string", - "description": "Operation mode", + "description": "Model modality. Routes requests to the matching endpoint/method: chat completions, embeddings, speech-to-text (transcription), text-to-speech (speech), or image generation (image).", + "enum": ["chat", "embedding", "transcription", "speech", "image"], "default": "chat" }, "api_base": { @@ -125,7 +126,7 @@ }, "pricing": { "type": "object", - "description": "Token pricing information", + "description": "Pricing information. Token costs apply to chat/embedding models; the modality-specific costs apply to image/audio models.", "properties": { "prompt_token_cost": { "type": "number", @@ -134,6 +135,21 @@ "completion_token_cost": { "type": "number", "minimum": 0 + }, + "image_cost": { + "type": "number", + "description": "Cost per generated image (mode: image)", + "minimum": 0 + }, + "audio_input_cost_per_second": { + "type": "number", + "description": "Cost per second of transcribed audio (mode: transcription)", + "minimum": 0 + }, + "audio_output_cost_per_character": { + "type": "number", + "description": "Cost per character of synthesized speech (mode: speech)", + "minimum": 0 } } }, @@ -305,6 +321,56 @@ "description": "Output format produced by the model. When set (e.g. 'harmony'), the server converts the model's native output to standard OpenAI API format so clients don't need to understand the model's native format.", "enum": ["", "harmony"], "default": "" + }, + "whisper_options": { + "type": "object", + "description": "Engine-specific defaults for the whisper (transcription) provider. Applied when the request omits the corresponding field.", + "properties": { + "language": { + "type": "string", + "description": "Default language hint (ISO-639-1, or 'auto' to detect)" + }, + "translate": { + "type": "boolean", + "description": "Translate to English instead of transcribing in the source language" + } + }, + "additionalProperties": false + }, + "sd_options": { + "type": "object", + "description": "Engine-specific defaults for the stable-diffusion (image) provider. Applied when the request omits the corresponding field.", + "properties": { + "steps": { + "type": "integer", + "description": "Default number of diffusion sample steps", + "minimum": 1 + }, + "cfg_scale": { + "type": "number", + "description": "Default classifier-free guidance scale", + "minimum": 0 + }, + "sampler": { + "type": "string", + "description": "Sampling method (e.g. euler_a, dpm++2m, dpm++2mv2)" + }, + "width": { + "type": "integer", + "description": "Default image width in pixels", + "minimum": 64 + }, + "height": { + "type": "integer", + "description": "Default image height in pixels", + "minimum": 64 + }, + "vae_path": { + "type": "string", + "description": "Optional path to an external VAE file" + } + }, + "additionalProperties": false } } } diff --git a/src/arbiterAI/arbiterAI.cpp b/src/arbiterAI/arbiterAI.cpp index 4164225..eeb2a8e 100644 --- a/src/arbiterAI/arbiterAI.cpp +++ b/src/arbiterAI/arbiterAI.cpp @@ -19,6 +19,22 @@ #include "arbiterAI/providers/llama.h" #endif +#ifdef ARBITERAI_ENABLE_WHISPER +#include "arbiterAI/providers/whisper.h" +#endif + +#ifdef ARBITERAI_ENABLE_STABLE_DIFFUSION +#include "arbiterAI/providers/stableDiffusion.h" +#endif + +#ifdef ARBITERAI_ENABLE_VIBEVOICE +#include "arbiterAI/providers/vibevoice.h" +#endif + +#ifdef ARBITERAI_ENABLE_ORPHEUS +#include "arbiterAI/providers/orpheus.h" +#endif + #include namespace arbiterAI @@ -140,6 +156,30 @@ std::unique_ptr createProvider(const std::string &provider) { return std::make_unique(); } +#endif +#ifdef ARBITERAI_ENABLE_WHISPER + else if(provider=="whisper") + { + return std::make_unique(); + } +#endif +#ifdef ARBITERAI_ENABLE_STABLE_DIFFUSION + else if(provider=="stable-diffusion") + { + return std::make_unique(); + } +#endif +#ifdef ARBITERAI_ENABLE_VIBEVOICE + else if(provider=="vibevoice") + { + return std::make_unique(); + } +#endif +#ifdef ARBITERAI_ENABLE_ORPHEUS + else if(provider=="orpheus") + { + return std::make_unique(); + } #endif else if(provider=="openrouter") { @@ -434,6 +474,139 @@ ErrorCode ArbiterAI::getEmbeddings(const EmbeddingRequest &request, EmbeddingRes return provider->getEmbeddings(request, response); } +ErrorCode ArbiterAI::transcribe(const AudioTranscriptionRequest &request, AudioTranscriptionResponse &response) +{ + if(!ArbiterAI::instance().initialized) + { + return ErrorCode::InvalidRequest; + } + + std::optional modelInfo=ModelManager::instance().getModelInfo(request.model); + if(!modelInfo) + { + return ErrorCode::UnknownModel; + } + + BaseProvider *provider=getProvider(modelInfo->provider, request.model); + if(!provider) + { + return ErrorCode::UnsupportedProvider; + } + + auto start=std::chrono::steady_clock::now(); + ErrorCode result=provider->transcribe(request, *modelInfo, response); + if(result==ErrorCode::Success) + { + if(response.cost==0.0 + && modelInfo->pricing.audio_input_cost_per_second>0.0 && response.duration>0.0) + { + response.cost=response.duration*modelInfo->pricing.audio_input_cost_per_second; + } + + InferenceStats stats; + stats.model=request.model; + stats.modality=modes::Transcription; + stats.audioSeconds=response.duration; + stats.cost=response.cost; + stats.totalTimeMs=std::chrono::duration( + std::chrono::steady_clock::now()-start).count(); + if(stats.totalTimeMs>0.0 && response.duration>0.0) + stats.realtimeFactor=response.duration/(stats.totalTimeMs/1000.0); + stats.timestamp=std::chrono::system_clock::now(); + TelemetryCollector::instance().recordInference(stats); + } + return result; +} + +ErrorCode ArbiterAI::synthesizeSpeech(const SpeechRequest &request, SpeechResponse &response) +{ + if(!ArbiterAI::instance().initialized) + { + return ErrorCode::InvalidRequest; + } + + std::optional modelInfo=ModelManager::instance().getModelInfo(request.model); + if(!modelInfo) + { + return ErrorCode::UnknownModel; + } + + BaseProvider *provider=getProvider(modelInfo->provider, request.model); + if(!provider) + { + return ErrorCode::UnsupportedProvider; + } + + auto start=std::chrono::steady_clock::now(); + ErrorCode result=provider->synthesizeSpeech(request, *modelInfo, response); + if(result==ErrorCode::Success) + { + if(response.cost==0.0 && modelInfo->pricing.audio_output_cost_per_character>0.0) + { + response.cost=static_cast(request.input.length()) + *modelInfo->pricing.audio_output_cost_per_character; + } + + InferenceStats stats; + stats.model=request.model; + stats.modality=modes::Speech; + stats.audioCharacters=static_cast(request.input.length()); + stats.cost=response.cost; + stats.totalTimeMs=std::chrono::duration( + std::chrono::steady_clock::now()-start).count(); + stats.timestamp=std::chrono::system_clock::now(); + TelemetryCollector::instance().recordInference(stats); + } + return result; +} + +ErrorCode ArbiterAI::generateImage(const ImageGenerationRequest &request, ImageGenerationResponse &response) +{ + if(!ArbiterAI::instance().initialized) + { + return ErrorCode::InvalidRequest; + } + + std::optional modelInfo=ModelManager::instance().getModelInfo(request.model); + if(!modelInfo) + { + return ErrorCode::UnknownModel; + } + + BaseProvider *provider=getProvider(modelInfo->provider, request.model); + if(!provider) + { + return ErrorCode::UnsupportedProvider; + } + + auto start=std::chrono::steady_clock::now(); + ErrorCode result=provider->generateImage(request, *modelInfo, response); + if(result==ErrorCode::Success) + { + if(response.cost==0.0 && modelInfo->pricing.image_cost>0.0) + { + response.cost=static_cast(response.images.size())*modelInfo->pricing.image_cost; + } + + InferenceStats stats; + stats.model=request.model; + stats.modality=modes::Image; + stats.imagesGenerated=static_cast(response.images.size()); + stats.imageSteps=request.steps.value_or(0); + stats.cost=response.cost; + stats.totalTimeMs=std::chrono::duration( + std::chrono::steady_clock::now()-start).count(); + if(stats.totalTimeMs>0.0 && stats.imageSteps>0) + { + double totalSteps=static_cast(stats.imageSteps)*stats.imagesGenerated; + stats.stepsPerSecond=totalSteps/(stats.totalTimeMs/1000.0); + } + stats.timestamp=std::chrono::system_clock::now(); + TelemetryCollector::instance().recordInference(stats); + } + return result; +} + ErrorCode ArbiterAI::getDownloadStatus(const std::string &modelName, std::string &error) { std::optional modelInfo=ModelManager::instance().getModelInfo(modelName); diff --git a/src/arbiterAI/arbiterAI.h b/src/arbiterAI/arbiterAI.h index 69df49f..ea70070 100644 --- a/src/arbiterAI/arbiterAI.h +++ b/src/arbiterAI/arbiterAI.h @@ -56,6 +56,17 @@ struct VersionInfo { */ VersionInfo getVersion(); +/// Canonical model modality identifiers (see ModelInfo::mode). Used to route a +/// request to the correct provider method and server endpoint. +namespace modes +{ + constexpr const char *Chat="chat"; + constexpr const char *Embedding="embedding"; + constexpr const char *Transcription="transcription"; + constexpr const char *Speech="speech"; + constexpr const char *Image="image"; +} + /** * @enum ErrorCode * @brief Error codes returned by ArbiterAI operations @@ -506,6 +517,112 @@ struct EmbeddingResponse Usage usage; }; +// ========== Multimodal: Speech-to-text (STT) ========== + +/** + * @struct AudioTranscriptionRequest + * @brief Parameters for audio transcription (speech-to-text) requests + */ +struct AudioTranscriptionRequest +{ + std::string model; + std::vector audio; ///< Raw audio file bytes + std::string filename{ "audio.wav" }; ///< Original filename; extension informs the decoder + std::optional language; ///< ISO-639-1 language hint + std::optional prompt; ///< Optional decoding prompt / bias text + std::optional temperature; ///< Sampling temperature + std::optional responseFormat; ///< json, text, verbose_json, srt, vtt + std::optional api_key; + std::optional provider; +}; + +/** + * @struct AudioTranscriptionResponse + * @brief Results from audio transcription requests + */ +struct AudioTranscriptionResponse +{ + std::string text; + std::string model; + std::string provider; + std::string language; ///< Detected language, if reported + double duration = 0.0; ///< Audio duration in seconds, if reported + double cost = 0.0; +}; + +// ========== Multimodal: Text-to-speech (TTS) ========== + +/** + * @struct SpeechRequest + * @brief Parameters for speech synthesis (text-to-speech) requests + */ +struct SpeechRequest +{ + std::string model; + std::string input; ///< Text to synthesize + std::string voice{ "alloy" }; ///< Voice identifier + std::optional responseFormat; ///< mp3, opus, aac, flac, wav, pcm + std::optional speed; ///< Playback speed (0.25 - 4.0) + std::optional api_key; + std::optional provider; +}; + +/** + * @struct SpeechResponse + * @brief Results from speech synthesis requests + */ +struct SpeechResponse +{ + std::vector audio; ///< Raw synthesized audio bytes + std::string format; ///< Container/codec of the returned audio + std::string model; + std::string provider; + double cost = 0.0; +}; + +// ========== Multimodal: Image generation (diffusion) ========== + +/** + * @struct ImageGenerationRequest + * @brief Parameters for image generation (diffusion) requests + */ +struct ImageGenerationRequest +{ + std::string model; + std::string prompt; + std::optional negativePrompt; ///< Local diffusion only; ignored by OpenAI + std::optional n; ///< Number of images to generate + std::optional size; ///< e.g. "1024x1024" + std::optional steps; ///< Diffusion steps (local engines) + std::optional seed; ///< RNG seed (local / reproducibility) + std::optional responseFormat; ///< url or b64_json + std::optional api_key; + std::optional provider; +}; + +/** + * @struct GeneratedImage + * @brief A single generated image result + */ +struct GeneratedImage +{ + std::string url; ///< Populated when responseFormat == "url" + std::string b64Json; ///< Base64-encoded image when responseFormat == "b64_json" + std::string revisedPrompt; ///< Provider-revised prompt, if any +}; + +/** + * @struct ImageGenerationResponse + * @brief Results from image generation requests + */ +struct ImageGenerationResponse +{ + std::vector images; + std::string model; + std::string provider; + double cost = 0.0; +}; + /** * @class ArbiterAI * @brief Main interface for ArbiterAI LLM operations @@ -649,6 +766,32 @@ class ArbiterAI */ ErrorCode getEmbeddings(const EmbeddingRequest &request, EmbeddingResponse &response); + // ========== Multimodal (STT / TTS / Image) ========== + + /** + * @brief Transcribe audio to text (speech-to-text) + * @param request Transcription parameters (audio bytes + options) + * @param[out] response Transcription results + * @return ErrorCode indicating success or failure + */ + ErrorCode transcribe(const AudioTranscriptionRequest &request, AudioTranscriptionResponse &response); + + /** + * @brief Synthesize speech from text (text-to-speech) + * @param request Speech synthesis parameters + * @param[out] response Synthesized audio bytes + * @return ErrorCode indicating success or failure + */ + ErrorCode synthesizeSpeech(const SpeechRequest &request, SpeechResponse &response); + + /** + * @brief Generate image(s) from a text prompt (diffusion) + * @param request Image generation parameters + * @param[out] response Generated image(s) + * @return ErrorCode indicating success or failure + */ + ErrorCode generateImage(const ImageGenerationRequest &request, ImageGenerationResponse &response); + /** * @brief Get download status for a model * @param modelName Name of the model to check diff --git a/src/arbiterAI/inferenceScheduler.cpp b/src/arbiterAI/inferenceScheduler.cpp index 30d87ab..f4c18a7 100644 --- a/src/arbiterAI/inferenceScheduler.cpp +++ b/src/arbiterAI/inferenceScheduler.cpp @@ -563,6 +563,11 @@ void InferenceScheduler::acceleratorLoop(AcceleratorQueue &queue) runtime.endInference(job->request.model); job->completionTokens.store(completionTokens); + if(code!=ErrorCode::Success) + { + job->errorDetail=llamaProvider.lastErrorDetail(); + } + auto endTime=std::chrono::steady_clock::now(); double totalTimeMs=std::chrono::duration( endTime-job->inferenceStartTime).count(); diff --git a/src/arbiterAI/inferenceScheduler.h b/src/arbiterAI/inferenceScheduler.h index b21f04a..03eedb2 100644 --- a/src/arbiterAI/inferenceScheduler.h +++ b/src/arbiterAI/inferenceScheduler.h @@ -100,6 +100,11 @@ struct InferenceJob { double generationTimeMs=0.0; ErrorCode result=ErrorCode::Success; + /// Human-readable detail for why the job failed (set alongside a non-Success + /// result), surfaced in the API error response so clients see the reason, + /// not just the error code. Empty on success. + std::string errorDetail; + /// Completion token count. Atomic because the dashboard snapshots it /// while the accelerator thread is still generating. std::atomic completionTokens{0}; diff --git a/src/arbiterAI/modelManager.cpp b/src/arbiterAI/modelManager.cpp index e0de4bf..17f33dc 100644 --- a/src/arbiterAI/modelManager.cpp +++ b/src/arbiterAI/modelManager.cpp @@ -439,6 +439,16 @@ bool ModelManager::parseModelInfo(const nlohmann::json &modelJson, ModelInfo &in info.apiFormat=modelJson["api_format"].get(); } + if(modelJson.contains("whisper_options")&&modelJson["whisper_options"].is_object()) + { + info.whisperOptions=modelJson["whisper_options"]; + } + + if(modelJson.contains("sd_options")&&modelJson["sd_options"].is_object()) + { + info.sdOptions=modelJson["sd_options"]; + } + return true; } diff --git a/src/arbiterAI/modelManager.h b/src/arbiterAI/modelManager.h index ed12b07..c5bcf9e 100644 --- a/src/arbiterAI/modelManager.h +++ b/src/arbiterAI/modelManager.h @@ -81,6 +81,9 @@ struct Pricing { double prompt_token_cost=0.0; double completion_token_cost=0.0; + double image_cost=0.0; ///< Per generated image (mode: image) + double audio_input_cost_per_second=0.0; ///< Per second of transcribed audio (mode: transcription) + double audio_output_cost_per_character=0.0;///< Per character synthesized (mode: speech) }; inline void to_json(nlohmann::json &j, const Pricing &p) @@ -89,12 +92,21 @@ inline void to_json(nlohmann::json &j, const Pricing &p) {"prompt_token_cost", p.prompt_token_cost}, {"completion_token_cost", p.completion_token_cost} }; + if(p.image_cost>0.0) + j["image_cost"]=p.image_cost; + if(p.audio_input_cost_per_second>0.0) + j["audio_input_cost_per_second"]=p.audio_input_cost_per_second; + if(p.audio_output_cost_per_character>0.0) + j["audio_output_cost_per_character"]=p.audio_output_cost_per_character; } inline void from_json(const nlohmann::json &j, Pricing &p) { - j.at("prompt_token_cost").get_to(p.prompt_token_cost); - j.at("completion_token_cost").get_to(p.completion_token_cost); + if(j.contains("prompt_token_cost")) j.at("prompt_token_cost").get_to(p.prompt_token_cost); + if(j.contains("completion_token_cost")) j.at("completion_token_cost").get_to(p.completion_token_cost); + if(j.contains("image_cost")) j.at("image_cost").get_to(p.image_cost); + if(j.contains("audio_input_cost_per_second")) j.at("audio_input_cost_per_second").get_to(p.audio_input_cost_per_second); + if(j.contains("audio_output_cost_per_character")) j.at("audio_output_cost_per_character").get_to(p.audio_output_cost_per_character); } struct ModelInfo @@ -124,6 +136,8 @@ struct ModelInfo std::vector backendPriority; // Ordered preference: ["vulkan", "rocm", "cuda"] std::vector disabledBackends; // Backends to exclude (model-level override) std::string apiFormat; // API output format: "" (default/openai) or "harmony" + nlohmann::json whisperOptions; // Engine-specific whisper (STT) defaults (whisper_options) + nlohmann::json sdOptions; // Engine-specific stable-diffusion (image) defaults (sd_options) bool isCompatible(const std::string &clientVersion) const; bool isSchemaCompatible(const std::string &schemaVersion) const; diff --git a/src/arbiterAI/providers/baseProvider.cpp b/src/arbiterAI/providers/baseProvider.cpp index 983a2c4..200aac0 100644 --- a/src/arbiterAI/providers/baseProvider.cpp +++ b/src/arbiterAI/providers/baseProvider.cpp @@ -1,7 +1,11 @@ #include "arbiterAI/providers/baseProvider.h" #include "arbiterAI/modelManager.h" +#include "arbiterAI/modelDownloader.h" +#include "arbiterAI/storageManager.h" +#include #include #include +#include #include #include #include @@ -62,6 +66,62 @@ ErrorCode BaseProvider::getApiKey(const std::string &modelName, } +std::string BaseProvider::resolveDownloadableModelFile(const ModelInfo &model) +{ + namespace fs=std::filesystem; + + // 1. Explicit local file that already exists wins. + if(model.filePath.has_value() && !model.filePath->empty() + && fs::exists(model.filePath.value())) + { + return model.filePath.value(); + } + + // 2. Download from the primary variant's configured files if present. + if(!model.variants.empty()) + { + const ModelVariant &variant=model.variants.front(); + std::vector files=variant.getAllFiles(); + std::string primaryFilename=variant.getPrimaryFilename(); + + if(!files.empty() && !primaryFilename.empty()) + { + fs::path modelsDir=StorageManager::instance().getModelsDir(); + if(modelsDir.empty()) + { + spdlog::warn("{} provider: no models directory configured for auto-download of '{}'", + m_provider, model.model); + } + else + { + ModelDownloader downloader; + for(const VariantDownload &file:files) + { + if(file.url.empty() || file.filename.empty()) + continue; + fs::path localPath=modelsDir/file.filename; + if(fs::exists(localPath)) + continue; + spdlog::info("{} provider: downloading '{}' -> '{}'", + m_provider, file.url, localPath.string()); + std::optional hash; + if(!file.sha256.empty()) hash=file.sha256; + auto fut=downloader.downloadModel(file.url, localPath.string(), hash); + if(!fut.get()) + { + spdlog::error("{} provider: download failed for '{}'", m_provider, file.url); + return {}; + } + } + return (modelsDir/primaryFilename).string(); + } + } + } + + // 3. Fall back to file_path (may not exist / be empty — caller handles it). + return model.filePath.value_or(std::string{}); +} + DownloadStatus BaseProvider::getDownloadStatus(const std::string &modelName, std::string &error) { // Default implementation for cloud providers - no download needed @@ -85,6 +145,30 @@ ErrorCode BaseProvider::getAvailableModels(std::vector& models) return ErrorCode::NotImplemented; } +ErrorCode BaseProvider::transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) +{ + // Default: provider does not support speech-to-text + return ErrorCode::NotImplemented; +} + +ErrorCode BaseProvider::synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) +{ + // Default: provider does not support text-to-speech + return ErrorCode::NotImplemented; +} + +ErrorCode BaseProvider::generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) +{ + // Default: provider does not support image generation + return ErrorCode::NotImplemented; +} + std::vector BaseProvider::batchCompletion(const std::vector &requests) { std::vector> futures; diff --git a/src/arbiterAI/providers/baseProvider.h b/src/arbiterAI/providers/baseProvider.h index 2b80e4a..9f28c67 100644 --- a/src/arbiterAI/providers/baseProvider.h +++ b/src/arbiterAI/providers/baseProvider.h @@ -11,6 +11,12 @@ namespace arbiterAI struct EmbeddingRequest; struct EmbeddingResponse; +struct AudioTranscriptionRequest; +struct AudioTranscriptionResponse; +struct SpeechRequest; +struct SpeechResponse; +struct ImageGenerationRequest; +struct ImageGenerationResponse; /** * @class BaseProvider @@ -82,6 +88,36 @@ class BaseProvider virtual ErrorCode getEmbeddings(const EmbeddingRequest &request, EmbeddingResponse &response) = 0; + /** + * @brief Transcribe audio to text (speech-to-text) + * + * Default implementation returns ErrorCode::NotImplemented. Providers that + * support STT override this. + */ + virtual ErrorCode transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response); + + /** + * @brief Synthesize speech from text (text-to-speech) + * + * Default implementation returns ErrorCode::NotImplemented. Providers that + * support TTS override this. + */ + virtual ErrorCode synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response); + + /** + * @brief Generate image(s) from a text prompt (diffusion) + * + * Default implementation returns ErrorCode::NotImplemented. Providers that + * support image generation override this. + */ + virtual ErrorCode generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response); + /** * @brief Get download status for a model (legacy interface) * @param modelName Name of the model @@ -133,6 +169,24 @@ class BaseProvider ErrorCode getApiKey(const std::string &modelName, const std::optional &requestApiKey, std::string &apiKey); + /** + * @brief Resolve a local-engine model's weight file, downloading if needed. + * + * Resolution order: + * 1. If the model's `file_path` is set and the file exists, use it. + * 2. Otherwise, if the model's primary variant carries download info + * (`variants[].download.url` / `files[]`), download every file to the + * StorageManager models directory (skipping ones already present) and + * return the primary file's local path — same auto-download behavior as + * llama GGUFs. Multi-file variants (e.g. TTS model + tokenizer + voice) + * are all fetched. + * 3. Otherwise return `file_path` as-is (may be empty; caller errors). + * + * The download is synchronous (blocks the load/first request). Returns an + * empty string only if a required download fails. + */ + std::string resolveDownloadableModelFile(const ModelInfo &model); + protected: std::string m_provider; std::string m_apiKey; ///< API key set via setApiKey() diff --git a/src/arbiterAI/providers/llama.cpp b/src/arbiterAI/providers/llama.cpp index 54ecdba..6a05b20 100644 --- a/src/arbiterAI/providers/llama.cpp +++ b/src/arbiterAI/providers/llama.cpp @@ -16,6 +16,63 @@ namespace arbiterAI { +namespace +{ + +/// Byte length of the largest prefix of s that ends on a complete UTF-8 +/// sequence (i.e. does not split a multi-byte character). A lead byte with too +/// few following continuation bytes is held back; stray/invalid bytes are passed +/// through as-is. +size_t utf8CompleteLen(const std::string &s) +{ + size_t i=0, complete=0; + while(i(s[i]); + size_t need; + if(c<0x80) need=1; // 0xxxxxxx + else if((c>>5)==0x6) need=2; // 110xxxxx + else if((c>>4)==0xE) need=3; // 1110xxxx + else if((c>>3)==0x1E) need=4; // 11110xxx + else { i+=1; complete=i; continue; } // invalid lead / stray continuation + if(i+need>s.size()) break; // truncated multi-byte at the tail — hold back + i+=need; + complete=i; + } + return complete; +} + +/// Wraps a streaming callback so it never emits a partial multi-byte UTF-8 +/// character. Model tokens can split a character across token boundaries (or emit +/// a llama.cpp byte-fallback token); emitting that as a streaming delta yields +/// invalid UTF-8. Complete characters are forwarded immediately; an incomplete +/// tail is buffered until the next token completes it, and any residue is flushed +/// on destruction (end of generation). +class Utf8StreamBuffer +{ +public: + explicit Utf8StreamBuffer(const std::function &cb) : m_cb(cb) {} + ~Utf8StreamBuffer() { if(m_cb && !m_pending.empty()) m_cb(m_pending); } + + void feed(const std::string &text) + { + if(!m_cb) return; + m_pending+=text; + size_t n=utf8CompleteLen(m_pending); + if(n>0) + { + m_cb(m_pending.substr(0, n)); + m_pending.erase(0, n); + } + } + +private: + const std::function &m_cb; + std::string m_pending; +}; + +} // namespace + Llama::Llama(): BaseProvider("llama") { @@ -649,6 +706,7 @@ ErrorCode Llama::runInference(llama_model *model, llama_context *ctx, if(nTokens<0) { spdlog::error("Failed to tokenize prompt"); + m_lastErrorDetail="failed to tokenize prompt"; return ErrorCode::GenerationError; } } @@ -690,6 +748,9 @@ ErrorCode Llama::runInference(llama_model *model, llama_context *ctx, { spdlog::error("[llama] llama_decode failed during prompt processing (chunk at offset {}, chunkSize={}, totalTokens={}, result={})", start, chunkSize, nTokens, decodeResult); + m_lastErrorDetail="llama backend failed to process the prompt (llama_decode result=" + +std::to_string(decodeResult)+", "+std::to_string(nTokens) + +" prompt tokens) — the context may exceed the model/hardware limit or the GPU backend errored"; llama_batch_free(batch); return ErrorCode::GenerationError; } @@ -761,6 +822,8 @@ ErrorCode Llama::runInference(llama_model *model, llama_context *ctx, harmonyCallToken, harmonyReturnToken); } + Utf8StreamBuffer streamBuf(streamCallback); + for(int i=0; i"); + streamBuf.feed("<|call|>"); } completionTokens++; break; @@ -803,10 +866,7 @@ ErrorCode Llama::runInference(llama_model *model, llama_context *ctx, result+=tokenText; completionTokens++; - if(streamCallback) - { - streamCallback(tokenText); - } + streamBuf.feed(tokenText); } // Check stop sequences @@ -844,6 +904,9 @@ ErrorCode Llama::runInference(llama_model *model, llama_context *ctx, { spdlog::error("[llama] llama_decode failed during generation (token #{}, pos={}, result={})", i, nCur-1, decodeResult); + m_lastErrorDetail="llama backend failed during token generation (llama_decode result=" + +std::to_string(decodeResult)+" at position "+std::to_string(nCur-1) + +") — likely a GPU/backend error or the context was exhausted"; llama_sampler_free(samplerChain); llama_batch_free(batch); return ErrorCode::GenerationError; @@ -886,6 +949,7 @@ ErrorCode Llama::tokenizePrompt(llama_model *model, if(nTokens<0) { spdlog::error("Failed to tokenize prompt"); + m_lastErrorDetail="failed to tokenize the formatted prompt"; return ErrorCode::GenerationError; } } @@ -947,6 +1011,9 @@ ErrorCode Llama::runInferenceWithTokens(llama_model *model, llama_context *ctx, { spdlog::error("[llama] llama_decode failed during prompt processing (chunk at offset {}, chunkSize={}, totalTokens={}, result={})", start, chunkSize, nTokens, decodeResult); + m_lastErrorDetail="llama backend failed to process the prompt (llama_decode result=" + +std::to_string(decodeResult)+", "+std::to_string(nTokens) + +" prompt tokens) — the context may exceed the model/hardware limit or the GPU backend errored"; llama_batch_free(batch); return ErrorCode::GenerationError; } @@ -1009,6 +1076,8 @@ ErrorCode Llama::runInferenceWithTokens(llama_model *model, llama_context *ctx, harmonyCallToken, harmonyReturnToken); } + Utf8StreamBuffer streamBuf(streamCallback); + for(int i=0; i"); + streamBuf.feed("<|call|>"); } completionTokens++; break; @@ -1056,10 +1125,7 @@ ErrorCode Llama::runInferenceWithTokens(llama_model *model, llama_context *ctx, result+=tokenText; completionTokens++; - if(streamCallback) - { - streamCallback(tokenText); - } + streamBuf.feed(tokenText); } if(request.stop.has_value()) @@ -1094,6 +1160,9 @@ ErrorCode Llama::runInferenceWithTokens(llama_model *model, llama_context *ctx, { spdlog::error("[llama] llama_decode failed during generation (token #{}, pos={}, result={})", i, nCur-1, decodeResult); + m_lastErrorDetail="llama backend failed during token generation (llama_decode result=" + +std::to_string(decodeResult)+" at position "+std::to_string(nCur-1) + +") — likely a GPU/backend error or the context was exhausted"; llama_sampler_free(samplerChain); llama_batch_free(batch); return ErrorCode::GenerationError; diff --git a/src/arbiterAI/providers/llama.h b/src/arbiterAI/providers/llama.h index cd76433..494c188 100644 --- a/src/arbiterAI/providers/llama.h +++ b/src/arbiterAI/providers/llama.h @@ -39,6 +39,11 @@ class Llama : public BaseProvider { ErrorCode getAvailableModels(std::vector &models) override; + /// Human-readable detail for the most recent failure on this instance. + /// Set alongside a returned error code so callers can report *why* a + /// generation failed rather than just the error code. Empty on success. + const std::string &lastErrorDetail() const { return m_lastErrorDetail; } + /// Tokenize the prompt outside of the inference mutex. /// Only reads llama_model/vocab (thread-safe without context lock). ErrorCode tokenizePrompt(llama_model *model, @@ -69,6 +74,9 @@ class Llama : public BaseProvider { std::string &result, int &promptTokens, int &completionTokens, double &promptTimeMs, double &generationTimeMs, std::function streamCallback); + + /// Detail for the most recent failure (see lastErrorDetail()). + std::string m_lastErrorDetail; }; } // namespace arbiterAI diff --git a/src/arbiterAI/providers/mock.cpp b/src/arbiterAI/providers/mock.cpp index 5d22b03..b552395 100644 --- a/src/arbiterAI/providers/mock.cpp +++ b/src/arbiterAI/providers/mock.cpp @@ -111,6 +111,57 @@ ErrorCode Mock::getAvailableModels(std::vector& models) return ErrorCode::Success; } +ErrorCode Mock::transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) +{ + std::string echoContent; + if(request.prompt.has_value() && extractEchoContent(request.prompt.value(), echoContent)) + { + response.text=echoContent; + } + else + { + response.text="This is a mock transcription of " + +std::to_string(request.audio.size())+" bytes of audio."; + } + response.model=request.model; + response.provider="mock"; + response.language=request.language.value_or("en"); + return ErrorCode::Success; +} + +ErrorCode Mock::synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) +{ + // Deterministic "audio": the UTF-8 bytes of the input text. + response.audio.assign(request.input.begin(), request.input.end()); + response.format=request.responseFormat.value_or("wav"); + response.model=request.model; + response.provider="mock"; + return ErrorCode::Success; +} + +ErrorCode Mock::generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) +{ + int count=request.n.value_or(1); + if(count<1) + count=1; + for(int i=0; i... tags (supporting multiline and greedy matching) diff --git a/src/arbiterAI/providers/mock.h b/src/arbiterAI/providers/mock.h index 59de23c..f311759 100644 --- a/src/arbiterAI/providers/mock.h +++ b/src/arbiterAI/providers/mock.h @@ -89,6 +89,36 @@ class Mock : public BaseProvider ErrorCode getEmbeddings(const EmbeddingRequest &request, EmbeddingResponse &response) override; + /** + * @brief Mock transcription (speech-to-text) + * + * Returns deterministic text. If the request prompt contains an + * ... tag, the tagged content is returned instead. + */ + ErrorCode transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) override; + + /** + * @brief Mock speech synthesis (text-to-speech) + * + * Returns the UTF-8 bytes of the input text as the "audio" payload so + * tests can assert on deterministic output without real audio. + */ + ErrorCode synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) override; + + /** + * @brief Mock image generation (diffusion) + * + * Returns a single deterministic image entry whose b64Json is derived + * from the prompt, and echoes the prompt back as revisedPrompt. + */ + ErrorCode generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) override; + /** * @brief Get available mock models * diff --git a/src/arbiterAI/providers/openai.cpp b/src/arbiterAI/providers/openai.cpp index d39722f..de62d56 100644 --- a/src/arbiterAI/providers/openai.cpp +++ b/src/arbiterAI/providers/openai.cpp @@ -707,4 +707,206 @@ ErrorCode OpenAI::getAvailableModels(std::vector& models) return ErrorCode::Success; } +ErrorCode OpenAI::transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) +{ + std::string apiKey; + auto result=getApiKey(request.model, request.api_key, apiKey); + if(result!=ErrorCode::Success) + { + return result; + } + + std::string baseUrl=(model.apiBase.has_value() && !model.apiBase->empty()) + ? model.apiBase.value() : m_apiUrl; + std::string url=baseUrl+"/audio/transcriptions"; + + // Multipart upload: file + model + optional decoding hints. + // Do not set Content-Type here — cpr fills in the multipart boundary. + cpr::Multipart multipart{ + {"file", cpr::Buffer{request.audio.begin(), request.audio.end(), request.filename}}, + {"model", request.model} + }; + if(request.language.has_value()) + multipart.parts.emplace_back("language", request.language.value()); + if(request.prompt.has_value()) + multipart.parts.emplace_back("prompt", request.prompt.value()); + if(request.temperature.has_value()) + multipart.parts.emplace_back("temperature", std::to_string(request.temperature.value())); + if(request.responseFormat.has_value()) + multipart.parts.emplace_back("response_format", request.responseFormat.value()); + + cpr::Header headers; + if(!apiKey.empty()) + headers["Authorization"]="Bearer "+apiKey; + + auto raw_response=cpr::Post( + cpr::Url{ url }, + headers, + multipart, + cpr::VerifySsl{ true }, + cpr::Timeout{ 300000 } + ); + + if(raw_response.status_code!=200) + { + spdlog::warn("OpenAI provider: transcription HTTP {} from {} (body: {})", + raw_response.status_code, url, raw_response.text.substr(0, 500)); + return ErrorCode::NetworkError; + } + + response.model=request.model; + response.provider="openai"; + + // response_format=text returns raw text; json/verbose_json return an object. + const std::string trimmed=raw_response.text; + size_t firstNonWs=trimmed.find_first_not_of(" \t\r\n"); + if(firstNonWs!=std::string::npos && trimmed[firstNonWs]=='{') + { + nlohmann::json jsonResponse; + try + { + jsonResponse=nlohmann::json::parse(trimmed); + } + catch(const nlohmann::json::parse_error &) + { + return ErrorCode::InvalidResponse; + } + if(jsonResponse.contains("text")) + response.text=jsonResponse["text"].get(); + if(jsonResponse.contains("language") && !jsonResponse["language"].is_null()) + response.language=jsonResponse["language"].get(); + if(jsonResponse.contains("duration") && jsonResponse["duration"].is_number()) + response.duration=jsonResponse["duration"].get(); + } + else + { + response.text=trimmed; + } + + return ErrorCode::Success; +} + +ErrorCode OpenAI::synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) +{ + std::string apiKey; + auto result=getApiKey(request.model, request.api_key, apiKey); + if(result!=ErrorCode::Success) + { + return result; + } + + std::string baseUrl=(model.apiBase.has_value() && !model.apiBase->empty()) + ? model.apiBase.value() : m_apiUrl; + std::string url=baseUrl+"/audio/speech"; + + nlohmann::json body; + body["model"]=request.model; + body["input"]=request.input; + body["voice"]=request.voice; + std::string format=request.responseFormat.value_or("mp3"); + body["response_format"]=format; + if(request.speed.has_value()) + body["speed"]=request.speed.value(); + + auto raw_response=cpr::Post( + cpr::Url{ url }, + createHeaders(apiKey), + cpr::Body{ body.dump() }, + cpr::VerifySsl{ true }, + cpr::Timeout{ 300000 } + ); + + if(raw_response.status_code!=200) + { + spdlog::warn("OpenAI provider: speech HTTP {} from {} (body: {})", + raw_response.status_code, url, raw_response.text.substr(0, 500)); + return ErrorCode::NetworkError; + } + + // The body is the raw (binary-safe) audio payload. + response.audio.assign(raw_response.text.begin(), raw_response.text.end()); + response.format=format; + response.model=request.model; + response.provider="openai"; + + return ErrorCode::Success; +} + +ErrorCode OpenAI::generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) +{ + std::string apiKey; + auto result=getApiKey(request.model, request.api_key, apiKey); + if(result!=ErrorCode::Success) + { + return result; + } + + std::string baseUrl=(model.apiBase.has_value() && !model.apiBase->empty()) + ? model.apiBase.value() : m_apiUrl; + std::string url=baseUrl+"/images/generations"; + + nlohmann::json body; + body["model"]=request.model; + body["prompt"]=request.prompt; + if(request.n.has_value()) + body["n"]=request.n.value(); + if(request.size.has_value()) + body["size"]=request.size.value(); + if(request.responseFormat.has_value()) + body["response_format"]=request.responseFormat.value(); + + auto raw_response=cpr::Post( + cpr::Url{ url }, + createHeaders(apiKey), + cpr::Body{ body.dump() }, + cpr::VerifySsl{ true }, + cpr::Timeout{ 300000 } + ); + + if(raw_response.status_code!=200) + { + spdlog::warn("OpenAI provider: image HTTP {} from {} (body: {})", + raw_response.status_code, url, raw_response.text.substr(0, 500)); + return ErrorCode::NetworkError; + } + + nlohmann::json jsonResponse; + try + { + jsonResponse=nlohmann::json::parse(raw_response.text); + } + catch(const nlohmann::json::parse_error &) + { + return ErrorCode::InvalidResponse; + } + + if(!jsonResponse.contains("data") || !jsonResponse["data"].is_array()) + { + return ErrorCode::InvalidResponse; + } + + for(const auto &item:jsonResponse["data"]) + { + GeneratedImage image; + if(item.contains("url") && !item["url"].is_null()) + image.url=item["url"].get(); + if(item.contains("b64_json") && !item["b64_json"].is_null()) + image.b64Json=item["b64_json"].get(); + if(item.contains("revised_prompt") && !item["revised_prompt"].is_null()) + image.revisedPrompt=item["revised_prompt"].get(); + response.images.push_back(image); + } + + response.model=request.model; + response.provider="openai"; + + return ErrorCode::Success; +} + } // namespace arbiterAI diff --git a/src/arbiterAI/providers/openai.h b/src/arbiterAI/providers/openai.h index dd9691e..a8535b7 100644 --- a/src/arbiterAI/providers/openai.h +++ b/src/arbiterAI/providers/openai.h @@ -25,6 +25,18 @@ class OpenAI : public BaseProvider ErrorCode getEmbeddings(const EmbeddingRequest &request, EmbeddingResponse &response) override; + ErrorCode transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) override; + + ErrorCode synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) override; + + ErrorCode generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) override; + ErrorCode getAvailableModels(std::vector& models) override; void setApiUrl(const std::string &url) override { m_apiUrl = url; } diff --git a/src/arbiterAI/providers/orpheus.cpp b/src/arbiterAI/providers/orpheus.cpp new file mode 100644 index 0000000..79627a3 --- /dev/null +++ b/src/arbiterAI/providers/orpheus.cpp @@ -0,0 +1,143 @@ +#include "arbiterAI/providers/orpheus.h" + +#include +#include +#include +#include +#include + +namespace arbiterAI +{ + +Orpheus::Orpheus() + : BaseProvider("orpheus") +{ +} + +Orpheus::~Orpheus() +{ + std::lock_guard lock(m_mutex); + if(m_handle) + { + dlclose(m_handle); + m_handle = nullptr; + m_ttsFn = nullptr; + } +} + +ErrorCode Orpheus::completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode Orpheus::streamingCompletion(const CompletionRequest &request, + std::function callback) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode Orpheus::getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) +{ + return ErrorCode::NotImplemented; +} + +std::string Orpheus::resolveModelPath(const ModelInfo &model) +{ + return resolveDownloadableModelFile(model); +} + +Orpheus::OrpheusTtsFn Orpheus::acquireEngine() +{ + if(m_ttsFn) + return m_ttsFn; + if(m_loadAttempted) + return nullptr; // don't retry a known-failed load every request + + m_loadAttempted = true; + + // Engine library is isolated (RTLD_LOCAL) so its bundled/forked ggml does + // not clash with the llama provider's ggml in this process. + const char *envLib=std::getenv("ARBITER_ORPHEUS_ENGINE"); + std::string libPath=(envLib && *envLib) ? envLib : "liborpheus_engine.so"; + + m_handle=dlopen(libPath.c_str(), RTLD_NOW | RTLD_LOCAL); + if(!m_handle) + { + spdlog::warn("Orpheus provider: could not load engine library '{}': {}", + libPath, dlerror()); + return nullptr; + } + + dlerror(); // clear + m_ttsFn=reinterpret_cast(dlsym(m_handle, "arbiter_orpheus_tts")); + const char *symErr=dlerror(); + if(symErr || !m_ttsFn) + { + spdlog::warn("Orpheus provider: engine '{}' missing 'arbiter_orpheus_tts': {}", + libPath, symErr ? symErr : "not found"); + dlclose(m_handle); + m_handle=nullptr; + m_ttsFn=nullptr; + return nullptr; + } + + return m_ttsFn; +} + +ErrorCode Orpheus::synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) +{ + std::string modelPath=resolveModelPath(model); + if(modelPath.empty()) + { + spdlog::warn("Orpheus provider: no model file configured for '{}'", model.model); + return ErrorCode::ModelNotFound; + } + + std::lock_guard lock(m_mutex); + + OrpheusTtsFn tts=acquireEngine(); + if(!tts) + return ErrorCode::ModelLoadError; + + namespace fs=std::filesystem; + fs::path tmpWav=fs::temp_directory_path() + /("arbiter_orpheus_"+std::to_string(m_tempCounter.fetch_add(1))+".wav"); + + int rc=tts(modelPath.c_str(), request.input.c_str(), request.voice.c_str(), + tmpWav.string().c_str()); + if(rc!=0) + { + spdlog::warn("Orpheus provider: engine returned {} for '{}'", rc, model.model); + std::error_code ec; + fs::remove(tmpWav, ec); + return ErrorCode::GenerationError; + } + + std::ifstream in(tmpWav, std::ios::binary); + if(!in) + { + spdlog::warn("Orpheus provider: engine produced no WAV at '{}'", tmpWav.string()); + return ErrorCode::GenerationError; + } + response.audio.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); + in.close(); + + std::error_code ec; + fs::remove(tmpWav, ec); + + if(response.audio.empty()) + return ErrorCode::GenerationError; + + response.format="wav"; + response.model=model.model; + response.provider="orpheus"; + + return ErrorCode::Success; +} + +} // namespace arbiterAI diff --git a/src/arbiterAI/providers/orpheus.h b/src/arbiterAI/providers/orpheus.h new file mode 100644 index 0000000..f69ab56 --- /dev/null +++ b/src/arbiterAI/providers/orpheus.h @@ -0,0 +1,77 @@ +#ifndef _arbiterAI_providers_orpheus_h_ +#define _arbiterAI_providers_orpheus_h_ + +#include "arbiterAI/providers/baseProvider.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include + +namespace arbiterAI +{ + +/** + * @class Orpheus + * @brief Local text-to-speech provider for Orpheus 3B (SNAC-coded speech-LLM) + * + * Orpheus is a Llama-3 backbone that emits SNAC audio codes decoded to a 24 kHz + * waveform. The decode (and, in this design, the whole text->audio pipeline) is + * handled by chatllm.cpp, which ships a native C++ Orpheus + SNAC decoder. + * + * Because chatllm.cpp bundles a *forked* ggml that cannot statically coexist + * with the ggml linked by the llama provider, chatllm is isolated behind a + * self-contained engine shared library that we load at runtime with dlopen + + * RTLD_LOCAL. That library must export the flat C entry point: + * + * int arbiter_orpheus_tts(const char *model_path, const char *text, + * const char *voice, const char *out_wav_path); + * + * returning 0 on success and writing a WAV to out_wav_path. (chatllm's own C + * binding does not expose audio, so this thin shim wraps its internal + * speech_synthesis — see docs/tasks/multimodal_model_support.md.) + * + * The engine library path comes from the env var ARBITER_ORPHEUS_ENGINE, else + * "liborpheus_engine.so" resolved via the normal loader search path. + */ +class Orpheus : public BaseProvider +{ +public: + Orpheus(); + ~Orpheus() override; + + ErrorCode completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) override; + + ErrorCode streamingCompletion(const CompletionRequest &request, + std::function callback) override; + + ErrorCode getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) override; + + ErrorCode synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) override; + +private: + /// Signature of the C entry point the engine library must export. + typedef int (*OrpheusTtsFn)(const char *modelPath, const char *text, + const char *voice, const char *outWavPath); + + std::string resolveModelPath(const ModelInfo &model); + + /// dlopen the engine library and resolve the entry point (once). Returns the + /// function pointer, or nullptr on failure. Thread-safe. + OrpheusTtsFn acquireEngine(); + + std::mutex m_mutex; + void *m_handle = nullptr; ///< dlopen handle for the engine library + OrpheusTtsFn m_ttsFn = nullptr; ///< resolved arbiter_orpheus_tts + bool m_loadAttempted = false; + std::atomic m_tempCounter{ 0 }; +}; + +} // namespace arbiterAI + +#endif//_arbiterAI_providers_orpheus_h_ diff --git a/src/arbiterAI/providers/stableDiffusion.cpp b/src/arbiterAI/providers/stableDiffusion.cpp new file mode 100644 index 0000000..b6b2941 --- /dev/null +++ b/src/arbiterAI/providers/stableDiffusion.cpp @@ -0,0 +1,258 @@ +#include "arbiterAI/providers/stableDiffusion.h" + +#include +#include + +#include +#include +#include + +namespace arbiterAI +{ + +namespace +{ + +/// Standard base64 alphabet encoder. +std::string base64Encode(const uint8_t *data, size_t len) +{ + static const char *tbl="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve(((len+2)/3)*4); + size_t i=0; + for(; i+3<=len; i+=3) + { + uint32_t n=(data[i]<<16)|(data[i+1]<<8)|data[i+2]; + out.push_back(tbl[(n>>18)&0x3F]); + out.push_back(tbl[(n>>12)&0x3F]); + out.push_back(tbl[(n>>6)&0x3F]); + out.push_back(tbl[n&0x3F]); + } + if(i>18)&0x3F]); + out.push_back(tbl[(n>>12)&0x3F]); + out.push_back(two?tbl[(n>>6)&0x3F]:'='); + out.push_back('='); + } + return out; +} + +void appendBE32(std::vector &v, uint32_t x) +{ + v.push_back(static_cast((x>>24)&0xFF)); + v.push_back(static_cast((x>>16)&0xFF)); + v.push_back(static_cast((x>>8)&0xFF)); + v.push_back(static_cast(x&0xFF)); +} + +/// Append a PNG chunk (length, type, data, CRC-32 over type+data). +void appendChunk(std::vector &png, const char *type, const std::vector &data) +{ + appendBE32(png, static_cast(data.size())); + size_t crcStart=png.size(); + png.insert(png.end(), type, type+4); + png.insert(png.end(), data.begin(), data.end()); + uLong crc=crc32(0L, Z_NULL, 0); + crc=crc32(crc, png.data()+crcStart, static_cast(png.size()-crcStart)); + appendBE32(png, static_cast(crc)); +} + +/// Encode raw 8-bit RGB(A) pixels as a PNG (deflate via zlib). Returns empty on failure. +std::vector encodePng(uint32_t width, uint32_t height, uint32_t channels, const uint8_t *pixels) +{ + if(channels!=3 && channels!=4) + return {}; + + // Filter each scanline with filter type 0 (None): [0][row bytes]. + const size_t rowBytes=static_cast(width)*channels; + std::vector raw; + raw.reserve((rowBytes+1)*height); + for(uint32_t y=0; y(raw.size())); + std::vector comp(compBound); + if(compress2(comp.data(), &compBound, raw.data(), static_cast(raw.size()), Z_BEST_SPEED)!=Z_OK) + return {}; + comp.resize(compBound); + + std::vector png={0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A}; + + std::vector ihdr; + appendBE32(ihdr, width); + appendBE32(ihdr, height); + ihdr.push_back(8); // bit depth + ihdr.push_back(channels==4?6:2); // color type: 2=RGB, 6=RGBA + ihdr.push_back(0); // compression + ihdr.push_back(0); // filter + ihdr.push_back(0); // interlace + appendChunk(png, "IHDR", ihdr); + appendChunk(png, "IDAT", comp); + appendChunk(png, "IEND", {}); + + return png; +} + +} // namespace + +StableDiffusion::StableDiffusion() + : BaseProvider("stable-diffusion") +{ +} + +StableDiffusion::~StableDiffusion() +{ + std::lock_guard lock(m_mutex); + for(auto &entry:m_contexts) + { + if(entry.second) + free_sd_ctx(entry.second); + } + m_contexts.clear(); +} + +ErrorCode StableDiffusion::completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode StableDiffusion::streamingCompletion(const CompletionRequest &request, + std::function callback) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode StableDiffusion::getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) +{ + return ErrorCode::NotImplemented; +} + +std::string StableDiffusion::resolveModelPath(const ModelInfo &model) +{ + return resolveDownloadableModelFile(model); +} + +sd_ctx_t *StableDiffusion::acquireContext(const std::string &modelPath) +{ + std::lock_guard lock(m_mutex); + + auto it=m_contexts.find(modelPath); + if(it!=m_contexts.end()) + return it->second; + + sd_ctx_params_t params; + sd_ctx_params_init(¶ms); + params.model_path=modelPath.c_str(); + + sd_ctx_t *ctx=new_sd_ctx(¶ms); + if(!ctx) + { + spdlog::warn("StableDiffusion provider: failed to load model from '{}'", modelPath); + return nullptr; + } + + m_contexts.emplace(modelPath, ctx); + return ctx; +} + +ErrorCode StableDiffusion::generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) +{ + std::string modelPath=resolveModelPath(model); + if(modelPath.empty()) + { + spdlog::warn("StableDiffusion provider: no model file configured for '{}'", model.model); + return ErrorCode::ModelNotFound; + } + + // Defaults from the model's sd_options; request fields override. + bool haveOpts=model.sdOptions.is_object(); + int width=haveOpts ? model.sdOptions.value("width", 512) : 512; + int height=haveOpts ? model.sdOptions.value("height", 512) : 512; + int defaultSteps=haveOpts ? model.sdOptions.value("steps", 0) : 0; + + // Parse "WIDTHxHEIGHT" if provided (OpenAI-style "size") — overrides the default. + if(request.size.has_value()) + { + int w=0, h=0; + if(std::sscanf(request.size->c_str(), "%dx%d", &w, &h)==2 && w>0 && h>0) + { + width=w; + height=h; + } + } + + sd_ctx_t *ctx=acquireContext(modelPath); + if(!ctx) + return ErrorCode::ModelLoadError; + + sd_img_gen_params_t gp; + sd_img_gen_params_init(&gp); + gp.prompt=request.prompt.c_str(); + std::string negative=request.negativePrompt.value_or(""); + gp.negative_prompt=negative.c_str(); + gp.width=width; + gp.height=height; + gp.batch_count=request.n.value_or(1); + gp.seed=request.seed.value_or(-1); + if(request.steps.has_value() && request.steps.value()>0) + gp.sample_params.sample_steps=request.steps.value(); + else if(defaultSteps>0) + gp.sample_params.sample_steps=defaultSteps; + if(haveOpts && model.sdOptions.contains("cfg_scale")) + gp.sample_params.guidance.txt_cfg=model.sdOptions.value("cfg_scale", gp.sample_params.guidance.txt_cfg); + + sd_image_t *images=nullptr; + int numImages=0; + + { + // generate_image is not reentrant on a shared context — serialize. + std::lock_guard lock(m_inferenceMutex); + bool ok=generate_image(ctx, &gp, &images, &numImages); + if(!ok || !images || numImages<=0) + { + spdlog::warn("StableDiffusion provider: generate_image failed for '{}'", model.model); + if(images) + free(images); + return ErrorCode::GenerationError; + } + } + + for(int i=0; i png=encodePng(img.width, img.height, img.channel, img.data); + if(!png.empty()) + { + GeneratedImage out; + out.b64Json=base64Encode(png.data(), png.size()); + out.revisedPrompt=request.prompt; + response.images.push_back(std::move(out)); + } + // sd allocates each image's pixel buffer with malloc. + if(img.data) + free(img.data); + } + free(images); + + if(response.images.empty()) + return ErrorCode::GenerationError; + + response.model=model.model; + response.provider="stable-diffusion"; + + return ErrorCode::Success; +} + +} // namespace arbiterAI diff --git a/src/arbiterAI/providers/stableDiffusion.h b/src/arbiterAI/providers/stableDiffusion.h new file mode 100644 index 0000000..4f532e0 --- /dev/null +++ b/src/arbiterAI/providers/stableDiffusion.h @@ -0,0 +1,60 @@ +#ifndef _arbiterAI_providers_stableDiffusion_h_ +#define _arbiterAI_providers_stableDiffusion_h_ + +#include "arbiterAI/providers/baseProvider.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include + +struct sd_ctx_t; + +namespace arbiterAI +{ + +/** + * @class StableDiffusion + * @brief Local text-to-image provider backed by stable-diffusion.cpp + * + * Handles models configured with `"provider": "stable-diffusion"` and + * `"mode": "image"`. The model weights path comes from the model config + * (`file_path`, or the primary variant filename). Loaded sd contexts are cached + * per model file path; generated images are PNG-encoded and returned as base64 + * in `GeneratedImage::b64Json` (OpenAI-compatible `b64_json`). + */ +class StableDiffusion : public BaseProvider +{ +public: + StableDiffusion(); + ~StableDiffusion() override; + + ErrorCode completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) override; + + ErrorCode streamingCompletion(const CompletionRequest &request, + std::function callback) override; + + ErrorCode getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) override; + + ErrorCode generateImage(const ImageGenerationRequest &request, + const ModelInfo &model, + ImageGenerationResponse &response) override; + +private: + std::string resolveModelPath(const ModelInfo &model); + + /// Get (loading on first use) an sd context for the given model file. + /// Returns nullptr on load failure. Thread-safe. + sd_ctx_t *acquireContext(const std::string &modelPath); + + std::mutex m_mutex; ///< Guards the context cache + std::mutex m_inferenceMutex; ///< Serializes generation (v1 limitation) + std::map m_contexts; +}; + +} // namespace arbiterAI + +#endif//_arbiterAI_providers_stableDiffusion_h_ diff --git a/src/arbiterAI/providers/vibevoice.cpp b/src/arbiterAI/providers/vibevoice.cpp new file mode 100644 index 0000000..914ec6e --- /dev/null +++ b/src/arbiterAI/providers/vibevoice.cpp @@ -0,0 +1,132 @@ +#include "arbiterAI/providers/vibevoice.h" + +#include + +#include +#include +#include + +namespace arbiterAI +{ + +VibeVoice::VibeVoice() + : BaseProvider("vibevoice") +{ +} + +VibeVoice::~VibeVoice() +{ + std::lock_guard lock(m_mutex); + if(!m_loadedModel.empty()) + { + vv_capi_unload(); + m_loadedModel.clear(); + } +} + +ErrorCode VibeVoice::completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode VibeVoice::streamingCompletion(const CompletionRequest &request, + std::function callback) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode VibeVoice::getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) +{ + return ErrorCode::NotImplemented; +} + +std::string VibeVoice::resolveModelPath(const ModelInfo &model) +{ + return resolveDownloadableModelFile(model); +} + +ErrorCode VibeVoice::synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) +{ + std::string ttsModel=resolveModelPath(model); + if(ttsModel.empty()) + { + spdlog::warn("VibeVoice provider: no model file configured for '{}'", model.model); + return ErrorCode::ModelNotFound; + } + + namespace fs=std::filesystem; + fs::path modelDir=fs::path(ttsModel).parent_path(); + std::string tokenizer=(modelDir/"tokenizer.gguf").string(); + + // Voice: an explicit path in the request wins; otherwise a named voice + // (voice-.gguf) if it exists; otherwise the default voice.gguf. + std::string voicePath; + if(request.voice.find('/')!=std::string::npos) + { + voicePath=request.voice; + } + else + { + fs::path named=modelDir/("voice-"+request.voice+".gguf"); + voicePath=fs::exists(named) ? named.string() : (modelDir/"voice.gguf").string(); + } + + std::lock_guard lock(m_mutex); + + if(m_loadedModel!=ttsModel) + { + if(!m_loadedModel.empty()) + vv_capi_unload(); + int rc=vv_capi_load(ttsModel.c_str(), nullptr, tokenizer.c_str(), voicePath.c_str(), 0); + if(rc!=0) + { + spdlog::warn("VibeVoice provider: vv_capi_load failed (rc={}) for '{}'", rc, ttsModel); + m_loadedModel.clear(); + return ErrorCode::ModelLoadError; + } + m_loadedModel=ttsModel; + } + + fs::path tmpWav=fs::temp_directory_path() + /("arbiter_vv_"+std::to_string(m_tempCounter.fetch_add(1))+".wav"); + + // 0 selects vibevoice defaults: 20 diffusion steps, cfg 1.3, 200 max frames, + // random seed. + int rc=vv_capi_tts(request.input.c_str(), voicePath.c_str(), nullptr, 0, + tmpWav.string().c_str(), 0, 0.0f, 0, 0); + if(rc!=0) + { + spdlog::warn("VibeVoice provider: vv_capi_tts failed (rc={})", rc); + std::error_code ec; + fs::remove(tmpWav, ec); + return ErrorCode::GenerationError; + } + + std::ifstream in(tmpWav, std::ios::binary); + if(!in) + { + spdlog::warn("VibeVoice provider: could not read output WAV '{}'", tmpWav.string()); + return ErrorCode::GenerationError; + } + response.audio.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); + in.close(); + + std::error_code ec; + fs::remove(tmpWav, ec); + + if(response.audio.empty()) + return ErrorCode::GenerationError; + + response.format="wav"; + response.model=model.model; + response.provider="vibevoice"; + + return ErrorCode::Success; +} + +} // namespace arbiterAI diff --git a/src/arbiterAI/providers/vibevoice.h b/src/arbiterAI/providers/vibevoice.h new file mode 100644 index 0000000..8333d39 --- /dev/null +++ b/src/arbiterAI/providers/vibevoice.h @@ -0,0 +1,58 @@ +#ifndef _arbiterAI_providers_vibevoice_h_ +#define _arbiterAI_providers_vibevoice_h_ + +#include "arbiterAI/providers/baseProvider.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include + +namespace arbiterAI +{ + +/** + * @class VibeVoice + * @brief Local text-to-speech provider backed by vibevoice.cpp + * + * Handles models configured with `"provider": "vibevoice"` and + * `"mode": "speech"`. The vibevoice C API (vv_capi_*) is a global singleton, so + * this provider serializes all synthesis and reloads the engine when a different + * model is requested. + * + * Model resolution: `file_path` is the TTS gguf; the tokenizer is expected at + * `tokenizer.gguf` and the voice at `voice.gguf` (or `voice-.gguf`, or an + * explicit path in the request `voice`) alongside it. Output is a 24 kHz mono + * WAV, returned as bytes in SpeechResponse::audio. + */ +class VibeVoice : public BaseProvider +{ +public: + VibeVoice(); + ~VibeVoice() override; + + ErrorCode completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) override; + + ErrorCode streamingCompletion(const CompletionRequest &request, + std::function callback) override; + + ErrorCode getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) override; + + ErrorCode synthesizeSpeech(const SpeechRequest &request, + const ModelInfo &model, + SpeechResponse &response) override; + +private: + std::string resolveModelPath(const ModelInfo &model); + + std::mutex m_mutex; ///< Serializes the global vv_capi_* engine + std::string m_loadedModel; ///< Path of the currently loaded TTS model ("" = none) + std::atomic m_tempCounter{ 0 }; ///< Unique-name counter for temp WAV files +}; + +} // namespace arbiterAI + +#endif//_arbiterAI_providers_vibevoice_h_ diff --git a/src/arbiterAI/providers/whisper.cpp b/src/arbiterAI/providers/whisper.cpp new file mode 100644 index 0000000..8f15092 --- /dev/null +++ b/src/arbiterAI/providers/whisper.cpp @@ -0,0 +1,239 @@ +#include "arbiterAI/providers/whisper.h" + +#include + +#include +#include + +namespace arbiterAI +{ + +namespace +{ + +/// Read a little-endian unsigned integer of the given width from a byte buffer. +uint32_t readLE(const std::vector &b, size_t offset, size_t width) +{ + uint32_t value=0; + for(size_t i=0; i(b[offset+i])<<(8*i); + return value; +} + +constexpr int WHISPER_SAMPLE_RATE_HZ=16000; + +} // namespace + +Whisper::Whisper() + : BaseProvider("whisper") +{ +} + +Whisper::~Whisper() +{ + std::lock_guard lock(m_mutex); + for(auto &entry:m_contexts) + { + if(entry.second) + whisper_free(entry.second); + } + m_contexts.clear(); +} + +ErrorCode Whisper::completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) +{ + // whisper is a speech-to-text engine; text completion is not supported. + return ErrorCode::NotImplemented; +} + +ErrorCode Whisper::streamingCompletion(const CompletionRequest &request, + std::function callback) +{ + return ErrorCode::NotImplemented; +} + +ErrorCode Whisper::getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) +{ + return ErrorCode::NotImplemented; +} + +std::string Whisper::resolveModelPath(const ModelInfo &model) +{ + return resolveDownloadableModelFile(model); +} + +whisper_context *Whisper::acquireContext(const std::string &modelPath) +{ + std::lock_guard lock(m_mutex); + + auto it=m_contexts.find(modelPath); + if(it!=m_contexts.end()) + return it->second; + + whisper_context_params cparams=whisper_context_default_params(); + whisper_context *ctx=whisper_init_from_file_with_params(modelPath.c_str(), cparams); + if(!ctx) + { + spdlog::warn("Whisper provider: failed to load model from '{}'", modelPath); + return nullptr; + } + + m_contexts.emplace(modelPath, ctx); + return ctx; +} + +ErrorCode Whisper::decodeAudio(const std::vector &bytes, + std::vector &samplesOut, std::string &error) +{ + // Minimal RIFF/WAVE parser: supports uncompressed 16-bit PCM, 16 kHz, mono. + if(bytes.size()<44 + || std::memcmp(bytes.data(), "RIFF", 4)!=0 + || std::memcmp(bytes.data()+8, "WAVE", 4)!=0) + { + error="Unsupported audio: expected a WAV (RIFF/WAVE) file"; + return ErrorCode::InvalidRequest; + } + + uint16_t audioFormat=0, numChannels=0, bitsPerSample=0; + uint32_t sampleRate=0; + size_t dataOffset=0, dataSize=0; + bool haveFmt=false, haveData=false; + + // Walk the chunk list starting after the 12-byte RIFF header. + size_t pos=12; + while(pos+8<=bytes.size()) + { + const char *id=reinterpret_cast(bytes.data()+pos); + uint32_t chunkSize=readLE(bytes, pos+4, 4); + size_t body=pos+8; + + if(std::memcmp(id, "fmt ", 4)==0 && body+16<=bytes.size()) + { + audioFormat=static_cast(readLE(bytes, body, 2)); + numChannels=static_cast(readLE(bytes, body+2, 2)); + sampleRate=readLE(bytes, body+4, 4); + bitsPerSample=static_cast(readLE(bytes, body+14, 2)); + haveFmt=true; + } + else if(std::memcmp(id, "data", 4)==0) + { + dataOffset=body; + dataSize=std::min(chunkSize, bytes.size()-body); + haveData=true; + } + + // Chunks are word-aligned (padded to even size). + pos=body+chunkSize+(chunkSize&1); + } + + if(!haveFmt || !haveData) + { + error="Malformed WAV: missing 'fmt ' or 'data' chunk"; + return ErrorCode::InvalidRequest; + } + if(audioFormat!=1 || bitsPerSample!=16) + { + error="Unsupported WAV: only uncompressed 16-bit PCM is supported"; + return ErrorCode::InvalidRequest; + } + if(numChannels!=1 || sampleRate!=static_cast(WHISPER_SAMPLE_RATE_HZ)) + { + error="Unsupported WAV: expected 16 kHz mono (transcode before calling)"; + return ErrorCode::InvalidRequest; + } + + const size_t sampleCount=dataSize/2; + samplesOut.resize(sampleCount); + for(size_t i=0; i(readLE(bytes, dataOffset+i*2, 2)); + samplesOut[i]=static_cast(s)/32768.0f; + } + + return ErrorCode::Success; +} + +ErrorCode Whisper::transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) +{ + std::string modelPath=resolveModelPath(model); + if(modelPath.empty()) + { + spdlog::warn("Whisper provider: no model file configured for '{}'", model.model); + return ErrorCode::ModelNotFound; + } + + std::vector samples; + std::string decodeError; + ErrorCode decodeResult=decodeAudio(request.audio, samples, decodeError); + if(decodeResult!=ErrorCode::Success) + { + spdlog::warn("Whisper provider: {}", decodeError); + return decodeResult; + } + + whisper_context *ctx=acquireContext(modelPath); + if(!ctx) + return ErrorCode::ModelLoadError; + + whisper_full_params wparams=whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + wparams.print_progress=false; + wparams.print_realtime=false; + wparams.print_special=false; + wparams.print_timestamps=false; + + // Language selection: request hint, else the model's whisper_options default, + // else auto-detect. `translate` comes from whisper_options. + std::string defaultLanguage="auto"; + bool translate=false; + if(model.whisperOptions.is_object()) + { + defaultLanguage=model.whisperOptions.value("language", std::string("auto")); + translate=model.whisperOptions.value("translate", false); + } + std::string language=request.language.value_or(defaultLanguage); + wparams.language=language.c_str(); + wparams.detect_language=(language=="auto"); + wparams.translate=translate; + + // whisper_full is not reentrant on a shared context — serialize inference. + { + std::lock_guard lock(m_inferenceMutex); + int rc=whisper_full(ctx, wparams, samples.data(), static_cast(samples.size())); + if(rc!=0) + { + spdlog::warn("Whisper provider: whisper_full failed (rc={})", rc); + return ErrorCode::GenerationError; + } + + std::string text; + int segments=whisper_full_n_segments(ctx); + for(int i=0; i=0) + { + const char *langStr=whisper_lang_str(langId); + if(langStr) + response.language=langStr; + } + } + + response.model=model.model; + response.provider="whisper"; + response.duration=static_cast(samples.size())/WHISPER_SAMPLE_RATE_HZ; + + return ErrorCode::Success; +} + +} // namespace arbiterAI diff --git a/src/arbiterAI/providers/whisper.h b/src/arbiterAI/providers/whisper.h new file mode 100644 index 0000000..2c83066 --- /dev/null +++ b/src/arbiterAI/providers/whisper.h @@ -0,0 +1,72 @@ +#ifndef _arbiterAI_providers_whisper_h_ +#define _arbiterAI_providers_whisper_h_ + +#include "arbiterAI/providers/baseProvider.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include +#include + +struct whisper_context; + +namespace arbiterAI +{ + +/** + * @class Whisper + * @brief Local speech-to-text provider backed by whisper.cpp + * + * Handles models configured with `"provider": "whisper"` and + * `"mode": "transcription"`. The GGUF/bin model path is taken from the model + * config (`file_path`, or the primary variant download filename resolved under + * the storage root). Loaded whisper contexts are cached per model file path and + * reused across requests. + * + * First-cut audio support: 16 kHz mono 16-bit PCM WAV. Other formats / sample + * rates return ErrorCode::InvalidRequest (transcode before calling). Resampling + * and compressed-format decoding are follow-ups. + */ +class Whisper : public BaseProvider +{ +public: + Whisper(); + ~Whisper() override; + + ErrorCode completion(const CompletionRequest &request, + const ModelInfo &model, + CompletionResponse &response) override; + + ErrorCode streamingCompletion(const CompletionRequest &request, + std::function callback) override; + + ErrorCode getEmbeddings(const EmbeddingRequest &request, + EmbeddingResponse &response) override; + + ErrorCode transcribe(const AudioTranscriptionRequest &request, + const ModelInfo &model, + AudioTranscriptionResponse &response) override; + +private: + /// Resolve the on-disk whisper model file for a model config. + /// Returns empty string if no usable path is configured. + std::string resolveModelPath(const ModelInfo &model); + + /// Get (loading on first use) a whisper context for the given model file. + /// Returns nullptr on load failure. Thread-safe. + whisper_context *acquireContext(const std::string &modelPath); + + /// Decode raw audio file bytes into 16 kHz mono f32 PCM samples. + /// Only uncompressed 16-bit PCM WAV at 16 kHz mono is supported for now. + static ErrorCode decodeAudio(const std::vector &bytes, + std::vector &samplesOut, std::string &error); + + std::mutex m_mutex; ///< Guards the context cache + std::mutex m_inferenceMutex; ///< Serializes whisper_full (not reentrant on a shared ctx) — v1 limitation + std::map m_contexts; +}; + +} // namespace arbiterAI + +#endif//_arbiterAI_providers_whisper_h_ diff --git a/src/arbiterAI/storageManager.h b/src/arbiterAI/storageManager.h index 6718549..b6ed381 100644 --- a/src/arbiterAI/storageManager.h +++ b/src/arbiterAI/storageManager.h @@ -65,6 +65,9 @@ class StorageManager { /// Initialize with the models directory path. void initialize(const std::filesystem::path &modelsDir); + /// Get the configured models directory (where downloaded weights live). + std::filesystem::path getModelsDir() const { return m_modelsDir; } + /// Shut down the background flush/cleanup timers. void shutdown(); diff --git a/src/arbiterAI/telemetryCollector.cpp b/src/arbiterAI/telemetryCollector.cpp index 7648345..c012a19 100644 --- a/src/arbiterAI/telemetryCollector.cpp +++ b/src/arbiterAI/telemetryCollector.cpp @@ -96,6 +96,12 @@ SystemSnapshot TelemetryCollector::getSnapshot() const genSum+=stat.generationTokensPerSecond; genCount++; } + + // Modality-aware aggregates + snapshot.requestsByModality[stat.modality]++; + snapshot.imagesGenerated+=stat.imagesGenerated; + snapshot.audioSecondsTranscribed+=stat.audioSeconds; + snapshot.charactersSynthesized+=stat.audioCharacters; } } diff --git a/src/arbiterAI/telemetryCollector.h b/src/arbiterAI/telemetryCollector.h index abf2d08..afdc100 100644 --- a/src/arbiterAI/telemetryCollector.h +++ b/src/arbiterAI/telemetryCollector.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace arbiterAI { @@ -28,6 +29,7 @@ struct LoadedModel { struct InferenceStats { std::string model; std::string variant; + std::string modality{ "chat" }; // chat, embedding, transcription, speech, image uint64_t jobId=0; bool cancelled=false; double tokensPerSecond=0.0; @@ -35,6 +37,15 @@ struct InferenceStats { double generationTokensPerSecond=0.0; // generation speed (tokens out / sec) int promptTokens=0; int completionTokens=0; + // Non-token modality counters (0 for chat/embedding). + int imagesGenerated=0; // image: number of images produced + int imageSteps=0; // image: diffusion steps used (per image) + double audioSeconds=0.0; // transcription: input audio duration (s) + int audioCharacters=0; // speech: input characters synthesized + // Non-token modality speed metrics (0 when N/A). + double realtimeFactor=0.0; // transcription: audioSeconds / wall-time (x realtime) + double stepsPerSecond=0.0; // image: total diffusion steps / wall-time + double cost=0.0; // estimated cost for this request double latencyMs=0.0; // time to first token double totalTimeMs=0.0; // total request time double promptTimeMs=0.0; // time spent processing prompt @@ -56,6 +67,11 @@ struct SystemSnapshot { double avgPromptTokensPerSecond=0.0; double avgGenerationTokensPerSecond=0.0; int activeRequests=0; + // Modality-aware aggregates over the recent window (5 min). + std::map requestsByModality; // request count per modality + int imagesGenerated=0; // total images generated + double audioSecondsTranscribed=0.0; // total STT audio seconds + int charactersSynthesized=0; // total TTS input characters }; class TelemetryCollector { diff --git a/src/server/routes.cpp b/src/server/routes.cpp index b77d018..f151f7f 100644 --- a/src/server/routes.cpp +++ b/src/server/routes.cpp @@ -670,6 +670,25 @@ nlohmann::json buildServerConfigResponse(const nlohmann::json &cfg) return response; } +/// Serialize a JSON body that may contain model-generated text. Model output can +/// contain invalid UTF-8 — a multi-byte character clipped at a token/length +/// boundary, or a llama.cpp byte-fallback token — and nlohmann's default (strict) +/// dump() throws json.exception.type_error.316 on that. Uncaught, it terminates +/// the whole server (SIGABRT), killing every in-flight request. Serialize with the +/// 'replace' error handler and never let a serialization error escape. +std::string safeDump(const nlohmann::json &j) +{ + try + { + return j.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + } + catch(const std::exception &e) + { + spdlog::error("safeDump: failed to serialize response body: {}", e.what()); + return "{}"; + } +} + /// Generate a unique ID with the given prefix (e.g., "chatcmpl-"). std::string generateId(const std::string &prefix="chatcmpl-") { @@ -920,7 +939,17 @@ nlohmann::json inferenceStatsToJson(const InferenceStats &s) {"latency_ms", s.latencyMs}, {"total_time_ms", s.totalTimeMs}, {"prompt_time_ms", s.promptTimeMs}, - {"generation_time_ms", s.generationTimeMs} + {"generation_time_ms", s.generationTimeMs}, + // Modality-aware fields (non-token engines). Clients group by + // (model, variant, hardware) to benchmark/normalize STT/image speed. + {"modality", s.modality}, + {"images_generated", s.imagesGenerated}, + {"image_steps", s.imageSteps}, + {"audio_seconds", s.audioSeconds}, + {"audio_characters", s.audioCharacters}, + {"realtime_factor", s.realtimeFactor}, + {"steps_per_second", s.stepsPerSecond}, + {"cost", s.cost} }; } @@ -1476,6 +1505,11 @@ void registerRoutes(httplib::Server &server) // Embeddings (OpenAI-compatible) server.Post("/v1/embeddings", handleEmbeddings); + // Multimodal: audio / image (OpenAI-compatible) + server.Post("/v1/audio/transcriptions", handleAudioTranscriptions); + server.Post("/v1/audio/speech", handleAudioSpeech); + server.Post("/v1/images/generations", handleImageGenerations); + // Model management server.Get("/api/models", handleGetModels); server.Get("/api/models/loaded", handleGetLoadedModels); @@ -1800,7 +1834,7 @@ void handleChatCompletions(const httplib::Request &req, httplib::Response &res) {"finish_reason", nullptr} }}} }; - std::string line="data: "+sseChunk.dump()+"\n\n"; + std::string line="data: "+safeDump(sseChunk)+"\n\n"; if(!sink.write(line.c_str(), line.length())) { // Client disconnected — cancel the job @@ -1843,7 +1877,7 @@ void handleChatCompletions(const httplib::Request &req, httplib::Response &res) {"finish_reason", nullptr} }}} }; - std::string line="data: "+sseChunk.dump()+"\n\n"; + std::string line="data: "+safeDump(sseChunk)+"\n\n"; sink.write(line.c_str(), line.length()); }; @@ -1978,8 +2012,13 @@ void handleChatCompletions(const httplib::Request &req, httplib::Response &res) status=503; } + std::string errMsg="Completion failed: "+errCode; + if(!job->errorDetail.empty()) + { + errMsg+=" — "+job->errorDetail; + } res.status=status; - res.set_content(errorJson("Completion failed: "+errCode, errType, "", errCode).dump(), "application/json"); + res.set_content(errorJson(errMsg, errType, "", errCode).dump(), "application/json"); return; } @@ -2031,7 +2070,12 @@ void handleChatCompletions(const httplib::Request &req, httplib::Response &res) if(err!=ErrorCode::Success) { std::string errCode=errorCodeToString(err); - std::string errBody=errorJson("Completion failed: "+errCode, "server_error", "", errCode).dump(); + std::string errMsg="Completion failed: "+errCode; + if(!job->errorDetail.empty()) + { + errMsg+=" — "+job->errorDetail; + } + std::string errBody=errorJson(errMsg, "server_error", "", errCode).dump(); sink.write(errBody.c_str(), errBody.length()); sink.done(); return true; @@ -2123,7 +2167,7 @@ void handleChatCompletions(const httplib::Request &req, httplib::Response &res) }} }; - std::string body=responseJson.dump(); + std::string body=safeDump(responseJson); sink.write(body.c_str(), body.length()); sink.done(); return true; @@ -2255,7 +2299,7 @@ void handleChatCompletions(const httplib::Request &req, httplib::Response &res) }} }; - res.set_content(responseJson.dump(), "application/json"); + res.set_content(safeDump(responseJson), "application/json"); } } @@ -2437,7 +2481,255 @@ void handleEmbeddings(const httplib::Request &req, httplib::Response &res) }} }; - res.set_content(responseJson.dump(), "application/json"); + res.set_content(safeDump(responseJson), "application/json"); +} + +// ========== Multimodal: Audio / Image (OpenAI-compatible) ========== + +namespace +{ + +/// Map an audio container/codec name to a MIME type for the response body. +std::string audioMimeType(const std::string &format) +{ + if(format=="mp3") return "audio/mpeg"; + if(format=="opus") return "audio/opus"; + if(format=="aac") return "audio/aac"; + if(format=="flac") return "audio/flac"; + if(format=="wav") return "audio/wav"; + if(format=="pcm") return "audio/L16"; + return "application/octet-stream"; +} + +/// Verify the requested model exists and is configured for the expected +/// modality (ModelInfo::mode). On mismatch, fills res with an error and returns +/// false. This is where mode is authoritative: it prevents e.g. pointing the +/// transcription endpoint at a chat model. +bool requireModelMode(const std::string &model, const char *expectedMode, httplib::Response &res) +{ + auto info=ModelManager::instance().getModelInfo(model); + if(!info) + { + res.status=404; + res.set_content(errorJson("Model '"+model+"' not found", "invalid_request_error", "model", "model_not_found").dump(), "application/json"); + return false; + } + if(info->mode!=expectedMode) + { + res.status=400; + res.set_content(errorJson("Model '"+model+"' has mode '"+info->mode + +"', but this endpoint requires mode '"+expectedMode+"'", + "invalid_request_error", "model", "wrong_mode").dump(), "application/json"); + return false; + } + return true; +} + +/// Translate a multimodal ErrorCode into an OpenAI-style HTTP status. +int multimodalHttpStatus(ErrorCode err) +{ + switch(err) + { + case ErrorCode::UnknownModel: + case ErrorCode::InvalidRequest: + case ErrorCode::InvalidResponse: + return 400; + case ErrorCode::NotImplemented: + case ErrorCode::UnsupportedProvider: + return 501; + case ErrorCode::ApiKeyNotFound: + return 401; + default: + return 500; + } +} + +} // namespace + +void handleAudioTranscriptions(const httplib::Request &req, httplib::Response &res) +{ + if(!req.has_file("file")) + { + res.status=400; + res.set_content(errorJson("Missing 'file' in multipart form data", "invalid_request_error", "file", "missing_field").dump(), "application/json"); + return; + } + + const auto &file=req.get_file_value("file"); + + AudioTranscriptionRequest request; + request.audio.assign(file.content.begin(), file.content.end()); + if(!file.filename.empty()) + request.filename=file.filename; + + if(req.has_file("model")) + request.model=req.get_file_value("model").content; + if(request.model.empty()) + { + res.status=400; + res.set_content(errorJson("Missing 'model' in multipart form data", "invalid_request_error", "model", "missing_field").dump(), "application/json"); + return; + } + + if(!requireModelMode(request.model, modes::Transcription, res)) + return; + + if(req.has_file("language")) + request.language=req.get_file_value("language").content; + if(req.has_file("prompt")) + request.prompt=req.get_file_value("prompt").content; + if(req.has_file("response_format")) + request.responseFormat=req.get_file_value("response_format").content; + if(req.has_file("temperature")) + { + try { request.temperature=std::stod(req.get_file_value("temperature").content); } + catch(const std::exception &) {} + } + + AudioTranscriptionResponse response; + ErrorCode err=ArbiterAI::instance().transcribe(request, response); + if(err!=ErrorCode::Success) + { + res.status=multimodalHttpStatus(err); + res.set_content(errorJson("Transcription failed: "+errorCodeToString(err), "server_error", "", errorCodeToString(err)).dump(), "application/json"); + return; + } + + // response_format=text (or srt/vtt) returns the raw transcript; the JSON + // formats return an object. + const std::string format=request.responseFormat.value_or("json"); + if(format=="text" || format=="srt" || format=="vtt") + { + res.set_content(response.text, "text/plain"); + return; + } + + nlohmann::json responseJson={{"text", response.text}}; + if(!response.language.empty()) + responseJson["language"]=response.language; + if(response.duration>0.0) + responseJson["duration"]=response.duration; + res.set_content(safeDump(responseJson), "application/json"); +} + +void handleAudioSpeech(const httplib::Request &req, httplib::Response &res) +{ + nlohmann::json requestJson; + try + { + requestJson=nlohmann::json::parse(req.body); + } + catch(const nlohmann::json::parse_error &) + { + res.status=400; + res.set_content(errorJson("Failed to parse JSON body", "invalid_request_error", "", "parse_error").dump(), "application/json"); + return; + } + + SpeechRequest request; + try + { + request.model=requestJson.at("model").get(); + request.input=requestJson.at("input").get(); + if(requestJson.contains("voice")) + request.voice=requestJson.at("voice").get(); + if(requestJson.contains("response_format")) + request.responseFormat=requestJson.at("response_format").get(); + if(requestJson.contains("speed")) + request.speed=requestJson.at("speed").get(); + } + catch(const nlohmann::json::exception &e) + { + res.status=400; + res.set_content(errorJson(std::string("JSON validation error: ")+e.what(), "invalid_request_error", "", "invalid_request").dump(), "application/json"); + return; + } + + if(!requireModelMode(request.model, modes::Speech, res)) + return; + + SpeechResponse response; + ErrorCode err=ArbiterAI::instance().synthesizeSpeech(request, response); + if(err!=ErrorCode::Success) + { + res.status=multimodalHttpStatus(err); + res.set_content(errorJson("Speech synthesis failed: "+errorCodeToString(err), "server_error", "", errorCodeToString(err)).dump(), "application/json"); + return; + } + + res.set_content(reinterpret_cast(response.audio.data()), + response.audio.size(), audioMimeType(response.format)); +} + +void handleImageGenerations(const httplib::Request &req, httplib::Response &res) +{ + nlohmann::json requestJson; + try + { + requestJson=nlohmann::json::parse(req.body); + } + catch(const nlohmann::json::parse_error &) + { + res.status=400; + res.set_content(errorJson("Failed to parse JSON body", "invalid_request_error", "", "parse_error").dump(), "application/json"); + return; + } + + ImageGenerationRequest request; + try + { + request.model=requestJson.at("model").get(); + request.prompt=requestJson.at("prompt").get(); + if(requestJson.contains("negative_prompt")) + request.negativePrompt=requestJson.at("negative_prompt").get(); + if(requestJson.contains("n")) + request.n=requestJson.at("n").get(); + if(requestJson.contains("size")) + request.size=requestJson.at("size").get(); + if(requestJson.contains("steps")) + request.steps=requestJson.at("steps").get(); + if(requestJson.contains("seed")) + request.seed=requestJson.at("seed").get(); + if(requestJson.contains("response_format")) + request.responseFormat=requestJson.at("response_format").get(); + } + catch(const nlohmann::json::exception &e) + { + res.status=400; + res.set_content(errorJson(std::string("JSON validation error: ")+e.what(), "invalid_request_error", "", "invalid_request").dump(), "application/json"); + return; + } + + if(!requireModelMode(request.model, modes::Image, res)) + return; + + ImageGenerationResponse response; + ErrorCode err=ArbiterAI::instance().generateImage(request, response); + if(err!=ErrorCode::Success) + { + res.status=multimodalHttpStatus(err); + res.set_content(errorJson("Image generation failed: "+errorCodeToString(err), "server_error", "", errorCodeToString(err)).dump(), "application/json"); + return; + } + + nlohmann::json data=nlohmann::json::array(); + for(const GeneratedImage &image:response.images) + { + nlohmann::json item=nlohmann::json::object(); + if(!image.url.empty()) + item["url"]=image.url; + if(!image.b64Json.empty()) + item["b64_json"]=image.b64Json; + if(!image.revisedPrompt.empty()) + item["revised_prompt"]=image.revisedPrompt; + data.push_back(item); + } + + nlohmann::json responseJson={ + {"created", std::time(nullptr)}, + {"data", data} + }; + res.set_content(safeDump(responseJson), "application/json"); } // ========== Health ========== @@ -3269,7 +3561,13 @@ void handleGetStats(const httplib::Request &, httplib::Response &res) {"avg_tokens_per_second", snapshot.avgTokensPerSecond}, {"avg_prompt_tokens_per_second", snapshot.avgPromptTokensPerSecond}, {"avg_generation_tokens_per_second", snapshot.avgGenerationTokensPerSecond}, - {"active_requests", snapshot.activeRequests} + {"active_requests", snapshot.activeRequests}, + {"modality", { + {"requests_by_modality", snapshot.requestsByModality}, + {"images_generated", snapshot.imagesGenerated}, + {"audio_seconds_transcribed", snapshot.audioSecondsTranscribed}, + {"characters_synthesized", snapshot.charactersSynthesized} + }} }; res.set_content(response.dump(), "application/json"); diff --git a/src/server/routes.h b/src/server/routes.h index 6b20831..6e33041 100644 --- a/src/server/routes.h +++ b/src/server/routes.h @@ -27,6 +27,12 @@ void handleGetModelV1(const httplib::Request &req, httplib::Response &res); void handleEmbeddings(const httplib::Request &req, httplib::Response &res); +// ========== Multimodal: Audio / Image (OpenAI-compatible) ========== + +void handleAudioTranscriptions(const httplib::Request &req, httplib::Response &res); +void handleAudioSpeech(const httplib::Request &req, httplib::Response &res); +void handleImageGenerations(const httplib::Request &req, httplib::Response &res); + // ========== Health ========== void handleHealth(const httplib::Request &req, httplib::Response &res); diff --git a/tests/arbiterAITests.cpp b/tests/arbiterAITests.cpp index bc80927..05928c8 100644 --- a/tests/arbiterAITests.cpp +++ b/tests/arbiterAITests.cpp @@ -1,5 +1,6 @@ #include "arbiterAI/arbiterAI.h" #include "arbiterAI/modelManager.h" +#include "arbiterAI/telemetryCollector.h" #include #include @@ -51,4 +52,118 @@ TEST_F(ArbiterAITest, SupportModelDownload) EXPECT_FALSE(ai.supportModelDownload("unknown-provider")); } +// --- Multimodal dispatch + modality-aware pricing --- + +class ArbiterAIMultimodalTest : public ::testing::Test +{ +protected: + void SetUp() override + { + ModelManager::reset(); + m_wasInitialized=ArbiterAI::instance().initialized; + ArbiterAI::instance().initialized=true; + } + + void TearDown() override + { + ArbiterAI::instance().initialized=m_wasInitialized; + } + + bool m_wasInitialized=false; +}; + +TEST_F(ArbiterAIMultimodalTest, GenerateImageComputesCost) +{ + ModelInfo model; + model.model="mock-image"; + model.provider="mock"; + model.mode=modes::Image; + model.pricing.image_cost=0.04; + ModelManager::instance().addModel(model); + + ImageGenerationRequest request; + request.model="mock-image"; + request.prompt="a blue sphere"; + request.n=3; + + ImageGenerationResponse response; + ErrorCode err=ArbiterAI::instance().generateImage(request, response); + + EXPECT_EQ(err, ErrorCode::Success); + ASSERT_EQ(response.images.size(), 3u); + EXPECT_DOUBLE_EQ(response.cost, 3*0.04); +} + +TEST_F(ArbiterAIMultimodalTest, SynthesizeSpeechComputesCost) +{ + ModelInfo model; + model.model="mock-speech"; + model.provider="mock"; + model.mode=modes::Speech; + model.pricing.audio_output_cost_per_character=0.001; + ModelManager::instance().addModel(model); + + SpeechRequest request; + request.model="mock-speech"; + request.input="hello"; // 5 characters + + SpeechResponse response; + ErrorCode err=ArbiterAI::instance().synthesizeSpeech(request, response); + + EXPECT_EQ(err, ErrorCode::Success); + EXPECT_DOUBLE_EQ(response.cost, 5*0.001); +} + +TEST_F(ArbiterAIMultimodalTest, TranscribeSucceeds) +{ + ModelInfo model; + model.model="mock-transcribe"; + model.provider="mock"; + model.mode=modes::Transcription; + ModelManager::instance().addModel(model); + + AudioTranscriptionRequest request; + request.model="mock-transcribe"; + request.audio={0x01, 0x02, 0x03}; + + AudioTranscriptionResponse response; + ErrorCode err=ArbiterAI::instance().transcribe(request, response); + + EXPECT_EQ(err, ErrorCode::Success); + EXPECT_FALSE(response.text.empty()); +} + +TEST_F(ArbiterAIMultimodalTest, UnknownModelRejected) +{ + ImageGenerationRequest request; + request.model="does-not-exist"; + request.prompt="x"; + + ImageGenerationResponse response; + EXPECT_EQ(ArbiterAI::instance().generateImage(request, response), ErrorCode::UnknownModel); +} + +TEST_F(ArbiterAIMultimodalTest, DispatchRecordsModalityTelemetry) +{ + TelemetryCollector::reset(); + + ModelInfo model; + model.model="mock-image-tel"; + model.provider="mock"; + model.mode=modes::Image; + ModelManager::instance().addModel(model); + + ImageGenerationRequest request; + request.model="mock-image-tel"; + request.prompt="a green triangle"; + request.n=2; + + ImageGenerationResponse response; + ASSERT_EQ(ArbiterAI::instance().generateImage(request, response), ErrorCode::Success); + + SystemSnapshot snap=TelemetryCollector::instance().getSnapshot(); + EXPECT_EQ(snap.requestsByModality["image"], 1); + EXPECT_EQ(snap.imagesGenerated, 2); +} + } // namespace arbiterAI \ No newline at end of file diff --git a/tests/mockProviderTests.cpp b/tests/mockProviderTests.cpp index 2c147f0..3e280dd 100644 --- a/tests/mockProviderTests.cpp +++ b/tests/mockProviderTests.cpp @@ -390,4 +390,71 @@ TEST_F(MockProviderTest, SpecialCharactersInEcho) EXPECT_EQ(response.text, R"(Special chars: !@#$%^&*(){}[]|\"')"); } +// --- Multimodal (STT / TTS / Image) Tests --- + +TEST_F(MockProviderTest, TranscribeDefault) +{ + AudioTranscriptionRequest request; + request.model = "mock-model"; + request.audio = {0x01, 0x02, 0x03, 0x04}; + + AudioTranscriptionResponse response; + ErrorCode result = provider->transcribe(request, modelInfo, response); + + EXPECT_EQ(result, ErrorCode::Success); + EXPECT_THAT(response.text, ::testing::HasSubstr("mock transcription")); + EXPECT_EQ(response.provider, "mock"); + EXPECT_EQ(response.language, "en"); +} + +TEST_F(MockProviderTest, TranscribeEchoPrompt) +{ + AudioTranscriptionRequest request; + request.model = "mock-model"; + request.audio = {0x01, 0x02}; + request.prompt = "hint hello world"; + request.language = "fr"; + + AudioTranscriptionResponse response; + ErrorCode result = provider->transcribe(request, modelInfo, response); + + EXPECT_EQ(result, ErrorCode::Success); + EXPECT_EQ(response.text, "hello world"); + EXPECT_EQ(response.language, "fr"); +} + +TEST_F(MockProviderTest, SynthesizeSpeech) +{ + SpeechRequest request; + request.model = "mock-model"; + request.input = "hello"; + request.responseFormat = "mp3"; + + SpeechResponse response; + ErrorCode result = provider->synthesizeSpeech(request, modelInfo, response); + + EXPECT_EQ(result, ErrorCode::Success); + ASSERT_EQ(response.audio.size(), 5u); + EXPECT_EQ(std::string(response.audio.begin(), response.audio.end()), "hello"); + EXPECT_EQ(response.format, "mp3"); + EXPECT_EQ(response.provider, "mock"); +} + +TEST_F(MockProviderTest, GenerateImage) +{ + ImageGenerationRequest request; + request.model = "mock-model"; + request.prompt = "a red cube"; + request.n = 2; + + ImageGenerationResponse response; + ErrorCode result = provider->generateImage(request, modelInfo, response); + + EXPECT_EQ(result, ErrorCode::Success); + ASSERT_EQ(response.images.size(), 2u); + EXPECT_FALSE(response.images[0].b64Json.empty()); + EXPECT_EQ(response.images[0].revisedPrompt, "a red cube"); + EXPECT_EQ(response.provider, "mock"); +} + } // namespace arbiterAI diff --git a/tests/orpheusProviderTests.cpp b/tests/orpheusProviderTests.cpp new file mode 100644 index 0000000..a87ea5c --- /dev/null +++ b/tests/orpheusProviderTests.cpp @@ -0,0 +1,124 @@ +/** + * @file orpheusProviderTests.cpp + * @brief Unit tests for the local Orpheus TTS provider (dlopen engine). + * + * Only compiled when ARBITERAI_ENABLE_ORPHEUS is on. The end-to-end synthesis + * test skips unless both ORPHEUS_MODEL_PATH and ARBITER_ORPHEUS_ENGINE (a built + * liborpheus_engine.so) are present, so the suite stays green without them. + */ + +#include "arbiterAI/providers/orpheus.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include + +namespace arbiterAI +{ + +class OrpheusProviderTest : public ::testing::Test +{ +protected: + Orpheus provider; + ModelInfo model; + + void SetUp() override + { + model.model="orpheus-local"; + model.provider="orpheus"; + model.mode=modes::Speech; + const char *path=std::getenv("ORPHEUS_MODEL_PATH"); + if(path) + model.filePath=std::string(path); + } +}; + +// Orpheus is TTS-only: text methods must report NotImplemented. +TEST_F(OrpheusProviderTest, TextMethodsNotImplemented) +{ + CompletionRequest creq; + CompletionResponse cresp; + EXPECT_EQ(provider.completion(creq, model, cresp), ErrorCode::NotImplemented); + + EmbeddingRequest ereq; + EmbeddingResponse eresp; + EXPECT_EQ(provider.getEmbeddings(ereq, eresp), ErrorCode::NotImplemented); +} + +// No model file configured → ModelNotFound before touching the engine. +TEST_F(OrpheusProviderTest, MissingModelPathReported) +{ + ModelInfo noPath; + noPath.model="orpheus-nopath"; + noPath.provider="orpheus"; + noPath.mode=modes::Speech; + + SpeechRequest req; + req.model="orpheus-nopath"; + req.input="hello world"; + + SpeechResponse resp; + EXPECT_EQ(provider.synthesizeSpeech(req, noPath, resp), ErrorCode::ModelNotFound); +} + +// With a model configured but no engine library available, the provider must +// fail cleanly (ModelLoadError) rather than crash. +TEST_F(OrpheusProviderTest, MissingEngineReportedCleanly) +{ + if(!model.filePath.has_value()) + GTEST_SKIP() << "ORPHEUS_MODEL_PATH not set"; + if(std::getenv("ARBITER_ORPHEUS_ENGINE")) + GTEST_SKIP() << "engine present — covered by SynthesizeProducesWav"; + + SpeechRequest req; + req.model=model.model; + req.input="hello"; + + SpeechResponse resp; + EXPECT_EQ(provider.synthesizeSpeech(req, model, resp), ErrorCode::ModelLoadError); +} + +// Proves the provider actually dlopen's + invokes the engine library: with the +// engine present, the result must NOT be ModelLoadError (engine-not-loaded) or +// ModelNotFound. A bogus model yields GenerationError (engine ran, chatllm +// rejected the model); a real model yields Success. +TEST_F(OrpheusProviderTest, ProviderLoadsRealEngine) +{ + const char *engine=std::getenv("ARBITER_ORPHEUS_ENGINE"); + if(!engine) + GTEST_SKIP() << "ARBITER_ORPHEUS_ENGINE not set"; + + if(!model.filePath.has_value()) + model.filePath=std::string("/nonexistent/orpheus.gguf"); + + SpeechRequest req; + req.model=model.model; + req.input="hello world"; + + SpeechResponse resp; + ErrorCode err=provider.synthesizeSpeech(req, model, resp); + EXPECT_NE(err, ErrorCode::ModelLoadError); // engine .so loaded fine + EXPECT_NE(err, ErrorCode::ModelNotFound); // model path was provided +} + +// End-to-end synthesis with a real model + engine library. +TEST_F(OrpheusProviderTest, SynthesizeProducesWav) +{ + const char *engine=std::getenv("ARBITER_ORPHEUS_ENGINE"); + if(!engine || !model.filePath.has_value() || !std::filesystem::exists(model.filePath.value())) + GTEST_SKIP() << "ARBITER_ORPHEUS_ENGINE / ORPHEUS_MODEL_PATH not set"; + + SpeechRequest req; + req.model=model.model; + req.input="Hello from arbiter A I."; + + SpeechResponse resp; + ErrorCode err=provider.synthesizeSpeech(req, model, resp); + EXPECT_EQ(err, ErrorCode::Success); + ASSERT_GE(resp.audio.size(), 4u); + EXPECT_EQ(std::string(resp.audio.begin(), resp.audio.begin()+4), "RIFF"); + EXPECT_EQ(resp.provider, "orpheus"); +} + +} // namespace arbiterAI diff --git a/tests/stableDiffusionProviderTests.cpp b/tests/stableDiffusionProviderTests.cpp new file mode 100644 index 0000000..c5e304e --- /dev/null +++ b/tests/stableDiffusionProviderTests.cpp @@ -0,0 +1,86 @@ +/** + * @file stableDiffusionProviderTests.cpp + * @brief Unit tests for the local stable-diffusion.cpp image provider. + * + * Only compiled when ARBITERAI_ENABLE_STABLE_DIFFUSION is on. The end-to-end + * generation test skips unless SD_MODEL_PATH points at a real model file (SD + * weights + inference are heavy), so the suite stays green without weights. + */ + +#include "arbiterAI/providers/stableDiffusion.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include + +namespace arbiterAI +{ + +class StableDiffusionProviderTest : public ::testing::Test +{ +protected: + StableDiffusion provider; + ModelInfo model; + + void SetUp() override + { + model.model="sd-local"; + model.provider="stable-diffusion"; + model.mode=modes::Image; + const char *path=std::getenv("SD_MODEL_PATH"); + if(path) + model.filePath=std::string(path); + } +}; + +// stable-diffusion is image-only: text methods must report NotImplemented. +TEST_F(StableDiffusionProviderTest, TextMethodsNotImplemented) +{ + CompletionRequest creq; + CompletionResponse cresp; + EXPECT_EQ(provider.completion(creq, model, cresp), ErrorCode::NotImplemented); + + EmbeddingRequest ereq; + EmbeddingResponse eresp; + EXPECT_EQ(provider.getEmbeddings(ereq, eresp), ErrorCode::NotImplemented); +} + +// No model file configured → ModelNotFound before any generation work. +TEST_F(StableDiffusionProviderTest, MissingModelPathReported) +{ + ModelInfo noPath; + noPath.model="sd-nopath"; + noPath.provider="stable-diffusion"; + noPath.mode=modes::Image; + + ImageGenerationRequest req; + req.model="sd-nopath"; + req.prompt="a red cube"; + + ImageGenerationResponse resp; + EXPECT_EQ(provider.generateImage(req, noPath, resp), ErrorCode::ModelNotFound); +} + +// End-to-end generation with a real model. +TEST_F(StableDiffusionProviderTest, GenerateProducesPng) +{ + if(!model.filePath.has_value() || !std::filesystem::exists(model.filePath.value())) + GTEST_SKIP() << "SD_MODEL_PATH not set or file missing"; + + ImageGenerationRequest req; + req.model=model.model; + req.prompt="a red cube on a white background"; + req.size="256x256"; + req.steps=4; + + ImageGenerationResponse resp; + ErrorCode err=provider.generateImage(req, model, resp); + EXPECT_EQ(err, ErrorCode::Success); + ASSERT_FALSE(resp.images.empty()); + // base64 PNG starts with iVBORw0KGgo (the PNG signature encoded). + EXPECT_EQ(resp.images[0].b64Json.rfind("iVBORw0KGgo", 0), 0u); + EXPECT_EQ(resp.provider, "stable-diffusion"); +} + +} // namespace arbiterAI diff --git a/tests/telemetryCollectorTests.cpp b/tests/telemetryCollectorTests.cpp index ce95299..329b8ba 100644 --- a/tests/telemetryCollectorTests.cpp +++ b/tests/telemetryCollectorTests.cpp @@ -320,4 +320,57 @@ TEST_F(TelemetryCollectorTest, SwapModelRecordsTelemetry) EXPECT_GT(swaps[0].timeMs, 0.0); } +// --- Modality-aware telemetry --- + +TEST_F(TelemetryCollectorTest, ModalityAggregatesInSnapshot) +{ + TelemetryCollector &tc=TelemetryCollector::instance(); + auto now=std::chrono::system_clock::now(); + + InferenceStats img; + img.model="dall-e-3"; + img.modality="image"; + img.imagesGenerated=2; + img.cost=0.08; + img.timestamp=now; + tc.recordInference(img); + + InferenceStats speech; + speech.model="tts-1"; + speech.modality="speech"; + speech.audioCharacters=5; + speech.timestamp=now; + tc.recordInference(speech); + + InferenceStats stt; + stt.model="whisper-1"; + stt.modality="transcription"; + stt.audioSeconds=3.0; + stt.timestamp=now; + tc.recordInference(stt); + + SystemSnapshot snap=tc.getSnapshot(); + EXPECT_EQ(snap.imagesGenerated, 2); + EXPECT_EQ(snap.charactersSynthesized, 5); + EXPECT_DOUBLE_EQ(snap.audioSecondsTranscribed, 3.0); + EXPECT_EQ(snap.requestsByModality["image"], 1); + EXPECT_EQ(snap.requestsByModality["speech"], 1); + EXPECT_EQ(snap.requestsByModality["transcription"], 1); +} + +TEST_F(TelemetryCollectorTest, ModalityDefaultsToChat) +{ + TelemetryCollector &tc=TelemetryCollector::instance(); + InferenceStats stats; + stats.model="gpt-4"; + stats.promptTokens=10; + stats.completionTokens=20; + stats.timestamp=std::chrono::system_clock::now(); + tc.recordInference(stats); + + SystemSnapshot snap=tc.getSnapshot(); + EXPECT_EQ(snap.requestsByModality["chat"], 1); + EXPECT_EQ(snap.imagesGenerated, 0); +} + } // namespace arbiterAI diff --git a/tests/vibevoiceProviderTests.cpp b/tests/vibevoiceProviderTests.cpp new file mode 100644 index 0000000..9c654a1 --- /dev/null +++ b/tests/vibevoiceProviderTests.cpp @@ -0,0 +1,86 @@ +/** + * @file vibevoiceProviderTests.cpp + * @brief Unit tests for the local vibevoice.cpp text-to-speech provider. + * + * Only compiled when ARBITERAI_ENABLE_VIBEVOICE is on. The end-to-end synthesis + * test skips unless VIBEVOICE_MODEL_PATH points at a real TTS gguf (with a + * sibling tokenizer.gguf / voice.gguf), so the suite stays green without models. + */ + +#include "arbiterAI/providers/vibevoice.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include + +namespace arbiterAI +{ + +class VibeVoiceProviderTest : public ::testing::Test +{ +protected: + VibeVoice provider; + ModelInfo model; + + void SetUp() override + { + model.model="vibevoice-local"; + model.provider="vibevoice"; + model.mode=modes::Speech; + const char *path=std::getenv("VIBEVOICE_MODEL_PATH"); + if(path) + model.filePath=std::string(path); + } +}; + +// vibevoice is TTS-only: text methods must report NotImplemented. +TEST_F(VibeVoiceProviderTest, TextMethodsNotImplemented) +{ + CompletionRequest creq; + CompletionResponse cresp; + EXPECT_EQ(provider.completion(creq, model, cresp), ErrorCode::NotImplemented); + + EmbeddingRequest ereq; + EmbeddingResponse eresp; + EXPECT_EQ(provider.getEmbeddings(ereq, eresp), ErrorCode::NotImplemented); +} + +// No model file configured → ModelNotFound before any engine work. +TEST_F(VibeVoiceProviderTest, MissingModelPathReported) +{ + ModelInfo noPath; + noPath.model="vibevoice-nopath"; + noPath.provider="vibevoice"; + noPath.mode=modes::Speech; + + SpeechRequest req; + req.model="vibevoice-nopath"; + req.input="hello world"; + + SpeechResponse resp; + EXPECT_EQ(provider.synthesizeSpeech(req, noPath, resp), ErrorCode::ModelNotFound); +} + +// End-to-end synthesis with a real model. +TEST_F(VibeVoiceProviderTest, SynthesizeProducesWav) +{ + if(!model.filePath.has_value() || !std::filesystem::exists(model.filePath.value())) + GTEST_SKIP() << "VIBEVOICE_MODEL_PATH not set or file missing"; + + SpeechRequest req; + req.model=model.model; + req.input="Speaker 0: Hello from arbiter A I."; + + SpeechResponse resp; + ErrorCode err=provider.synthesizeSpeech(req, model, resp); + EXPECT_EQ(err, ErrorCode::Success); + ASSERT_FALSE(resp.audio.empty()); + // WAV files start with the "RIFF" magic. + ASSERT_GE(resp.audio.size(), 4u); + EXPECT_EQ(std::string(resp.audio.begin(), resp.audio.begin()+4), "RIFF"); + EXPECT_EQ(resp.format, "wav"); + EXPECT_EQ(resp.provider, "vibevoice"); +} + +} // namespace arbiterAI diff --git a/tests/whisperProviderTests.cpp b/tests/whisperProviderTests.cpp new file mode 100644 index 0000000..f242339 --- /dev/null +++ b/tests/whisperProviderTests.cpp @@ -0,0 +1,127 @@ +/** + * @file whisperProviderTests.cpp + * @brief Unit tests for the local whisper.cpp speech-to-text provider. + * + * Only compiled when ARBITERAI_ENABLE_WHISPER is on. Tests that need a real + * model file skip when WHISPER_MODEL_PATH is unset or the file is absent, so + * the suite stays green in environments without whisper weights. + */ + +#include "arbiterAI/providers/whisper.h" +#include "arbiterAI/modelManager.h" + +#include +#include +#include +#include +#include +#include + +namespace arbiterAI +{ + +namespace +{ + +/// Build a minimal 16 kHz mono 16-bit PCM WAV containing `sampleCount` samples. +std::vector makeWav(uint32_t sampleCount) +{ + const uint32_t sampleRate=16000; + const uint16_t channels=1; + const uint16_t bits=16; + const uint32_t dataBytes=sampleCount*2; + const uint32_t byteRate=sampleRate*channels*bits/8; + + std::vector b; + auto put32=[&](uint32_t v){ for(int i=0;i<4;++i) b.push_back(static_cast((v>>(8*i))&0xFF)); }; + auto put16=[&](uint16_t v){ for(int i=0;i<2;++i) b.push_back(static_cast((v>>(8*i))&0xFF)); }; + auto putStr=[&](const char *s){ b.insert(b.end(), s, s+4); }; + + putStr("RIFF"); put32(36+dataBytes); putStr("WAVE"); + putStr("fmt "); put32(16); put16(1); put16(channels); + put32(sampleRate); put32(byteRate); put16(channels*bits/8); put16(bits); + putStr("data"); put32(dataBytes); + for(uint32_t i=0;i