diff --git a/docs/community_models/parakeet_tdt.md b/docs/community_models/parakeet_tdt.md index d311de3b5..6b2da3128 100644 --- a/docs/community_models/parakeet_tdt.md +++ b/docs/community_models/parakeet_tdt.md @@ -135,6 +135,8 @@ Request options are bare names. Session options use the |---|---|---|---| | `max_tokens` | request | integer >= 0; `0` | TDT token limit; `0` uses the model-derived limit | | `keep_language_tags` | request | boolean; `false` | Preserve language-tag tokens in decoded text | +| `audio_chunk_mode` | request | `auto`, `fixed`, `vad`, `none`; `auto` | Offline chunking mode; `vad` skips silent spans with Silero VAD | +| `audio_chunk_duration_sec` | request | float >= 0.001; session value | Request-level chunk duration for fixed or VAD offline chunking | | `weight_type` | session | `native`, `f32`, `f16`, `bf16`, `q8_0`; `native` | Fallback storage type for matmul weights | | `matmul_weight_type` | session | same values; inherits `weight_type` | Encoder and decoder matmul storage type | | `conv_weight_type` | session | `native`, `f32`, `f16`; `native` | True convolution-weight storage type | @@ -148,6 +150,7 @@ Request options are bare names. Session options use the | `streaming_attention_mode` | session | `full_context`; `full_context` | Bidirectional attention within each bounded streaming window | | `offline_mode` | session | `full_context`, `long_form`, `auto`; `full_context` | Offline scheduling policy | | `audio_chunk_threshold_sec` | session | float >= 0.001; `30` | `auto` threshold for switching to long-form execution | +| `vad_model_path` | session | path; `assets/framework/models/silero_vad` | Silero VAD model directory for `audio_chunk_mode=vad` | Word timestamps are built from the decoder's actual nonblank token-emission frames. SentencePiece fragments and following punctuation are merged into @@ -183,6 +186,13 @@ decoding every center region exactly once. Because each region sees bounded rather than utterance-wide context, its transcript can differ from the default full-context result. +Set `--request-option audio_chunk_mode=vad` to use Silero VAD-planned speech +spans instead of fixed windows. VAD mode uses +`audio_chunk_duration_sec` as the maximum speech chunk duration, falling back to +`parakeet_tdt.audio_chunk_duration_sec` when omitted. `parakeet_tdt.vad_model_path` +locates the bundled Silero VAD model. `auto` keeps the existing Parakeet +`offline_mode` behavior. + ## Performance Reference clip (`2086-149220-0033.wav`, 7.435s) on an Intel i7-9750H (6C/12T) diff --git a/include/engine/community_models/parakeet_tdt/session.h b/include/engine/community_models/parakeet_tdt/session.h index 05a77827f..8292a00ef 100644 --- a/include/engine/community_models/parakeet_tdt/session.h +++ b/include/engine/community_models/parakeet_tdt/session.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -69,12 +70,20 @@ class ParakeetTDTOfflineSession final runtime::TaskResult run_long_form( const runtime::AudioBuffer & audio, const ParakeetDecodeOptions & options); + runtime::TaskResult run_vad_chunks( + const runtime::AudioBuffer & audio, + const std::unordered_map & request_options, + const ParakeetDecodeOptions & options); + runtime::IOfflineVoiceTaskSession & vad_session(); std::string offline_mode_ = "full_context"; int64_t center_samples_ = 0; int64_t left_context_samples_ = 0; int64_t right_context_samples_ = 0; int64_t auto_full_context_max_samples_ = 0; + std::filesystem::path vad_model_path_; + std::unique_ptr vad_model_; + std::unique_ptr vad_session_; }; class ParakeetTDTStreamingSession final diff --git a/model_specs/parakeet_tdt.json b/model_specs/parakeet_tdt.json index 7f9a338fc..7d9910dd8 100644 --- a/model_specs/parakeet_tdt.json +++ b/model_specs/parakeet_tdt.json @@ -48,7 +48,8 @@ "capabilities": { "asr": [ "word_timestamps", - "partial_results" + "partial_results", + "vad_chunking" ] }, "options": { @@ -67,6 +68,27 @@ "description": "Keep language tag tokens in decoded text; default false.", "required": false, "default": false + }, + { + "name": "audio_chunk_mode", + "type": "enum", + "description": "Offline audio chunking mode. vad uses Silero VAD to skip silence; auto keeps Parakeet's existing offline_mode behavior.", + "values": [ + "auto", + "fixed", + "vad", + "none" + ], + "required": false, + "default": "auto" + }, + { + "name": "audio_chunk_duration_sec", + "type": "float", + "description": "Request-level chunk duration in seconds for fixed or VAD offline chunking. Falls back to the session chunk duration when omitted.", + "required": false, + "min": 0.001, + "default": 2.0 } ], "session": [ @@ -178,6 +200,13 @@ "required": false, "min": 0.001, "default": 30.0 + }, + { + "name": "vad_model_path", + "type": "path", + "description": "Silero VAD model path used by audio_chunk_mode=vad; default assets/framework/models/silero_vad.", + "required": false, + "default": "assets/framework/models/silero_vad" } ], "load": [] diff --git a/src/community_models/parakeet_tdt/session.cpp b/src/community_models/parakeet_tdt/session.cpp index 1eec70ffc..ce3497252 100644 --- a/src/community_models/parakeet_tdt/session.cpp +++ b/src/community_models/parakeet_tdt/session.cpp @@ -1,8 +1,10 @@ #include "engine/community_models/parakeet_tdt/session.h" +#include "engine/framework/audio/chunking.h" #include "engine/framework/debug/profiler.h" #include "engine/framework/runtime/options.h" #include "engine/framework/runtime/spec_backed_model.h" +#include "engine/models/silero_vad/session.h" #include #include @@ -44,8 +46,16 @@ std::shared_ptr require_contract( void validate_session_option_keys( const runtime::SessionOptions & options, const engine::model_spec::ModelContract & contract) { + auto validation_options = options; + // Older standalone GGUF packages embed a v1 contract that predates native + // Parakeet VAD wiring. Keep those packages usable while validating the + // option against current local specs. + if (contract.session_option_keys.find("parakeet_tdt.vad_model_path") == + contract.session_option_keys.end()) { + validation_options.options.erase("parakeet_tdt.vad_model_path"); + } const std::string family_prefix = std::string(kFamily) + "."; - for (const auto & [key, _] : options.options) { + for (const auto & [key, _] : validation_options.options) { if (key.rfind(family_prefix, 0) == 0 && contract.session_option_keys.find(key) == contract.session_option_keys.end()) { throw std::runtime_error("unknown Parakeet TDT session option: " + key); @@ -53,6 +63,37 @@ void validate_session_option_keys( } } +std::unordered_map normalize_request_options( + std::unordered_map options, + const engine::model_spec::ModelContract & contract) { + options = runtime::apply_option_v1_compatibility( + std::move(options), + { + {"audio_chunk_seconds", "audio_chunk_duration_sec"}, + {"audio_chunk_duration_seconds", "audio_chunk_duration_sec"}, + {"audio_chunk_duration", "audio_chunk_duration_sec"}, + }, + "Parakeet TDT", + "request"); + auto validation_options = options; + // Older standalone GGUF packages embed a v1 contract that predates native + // Parakeet VAD request controls. Keep those packages usable while still + // rejecting unrelated unknown request options. + if (contract.request_option_keys.find("audio_chunk_mode") == + contract.request_option_keys.end()) { + validation_options.erase("audio_chunk_mode"); + } + if (contract.request_option_keys.find("audio_chunk_duration_sec") == + contract.request_option_keys.end()) { + validation_options.erase("audio_chunk_duration_sec"); + } + runtime::validate_spec_backed_request_options( + validation_options, + contract, + "Parakeet TDT"); + return options; +} + bool use_flash_attention(const runtime::SessionOptions & options) { const auto value = runtime::find_option(options.options, {"parakeet_tdt.perf_mode"}).value_or("off"); @@ -162,6 +203,10 @@ std::string streaming_attention_mode_option(const runtime::SessionOptions& optio return value; } +std::filesystem::path default_vad_model_path() { + return std::filesystem::path("assets") / "framework" / "models" / "silero_vad"; +} + void validate_enum_contract_options(const runtime::SessionOptions& options) { (void)offline_mode_option(options); (void)streaming_attention_mode_option(options); @@ -283,6 +328,9 @@ ParakeetTDTOfflineSession::ParakeetTDTOfflineSession( kDefaultAudioChunkThresholdSec, kMinimumPositiveDurationSec), sample_rate); + vad_model_path_ = + runtime::find_option(this->options().options, {"parakeet_tdt.vad_model_path"}) + .value_or(default_vad_model_path().string()); } std::string ParakeetTDTOfflineSession::family() const { return family_impl(); } @@ -321,37 +369,52 @@ void ParakeetTDTOfflineSession::prepare(const runtime::SessionPreparationRequest runtime::TaskResult ParakeetTDTOfflineSession::run(const runtime::TaskRequest & request) { require_prepared("Parakeet TDT run()"); - if (!request.audio_input.has_value()) { + auto normalized_request = request; + normalized_request.options = normalize_request_options(request.options, *contract_); + if (!normalized_request.audio_input.has_value()) { throw std::runtime_error("Parakeet TDT run() requires audio_input"); } - if (request.audio_input->sample_rate <= 0 || - request.audio_input->channels <= 0 || - request.audio_input->samples.size() % - static_cast(request.audio_input->channels) != + if (normalized_request.audio_input->sample_rate <= 0 || + normalized_request.audio_input->channels <= 0 || + normalized_request.audio_input->samples.size() % + static_cast(normalized_request.audio_input->channels) != 0) { throw std::runtime_error("Parakeet TDT run() received an invalid audio layout"); } const auto wall_start = Clock::now(); - const auto decode_options = decode_options_for_request(request); + const auto decode_options = decode_options_for_request(normalized_request); const int64_t source_frames = - static_cast(request.audio_input->samples.size()) / - std::max(request.audio_input->channels, 1); + static_cast(normalized_request.audio_input->samples.size()) / + std::max(normalized_request.audio_input->channels, 1); const int64_t target_samples = static_cast(std::ceil( static_cast(source_frames) * static_cast(assets_->config.frontend.sample_rate) / - static_cast(request.audio_input->sample_rate))); + static_cast(normalized_request.audio_input->sample_rate))); + const auto chunk_mode = engine::audio::parse_audio_chunk_mode(normalized_request.options); + if (chunk_mode == engine::audio::AudioChunkMode::QuietEnergy) { + throw std::runtime_error("Parakeet TDT supports audio_chunk_mode=auto, fixed, vad, or none"); + } + if (chunk_mode == engine::audio::AudioChunkMode::Vad) { + auto result = run_vad_chunks(*normalized_request.audio_input, normalized_request.options, decode_options); + debug::timing_log_scalar( + "session.wall_ms", + engine::debug::elapsed_ms(wall_start, Clock::now())); + return result; + } const bool use_long_form = - offline_mode_ == "long_form" || - (offline_mode_ == "auto" && target_samples > auto_full_context_max_samples_); + chunk_mode == engine::audio::AudioChunkMode::Fixed || + (chunk_mode == engine::audio::AudioChunkMode::Auto && + (offline_mode_ == "long_form" || + (offline_mode_ == "auto" && target_samples > auto_full_context_max_samples_))); if (use_long_form) { - auto result = run_long_form(*request.audio_input, decode_options); + auto result = run_long_form(*normalized_request.audio_input, decode_options); debug::timing_log_scalar( "session.wall_ms", engine::debug::elapsed_ms(wall_start, Clock::now())); return result; } - const auto frontend = frontend_.extract(*request.audio_input, true); + const auto frontend = frontend_.extract(*normalized_request.audio_input, true); const auto encoded = encoder_->encode(frontend); auto decoded = decoder_->decode(encoded, decode_options); @@ -461,6 +524,102 @@ runtime::TaskResult ParakeetTDTOfflineSession::run_long_form( return result; } +runtime::TaskResult ParakeetTDTOfflineSession::run_vad_chunks( + const runtime::AudioBuffer& audio, + const std::unordered_map & request_options, + const ParakeetDecodeOptions& decode_options) { + if (audio.sample_rate <= 0 || audio.channels <= 0 || + audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error("Parakeet TDT VAD mode received an invalid audio layout"); + } + auto chunk_seconds_override = + engine::audio::parse_audio_chunk_seconds_override(request_options); + if (!chunk_seconds_override.has_value()) { + chunk_seconds_override = + engine::audio::parse_audio_chunk_seconds_override(this->options().options); + } + const auto chunk_seconds = + chunk_seconds_override.value_or( + static_cast(center_samples_) / + static_cast(assets_->config.frontend.sample_rate)); + if (!(chunk_seconds > 0.0F)) { + throw std::runtime_error("Parakeet TDT audio_chunk_duration_sec must be positive"); + } + const auto vad_options = engine::audio::VadAudioChunkOptions{ + static_cast( + std::llround(static_cast(chunk_seconds) * static_cast(audio.sample_rate))), + static_cast(std::llround(0.5 * static_cast(audio.sample_rate))), + static_cast(std::llround(0.25 * static_cast(audio.sample_rate))), + }; + if (vad_options.max_chunk_samples <= 0) { + throw std::runtime_error("Parakeet TDT audio_chunk_duration_sec produced an empty chunk"); + } + const auto spans = engine::audio::plan_vad_audio_chunks(audio, vad_session(), vad_options); + if (spans.empty()) { + runtime::TaskResult empty; + empty.text_output = runtime::Transcript{"", ""}; + return empty; + } + + runtime::TaskResult result; + std::string text; + int64_t token_count = 0; + + for (const auto& span : spans) { + auto window = engine::audio::slice_audio_buffer(audio, span); + const auto features = frontend_.extract(window, true); + const auto encoded = encoder_->encode(features); + if (encoded.valid_frames <= 0) { + continue; + } + auto item_decode_options = decode_options; + if (item_decode_options.max_tokens > 0) { + item_decode_options.max_tokens = std::max( + 0, + item_decode_options.max_tokens - token_count); + } + if (item_decode_options.max_tokens == 0 && decode_options.max_tokens > 0) { + break; + } + auto decoded = decoder_->decode(encoded, item_decode_options); + token_count += static_cast(decoded.token_ids.size()); + if (!decoded.text.empty()) { + if (!text.empty()) { + text.push_back(' '); + } + text += decoded.text; + } + engine::audio::append_chunk_word_timestamps( + result.word_timestamps, + decoded.word_timestamps, + span, + span, + audio.sample_rate, + assets_->config.frontend.sample_rate); + } + + result.text_output = runtime::Transcript{std::move(text), ""}; + return result; +} + +runtime::IOfflineVoiceTaskSession& ParakeetTDTOfflineSession::vad_session() { + if (vad_session_ == nullptr) { + runtime::ModelLoadRequest load_request; + load_request.model_path = vad_model_path_; + vad_model_ = engine::models::silero_vad::load_silero_vad_model(load_request); + auto session = vad_model_->create_task_session( + runtime::TaskSpec{runtime::VoiceTaskKind::Vad, runtime::RunMode::Offline}, + runtime::SessionOptions{options().backend, {}}); + auto* offline = dynamic_cast(session.get()); + if (offline == nullptr) { + throw std::runtime_error("Parakeet TDT internal VAD session does not support offline execution"); + } + session.release(); + vad_session_.reset(offline); + } + return *vad_session_; +} + ParakeetTDTStreamingSession::ParakeetTDTStreamingSession( runtime::TaskSpec task, runtime::SessionOptions options, @@ -539,7 +698,9 @@ runtime::StreamingPolicy ParakeetTDTStreamingSession::streaming_policy() const { void ParakeetTDTStreamingSession::start_stream(const runtime::TaskRequest& request) { reset(); - streaming_decode_options_ = decode_options_for_request(request); + auto normalized_request = request; + normalized_request.options = normalize_request_options(request.options, *contract_); + streaming_decode_options_ = decode_options_for_request(normalized_request); } void ParakeetTDTStreamingSession::set_stream_event_sink( diff --git a/tools/audiocpp_cli/audiocpp_cli_path_cases.json b/tools/audiocpp_cli/audiocpp_cli_path_cases.json index 7a4a018b1..ec9888051 100644 --- a/tools/audiocpp_cli/audiocpp_cli_path_cases.json +++ b/tools/audiocpp_cli/audiocpp_cli_path_cases.json @@ -2187,6 +2187,30 @@ } ] }, + { + "id": "parakeet_tdt_offline_vad", + "coverage": "Parakeet offline Silero VAD chunk planning and TDT decoder", + "family": "parakeet_tdt", + "model": "models/parakeet-tdt-0.6b-v3", + "task": "asr", + "mode": "offline", + "session_options": { + "parakeet_tdt.audio_chunk_duration_sec": "2", + "parakeet_tdt.vad_model_path": "assets/framework/models/silero_vad" + }, + "outputs": [ + "text" + ], + "requests": [ + { + "id": "sample_vad", + "audio": "resources/sample_16k.wav", + "options": { + "audio_chunk_mode": "vad" + } + } + ] + }, { "id": "parakeet_tdt_streaming", "coverage": "Parakeet streaming chunk ingestion, buffered context, streaming finalize",