diff --git a/Cargo.lock b/Cargo.lock index a5093f441..fde36135f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2343,6 +2343,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "switchyard-runner" +version = "0.2.0" +dependencies = [ + "async-trait", + "futures-util", + "reqwest", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-llm-client", + "switchyard-protocol", + "thiserror 2.0.18", + "tokio", + "toml", + "tracing", +] + [[package]] name = "switchyard-server" version = "0.2.0" @@ -2362,17 +2380,16 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "prometheus", - "reqwest", "rustls", "serde", "serde_json", "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", + "switchyard-runner", "switchyard-translation", "tempfile", "tokio", - "toml", "tower", "tracing", "tracing-opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index 0fc5030a7..c1b8d1a94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/libsy-llm-client", "crates/switchyard-py", "crates/protocol", + "crates/switchyard-runner", "crates/switchyard-server", "crates/switchyard-skill-distillation", "crates/switchyard-soak", @@ -40,6 +41,7 @@ serde_json = { version = "1", features = ["preserve_order"] } switchyard-libsy = { path = "crates/libsy", version = "0.2.0" } switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.2.0" } switchyard-protocol = { path = "crates/protocol", version = "0.2.0" } +switchyard-runner = { path = "crates/switchyard-runner", version = "0.2.0" } switchyard-server = { path = "crates/switchyard-server", version = "0.2.0" } switchyard-translation = { path = "crates/switchyard-translation", version = "0.2.0" } thiserror = "2" diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs index 12b4395d6..1005d49f6 100644 --- a/crates/switchyard-py/src/server_bindings.rs +++ b/crates/switchyard-py/src/server_bindings.rs @@ -44,13 +44,8 @@ impl PyServer { .map_err(|error| ServerConfigError::new_err(error.to_string()))?; let caller_auth_by_model = state .models() - .map(|model| { - state - .caller_auth_kind(model) - .map(|kind| (model.to_string(), kind)) - }) - .collect::>>() - .map_err(server_error)?; + .map(|model| (model.to_string(), state.caller_auth_kind(model))) + .collect::>(); let runtime = pyo3_async_runtimes::tokio::get_runtime(); let server = { let _guard = runtime.enter(); diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml new file mode 100644 index 000000000..5545f2c19 --- /dev/null +++ b/crates/switchyard-runner/Cargo.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-runner" +version.workspace = true +description = "Configured routing execution for Switchyard serving surfaces" +authors.workspace = true +edition.workspace = true +homepage = "https://github.com/NVIDIA-NeMo/Switchyard" +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = ["crates-io"] + +[dependencies] +libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" } +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +switchyard-llm-client.workspace = true +switchyard-protocol.workspace = true +thiserror.workspace = true +toml = "1.1" +tracing.workspace = true + +[dev-dependencies] +async-trait.workspace = true +futures-util.workspace = true +tokio.workspace = true diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs new file mode 100644 index 000000000..b4d400d74 --- /dev/null +++ b/crates/switchyard-runner/src/algorithm.rs @@ -0,0 +1,1063 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Schema-neutral algorithm configuration and construction. + +use std::collections::BTreeMap; +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::sync::Arc; + +use libsy::{ + AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, + ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, + GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, + Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, + SubagentRouterConfig, TargetPrompts, TaskClassifierConfig, +}; +use serde::Deserialize; +use switchyard_protocol::ModelId; + +/// Error returned when an algorithm description cannot be constructed. +#[derive(Debug)] +pub struct AlgorithmConfigError { + message: String, + source: Option>, +} + +impl AlgorithmConfigError { + fn new(message: impl Into) -> Self { + Self { + message: message.into(), + source: None, + } + } + + fn with_source(message: impl Into, source: impl Error + Send + Sync + 'static) -> Self { + Self { + message: message.into(), + source: Some(Box::new(source)), + } + } +} + +impl Display for AlgorithmConfigError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl Error for AlgorithmConfigError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + self.source.as_deref().map(|source| source as _) + } +} + +type AlgorithmResult = Result; + +/// How a custom classifier turns the judge's JSON verdict into a target. +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum ClassifierPolicyConfig { + /// Reads the target name straight out of the judge's verdict. + TargetSelector { + /// JSON Pointer to the name, such as `/decision/target`. + selector: String, + }, +} + +/// Which of the three `llm_classifier` behaviors a route uses. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ClassifierMode { + /// Judges the request first, then serves the strong or weak target. + Capability, + /// Serves the weak target first and judges the finished turn, moving to the + /// strong target once the session latches. + Escalation, + /// Judges against your own JSON schema and routes to any configured target. + Custom, +} + +impl ClassifierPolicyConfig { + fn into_libsy(self) -> CustomClassifierPolicy { + match self { + Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector), + } + } +} + +#[derive(Clone, Debug)] +enum LlmClassifierModeConfig { + Capability(CapabilityClassifierRouteConfig), + Escalation(EscalationClassifierRouteConfig), + Custom(CustomClassifierRouteConfig), +} + +#[derive(Clone, Debug)] +struct CapabilityClassifierRouteConfig { + strong_target: String, + weak_target: String, + base_threshold: f64, + threshold_step: f64, + classify_trigger: ClassifyTrigger, + message_hash_fallback: bool, + recent_turn_window: Option, + prompt: Option, + response_format_type: ClassifierResponseFormat, + max_output_tokens: u64, +} + +#[derive(Clone, Debug)] +struct EscalationClassifierRouteConfig { + strong_target: String, + weak_target: String, + prompt: Option, + response_format_type: ClassifierResponseFormat, + max_output_tokens: u64, + judge: EscalationJudgeConfig, +} + +#[derive(Clone, Debug)] +struct CustomClassifierRouteConfig { + classifier_target: String, + targets: Vec, + default_target: String, + prompt: String, + response_schema: String, + policy: ClassifierPolicyConfig, + classify_trigger: ClassifyTrigger, + message_hash_fallback: bool, + recent_turn_window: Option, + max_output_tokens: u64, +} + +/// Settings for an `llm_classifier` route. Which fields are required depends on +/// the [`ClassifierMode`]; using a field from the wrong mode is an error. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct LlmClassifierRouteConfig { + /// Target the judge is called through. Never a routing destination itself. + pub classifier_target: String, + /// Mode to run. Defaults to escalation when `escalation` is set, otherwise capability. + pub mode: Option, + /// Capability and escalation modes: the capable tier. + pub strong_target: Option, + /// Capability and escalation modes: the efficient tier. + pub weak_target: Option, + /// Capability mode: lowest solve probability that still routes to the weak + /// target, from 0 to 1. + pub base_threshold: Option, + /// Capability mode: how much to raise the threshold when the judge is + /// uncertain. Added once for an uncertain verdict and twice for unsupported. + pub threshold_step: Option, + /// How often the judge runs: every request, once per user turn, or once per session. + pub classify_trigger: ClassifyTrigger, + /// Reuses the session's target by hashing the first user message when no + /// session ID is available. Needs `classify_trigger = "new_session"`. + pub message_hash_fallback: bool, + /// How many trailing turns the judge sees. Unset shows it the opening task + /// and the latest user follow-up only. + pub recent_turn_window: Option, + /// Replaces the packaged judge prompt. Required in custom mode. + pub prompt: Option, + /// How the judge is asked for structured output. Use `json_object` when the + /// provider cannot do JSON Schema. + pub response_format_type: ClassifierResponseFormat, + /// Most completion tokens the judge verdict may use. + #[serde(default = "default_classifier_max_output_tokens")] + pub max_output_tokens: u64, + /// Escalation mode: how many escalate verdicts latch the session, and how + /// much of the transcript the judge sees. + pub escalation: Option, + /// Custom mode: the target names the policy may pick from. + pub targets: Option>, + /// Custom mode: target used when the judge fails or its verdict cannot be routed. + pub default_target: Option, + /// Custom mode: JSON Schema the verdict must match, written as a string. + pub response_schema: Option, + /// Custom mode: how to read the chosen target out of the verdict. + pub policy: Option, +} + +/// Routing policy applied only to delegated sub-agent work, nested inside a +/// `passthrough` or `stage_router` route. +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum SubagentRouteConfig { + /// Sends all sub-agent work to one target. + Passthrough { + /// Target that serves sub-agent requests. + target: String, + }, + /// Judges each sub-agent request. Only [`ClassifierMode::Custom`] is supported here. + LlmClassifier(Box), +} + +impl SubagentRouteConfig { + fn routing_target_names(&self) -> Vec<&str> { + match self { + Self::Passthrough { target } => vec![target], + Self::LlmClassifier(classifier) => classifier + .targets + .iter() + .flatten() + .map(String::as_str) + .collect(), + } + } + + fn classifier_target_name(&self) -> Option<&str> { + match self { + Self::LlmClassifier(classifier) => Some(&classifier.classifier_target), + Self::Passthrough { .. } => None, + } + } +} + +/// A routing algorithm described by configured target names. +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum AlgorithmSpec { + /// Replies `OK` without calling any model. Useful for smoke tests. + Noop {}, + /// Splits traffic across several targets. + Random { + /// Target names to choose from. + targets: Vec, + /// Relative weights in `targets` order. Equal weights when unset. + weights: Option>, + /// Makes the sequence of choices repeatable. + seed: Option, + }, + /// Sends every request to one target. + Passthrough { + /// Target that serves the request. + target: String, + /// Separate policy for delegated sub-agent work. + #[serde(default)] + subagents: Option, + }, + /// Asks a judge model which target should serve the request. + LlmClassifier { + /// Judge and tier settings, written directly in the route table. + #[serde(flatten)] + config: LlmClassifierRouteConfig, + }, + /// Picks a tier per turn by scoring signals from recent tool results. + StageRouter { + /// The capable tier. + capable_target: String, + /// The efficient tier. + efficient_target: String, + /// Tier to use when the signals are not confident. + picker: PickerMode, + /// How much agreement a decisive pick needs, from 0 to 1. + confidence_threshold: f64, + /// How many trailing tool results the signals are scored over. + #[serde(default)] + recent_turn_window: Option, + /// Notes handed to a tier when the router switches to it. + #[serde(default)] + handoff_notes: Option, + /// System prompt handed to the capable tier. + #[serde(default)] + capable_system_prompt: Option, + /// System prompt handed to the efficient tier. + #[serde(default)] + efficient_system_prompt: Option, + /// Judge consulted for turns the tool signals cannot decide. + #[serde(default)] + classifier: Option, + /// Separate policy for delegated sub-agent work. + #[serde(default)] + subagents: Option, + }, + /// Serves every turn from one target, and has a second model review some of + /// those turns before the caller sees them. + Advisor { + /// Serves every client-visible turn. + executor_target: String, + /// Reviews gated turns. Never a routing destination. + advisor_target: String, + /// Replaces the built-in APPROVE/REDO reviewer prompt. + #[serde(default)] + reviewer_system_prompt: Option, + /// Replaces the built-in text put in front of a REDO plan. + #[serde(default)] + redo_feedback_prefix: Option, + /// What fires a review. + #[serde(default)] + gate_trigger: AdvisorTriggerConfig, + /// Regular expression for the `pattern` trigger. Required by it, and unused otherwise. + #[serde(default)] + gate_trigger_pattern: Option, + /// How many reviews one session may spend. + #[serde(default = "default_max_reviews")] + max_reviews: u32, + /// Reviews a turn after this many assistant turns, as a mid-task + /// checkpoint. Zero turns the checkpoint off. + #[serde(default)] + gate_stall_turns: u32, + /// Tool results a turn needs before it can be reviewed. Skips early chatty turns. + #[serde(default)] + gate_min_tool_results: u32, + /// Most output tokens one review may use. + #[serde(default = "default_advisor_max_tokens")] + advisor_max_tokens: u64, + /// Sampling temperature for reviews. Left off the request when unset. + #[serde(default)] + advisor_temperature: Option, + /// Size cap on the transcript sent to the advisor. Longer transcripts + /// are trimmed from the middle. + #[serde(default = "default_transcript_max_chars")] + transcript_max_chars: usize, + /// Lets the turn through when the advisor fails, instead of erroring. + #[serde(default = "default_fail_open")] + fail_open: bool, + }, +} + +/// What fires an advisor route's review. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum AdvisorTriggerConfig { + /// The executor's first turn without tool calls. + #[default] + NoToolCall, + /// The first turn whose text matches `gate_trigger_pattern`. + Pattern, +} + +/// The judge a `stage_router` route falls through to, and how it routes. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StageClassifierConfig { + /// Target the judge is called through. Not a routing destination. + pub target: String, + /// Lowest solve probability that still routes to the efficient tier, from 0 to 1. + pub base_threshold: f64, + /// How much to raise the threshold when the judge is uncertain. Added once + /// for an uncertain verdict and twice for unsupported. + #[serde(default)] + pub threshold_step: f64, + /// How often the judge runs. `new_session` has no effect here. + #[serde(default)] + pub classify_trigger: ClassifyTrigger, + /// Reuses the session's target by hashing the first user message when no + /// session ID is available. + #[serde(default)] + pub message_hash_fallback: bool, + /// How many trailing turns the judge sees. Unset shows it the opening task + /// and the latest user follow-up only. + #[serde(default)] + pub recent_turn_window: Option, + /// Replaces the packaged judge prompt. + #[serde(default)] + pub prompt: Option, + /// How the judge is asked for structured output. Use `json_object` when the + /// provider cannot do JSON Schema. + #[serde(default)] + pub response_format_type: ClassifierResponseFormat, + /// Most completion tokens the judge verdict may use. + #[serde(default = "default_classifier_max_output_tokens")] + pub max_output_tokens: u64, +} + +impl StageClassifierConfig { + fn task_classifier_config(&self) -> TaskClassifierConfig { + TaskClassifierConfig { + base_threshold: self.base_threshold, + threshold_step: self.threshold_step, + classify_trigger: self.classify_trigger, + message_hash_fallback: self.message_hash_fallback, + recent_turn_window: self.recent_turn_window, + contract: classifier_contract(self.prompt.as_deref()) + .with_response_format_type(self.response_format_type), + max_output_tokens: self.max_output_tokens, + } + } +} + +impl AlgorithmSpec { + /// Completion targets in algorithm order; judge-only targets are excluded. + pub fn routing_target_names(&self) -> Vec<&str> { + match self { + Self::Noop { .. } => Vec::new(), + Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(), + Self::Passthrough { + target, subagents, .. + } => { + let mut names = vec![target.as_str()]; + if let Some(subagents) = subagents { + names.extend(subagents.routing_target_names()); + } + names + } + Self::LlmClassifier { config, .. } => { + match config.mode.unwrap_or(if config.escalation.is_some() { + ClassifierMode::Escalation + } else { + ClassifierMode::Capability + }) { + ClassifierMode::Capability => config + .weak_target + .iter() + .chain(&config.strong_target) + .map(String::as_str) + .collect(), + ClassifierMode::Escalation => config + .strong_target + .iter() + .chain(&config.weak_target) + .map(String::as_str) + .collect(), + ClassifierMode::Custom => config + .targets + .iter() + .flatten() + .map(String::as_str) + .collect(), + } + } + Self::StageRouter { + capable_target, + efficient_target, + subagents, + .. + } => { + let mut names = vec![capable_target.as_str(), efficient_target.as_str()]; + if let Some(subagents) = subagents { + names.extend(subagents.routing_target_names()); + } + names + } + // The advisor is judge-only: reviews go through its own client, + // so it is not a completion (or count_tokens) destination. + Self::Advisor { + executor_target, .. + } => vec![executor_target], + } + } + + /// Every target the algorithm may call, including judge-only targets. + /// + /// [`routing_target_names`](Self::routing_target_names) covers completion destinations; + /// a classifier also calls its judge, and that call needs a client too. + pub fn callable_target_names(&self) -> Vec<&str> { + let mut names = self.routing_target_names(); + match self { + Self::LlmClassifier { config, .. } => names.push(&config.classifier_target), + Self::Passthrough { + subagents: Some(subagents), + .. + } => names.extend(subagents.classifier_target_name()), + Self::StageRouter { + classifier, + subagents, + .. + } => { + if let Some(classifier) = classifier { + names.push(&classifier.target); + } + if let Some(subagents) = subagents { + names.extend(subagents.classifier_target_name()); + } + } + Self::Advisor { advisor_target, .. } => names.push(advisor_target), + _ => {} + } + names + } + /// Builds this algorithm after resolving configured target names. + pub fn build( + &self, + context: &str, + targets: &BTreeMap, + ) -> AlgorithmResult> { + build_algorithm(context, self, targets) + } +} +impl LlmClassifierRouteConfig { + fn classifier_mode(&self, route_name: &str) -> AlgorithmResult { + let Self { + classifier_target, + mode, + strong_target, + weak_target, + base_threshold, + threshold_step, + classify_trigger, + message_hash_fallback, + recent_turn_window, + prompt, + response_format_type, + max_output_tokens, + escalation, + targets, + default_target, + response_schema, + policy, + } = self; + + let selected_mode = match (mode, escalation.is_some()) { + (Some(mode), _) => *mode, + (None, true) => ClassifierMode::Escalation, + (None, false) => ClassifierMode::Capability, + }; + + match selected_mode { + ClassifierMode::Capability => { + if escalation.is_some() { + return Err(classifier_field_error( + route_name, + "escalation", + "capability", + )); + } + reject_custom_fields( + route_name, + "capability", + targets, + default_target, + response_schema, + policy, + )?; + Ok(LlmClassifierModeConfig::Capability( + CapabilityClassifierRouteConfig { + strong_target: required_classifier_field( + route_name, + "strong_target", + strong_target, + )?, + weak_target: required_classifier_field( + route_name, + "weak_target", + weak_target, + )?, + base_threshold: required_classifier_field( + route_name, + "base_threshold", + base_threshold, + )?, + threshold_step: threshold_step.unwrap_or_default(), + classify_trigger: *classify_trigger, + message_hash_fallback: *message_hash_fallback, + recent_turn_window: *recent_turn_window, + prompt: prompt.clone(), + response_format_type: *response_format_type, + max_output_tokens: *max_output_tokens, + }, + )) + } + ClassifierMode::Escalation => { + reject_custom_fields( + route_name, + "escalation", + targets, + default_target, + response_schema, + policy, + )?; + if *classify_trigger != ClassifyTrigger::EveryRequest { + return Err(AlgorithmConfigError::new(format!( + "llm_classifier route {route_name} mode escalation cannot use classify_trigger" + ))); + } + if mode.is_some() + && (base_threshold.is_some() + || threshold_step.is_some() + || *message_hash_fallback + || recent_turn_window.is_some()) + { + return Err(AlgorithmConfigError::new(format!( + "llm_classifier route {route_name} mode escalation cannot use capability routing settings" + ))); + } + Ok(LlmClassifierModeConfig::Escalation( + EscalationClassifierRouteConfig { + strong_target: required_classifier_field( + route_name, + "strong_target", + strong_target, + )?, + weak_target: required_classifier_field( + route_name, + "weak_target", + weak_target, + )?, + prompt: prompt.clone(), + response_format_type: *response_format_type, + max_output_tokens: *max_output_tokens, + judge: required_classifier_field(route_name, "escalation", escalation)?, + }, + )) + } + ClassifierMode::Custom => { + if strong_target.is_some() + || weak_target.is_some() + || base_threshold.is_some() + || threshold_step.is_some() + || escalation.is_some() + || *response_format_type != ClassifierResponseFormat::JsonSchema + { + return Err(AlgorithmConfigError::new(format!( + "llm_classifier route {route_name} mode custom cannot use capability or escalation fields and response_format_type must be 'json_schema'" + ))); + } + Ok(LlmClassifierModeConfig::Custom( + CustomClassifierRouteConfig { + classifier_target: classifier_target.clone(), + targets: required_classifier_field(route_name, "targets", targets)?, + default_target: required_classifier_field( + route_name, + "default_target", + default_target, + )?, + prompt: required_classifier_field(route_name, "prompt", prompt)?, + response_schema: required_classifier_field( + route_name, + "response_schema", + response_schema, + )?, + policy: required_classifier_field(route_name, "policy", policy)?, + classify_trigger: *classify_trigger, + message_hash_fallback: *message_hash_fallback, + recent_turn_window: *recent_turn_window, + max_output_tokens: *max_output_tokens, + }, + )) + } + } + } +} + +fn reject_custom_fields( + route_name: &str, + mode: &str, + targets: &Option>, + default_target: &Option, + response_schema: &Option, + policy: &Option, +) -> AlgorithmResult<()> { + if targets.is_some() + || default_target.is_some() + || response_schema.is_some() + || policy.is_some() + { + return Err(AlgorithmConfigError::new(format!( + "llm_classifier route {route_name} mode {mode} cannot use custom classifier fields" + ))); + } + Ok(()) +} + +fn classifier_field_error(route_name: &str, field: &str, mode: &str) -> AlgorithmConfigError { + AlgorithmConfigError::new(format!( + "llm_classifier route {route_name} mode {mode} cannot use {field}" + )) +} + +fn required_classifier_field( + route_name: &str, + field: &str, + value: &Option, +) -> AlgorithmResult { + value.clone().ok_or_else(|| { + AlgorithmConfigError::new(format!( + "llm_classifier route {route_name} requires {field}" + )) + }) +} + +fn build_subagent_router_config( + route_name: &str, + config: &SubagentRouteConfig, + targets: &BTreeMap, +) -> AlgorithmResult { + match config { + SubagentRouteConfig::Passthrough { target } => Ok(SubagentRouterConfig::fixed_target( + resolve_target_model_id(route_name, target, targets)?, + )), + SubagentRouteConfig::LlmClassifier(config) => { + let LlmClassifierModeConfig::Custom(config) = config.classifier_mode(route_name)? + else { + return Err(AlgorithmConfigError::new(format!( + "route {route_name}: subagents llm_classifier only supports mode custom" + ))); + }; + let judge_target = + resolve_target_model_id(route_name, &config.classifier_target, targets)?; + let resolved_targets = config + .targets + .iter() + .map(|name| { + resolve_target_model_id(route_name, name, targets) + .map(|target| (name.clone(), target)) + }) + .collect::>>()?; + let default_target = resolved_targets + .iter() + .find(|(name, _)| *name == config.default_target) + .map(|(_, target)| target.clone()) + .ok_or_else(|| { + AlgorithmConfigError::new(format!( + "route {route_name}: subagents llm_classifier default_target {:?} must be one of its configured targets", + config.default_target + )) + })?; + let response_schema = + serde_json::from_str(&config.response_schema).map_err(|error| { + AlgorithmConfigError::with_source( + format!( + "route {route_name}: subagents llm_classifier response_schema is invalid JSON: {error}" + ), + error, + ) + })?; + let mut classifier_config = CustomClassifierConfig::new( + config.prompt, + response_schema, + config.policy.into_libsy(), + ); + classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.max_output_tokens = config.max_output_tokens; + let subagent_targets = resolved_targets + .iter() + .map(|(_, target)| target.clone()) + .collect(); + let classifier = Arc::new( + LlmTaskClassifier::new(LlmClassifierConfig::Custom { + judge_target, + targets: resolved_targets, + default_target: config.default_target, + config: classifier_config, + }) + .map_err(|error| { + AlgorithmConfigError::with_source( + format!("route {route_name}: subagents llm_classifier: {error}"), + error, + ) + })?, + ); + Ok(SubagentRouterConfig { + targets: subagent_targets, + classifier, + default_target, + classify_trigger: config.classify_trigger, + message_hash_fallback: config.message_hash_fallback, + }) + } + } +} + +fn attach_subagent_router( + route_name: &str, + parent: Arc, + config: Option<&SubagentRouteConfig>, + targets: &BTreeMap, +) -> AlgorithmResult> { + let Some(config) = config else { + return Ok(parent); + }; + let config = build_subagent_router_config(route_name, config, targets)?; + let algorithm = SubagentRouter::new(parent, config).map_err(|error| { + AlgorithmConfigError::with_source( + format!("route {route_name}: subagent routing: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) +} + +fn build_algorithm( + route_name: &str, + config: &AlgorithmSpec, + targets: &BTreeMap, +) -> AlgorithmResult> { + match config { + AlgorithmSpec::Noop { .. } => Ok(Arc::new(Noop {})), + AlgorithmSpec::Random { + targets: names, + weights, + seed, + .. + } => { + let target_set = + resolve_targets(route_name, names.iter().map(String::as_str), targets)?; + let algorithm = Random::new(target_set, weights.clone(), *seed).map_err(|error| { + AlgorithmConfigError::with_source( + format!("random route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } + AlgorithmSpec::Passthrough { + target, subagents, .. + } => { + let parent_target = resolve_target_model_id(route_name, target, targets)?; + let algorithm = Passthrough::new(parent_target); + let parent: Arc = Arc::new(algorithm); + attach_subagent_router(route_name, parent, subagents.as_ref(), targets) + } + AlgorithmSpec::LlmClassifier { + config: classifier_config, + .. + } => { + let classifier = + resolve_target_model_id(route_name, &classifier_config.classifier_target, targets)?; + let mode = classifier_config.classifier_mode(route_name)?; + let algorithm = match mode { + LlmClassifierModeConfig::Capability(config) => { + let strong = + resolve_target_model_id(route_name, &config.strong_target, targets)?; + let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; + let classifier_config = TaskClassifierConfig { + base_threshold: config.base_threshold, + threshold_step: config.threshold_step, + classify_trigger: config.classify_trigger, + message_hash_fallback: config.message_hash_fallback, + recent_turn_window: config.recent_turn_window, + contract: classifier_contract(config.prompt.as_deref()) + .with_response_format_type(config.response_format_type), + max_output_tokens: config.max_output_tokens, + }; + LlmTaskClassifier::new(LlmClassifierConfig::Capability { + judge_target: classifier, + efficient_target: weak, + capable_target: strong, + config: classifier_config, + }) + } + LlmClassifierModeConfig::Escalation(config) => { + let strong = + resolve_target_model_id(route_name, &config.strong_target, targets)?; + let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; + LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: classifier, + efficient_target: weak, + capable_target: strong, + contract: classifier_contract(config.prompt.as_deref()) + .with_response_format_type(config.response_format_type), + config: config.judge, + max_output_tokens: config.max_output_tokens, + }) + } + LlmClassifierModeConfig::Custom(config) => { + let resolved_targets = config + .targets + .iter() + .map(|name| { + resolve_target_model_id(route_name, name, targets) + .map(|target| (name.clone(), target)) + }) + .collect::>>()?; + let response_schema = serde_json::from_str(&config.response_schema).map_err( + |error| { + AlgorithmConfigError::with_source( + format!( + "llm_classifier route {route_name}: response_schema is invalid JSON: {error}" + ), + error, + ) + }, + )?; + let mut classifier_config = CustomClassifierConfig::new( + config.prompt, + response_schema, + config.policy.into_libsy(), + ); + classifier_config.classify_trigger = config.classify_trigger; + classifier_config.message_hash_fallback = config.message_hash_fallback; + classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.max_output_tokens = config.max_output_tokens; + LlmTaskClassifier::new(LlmClassifierConfig::Custom { + judge_target: classifier, + targets: resolved_targets, + default_target: config.default_target, + config: classifier_config, + }) + } + } + .map_err(|error| { + AlgorithmConfigError::with_source( + format!("llm_classifier route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } + AlgorithmSpec::StageRouter { + capable_target, + efficient_target, + picker, + confidence_threshold, + recent_turn_window, + handoff_notes, + capable_system_prompt, + efficient_system_prompt, + classifier, + subagents, + .. + } => { + if matches!(picker, PickerMode::CapableFirst) { + tracing::warn!( + "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration." + ); + } + let capable = resolve_target_model_id(route_name, capable_target, targets)?; + let efficient = resolve_target_model_id(route_name, efficient_target, targets)?; + let mut config = StageRouterConfig::new(*picker, *confidence_threshold); + config.recent_window = *recent_turn_window; + config.handoff_notes = handoff_notes.clone(); + config.tier_prompts = tier_prompts( + &capable, + capable_system_prompt.as_deref(), + &efficient, + efficient_system_prompt.as_deref(), + ); + // The judge is called through its own target, so it is not a routing + // destination and stays out of the tier pair. + config.llm_fallback = classifier + .as_ref() + .map(|classifier| { + resolve_target_model_id(route_name, &classifier.target, targets).map( + |judge_target| LlmFallback { + judge_target, + config: classifier.task_classifier_config(), + }, + ) + }) + .transpose()?; + let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { + AlgorithmConfigError::with_source( + format!("stage_router route {route_name}: {error}"), + error, + ) + })?; + let parent: Arc = Arc::new(algorithm); + attach_subagent_router(route_name, parent, subagents.as_ref(), targets) + } + AlgorithmSpec::Advisor { + executor_target, + advisor_target, + reviewer_system_prompt, + redo_feedback_prefix, + gate_trigger, + gate_trigger_pattern, + max_reviews, + gate_stall_turns, + gate_min_tool_results, + advisor_max_tokens, + advisor_temperature, + transcript_max_chars, + fail_open, + .. + } => { + let executor = resolve_target_model_id(route_name, executor_target, targets)?; + let advisor = resolve_target_model_id(route_name, advisor_target, targets)?; + // A pattern set under the default trigger would be silently + // ignored; reject the misconfiguration instead. + if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() { + return Err(AlgorithmConfigError::new(format!( + "advisor route {route_name}: gate_trigger_pattern requires \ + gate_trigger = \"pattern\"" + ))); + } + let mut config = AdvisorGateConfig::default(); + if let Some(prompt) = reviewer_system_prompt { + config.reviewer_system_prompt = prompt.clone(); + } + if let Some(prefix) = redo_feedback_prefix { + config.redo_feedback_prefix = prefix.clone(); + } + config.gate_trigger = match gate_trigger { + AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall, + AdvisorTriggerConfig::Pattern => { + GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default()) + } + }; + config.max_reviews = *max_reviews; + config.gate_stall_turns = *gate_stall_turns; + config.gate_min_tool_results = *gate_min_tool_results; + config.advisor_max_tokens = *advisor_max_tokens; + config.advisor_temperature = *advisor_temperature; + config.transcript_max_chars = *transcript_max_chars; + config.fail_open = *fail_open; + let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| { + AlgorithmConfigError::with_source( + format!("advisor route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } + } +} + +const fn default_max_reviews() -> u32 { + 1 +} + +const fn default_advisor_max_tokens() -> u64 { + 2048 +} + +const fn default_transcript_max_chars() -> usize { + 200_000 +} + +const fn default_fail_open() -> bool { + true +} + +fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig { + prompt.map_or_else(ClassifierContractConfig::default, |prompt| { + ClassifierContractConfig::default().with_prompt(prompt) + }) +} + +fn default_classifier_max_output_tokens() -> u64 { + TaskClassifierConfig::default().max_output_tokens +} + +/// Keys each configured system prompt by the target it belongs to. +fn tier_prompts( + capable: &str, + capable_prompt: Option<&str>, + efficient: &str, + efficient_prompt: Option<&str>, +) -> TargetPrompts { + let mut prompts = TargetPrompts::default(); + if let Some(prompt) = capable_prompt { + prompts = prompts.with(capable, prompt); + } + if let Some(prompt) = efficient_prompt { + prompts = prompts.with(efficient, prompt); + } + prompts +} + +fn resolve_targets<'a>( + route_name: &str, + names: impl IntoIterator, + targets: &BTreeMap, +) -> AlgorithmResult> { + names + .into_iter() + .map(|name| resolve_target_model_id(route_name, name, targets)) + .collect() +} + +fn resolve_target_model_id( + route_name: &str, + name: &str, + targets: &BTreeMap, +) -> AlgorithmResult { + targets.get(name).cloned().ok_or_else(|| { + AlgorithmConfigError::new(format!( + "route {route_name} references unknown target {name}" + )) + }) +} diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs new file mode 100644 index 000000000..7a90c7bba --- /dev/null +++ b/crates/switchyard-runner/src/config.rs @@ -0,0 +1,1401 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Version-1 TOML deployment loading for the shared runner. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fs; +use std::path::Path; +use std::sync::Arc; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use switchyard_llm_client::{ + Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, + TranslatingLlmClient, +}; +use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; + +use crate::{ + AlgorithmSpec, CallerAuthKind, CountTokensTarget, DecisionTarget, ModelCapabilities, Route, + Runner, RunnerError, +}; + +const SUPPORTED_SCHEMA_VERSION: u32 = 1; +const MAX_CONFIGURED_RETRIES: u32 = 10; + +type RunnerResult = Result; + +pub(crate) fn load_runner(path: impl AsRef) -> RunnerResult { + let path = path.as_ref(); + let source = fs::read_to_string(path).map_err(|error| { + RunnerError::configuration_source( + format!("failed to read server config {}: {error}", path.display()), + error, + ) + })?; + runner_from_toml(&source).map_err(|error| { + RunnerError::configuration_source( + format!("invalid server config {}: {error}", path.display()), + error, + ) + }) +} + +fn runner_from_toml(source: &str) -> RunnerResult { + let config: DeploymentConfig = toml::from_str(source).map_err(|error| { + RunnerError::configuration_source(format!("failed to parse TOML: {error}"), error) + })?; + config.build() +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct DeploymentConfig { + schema_version: u32, + #[serde(default)] + llm_clients: BTreeMap, + targets: BTreeMap, + routes: BTreeMap, +} + +#[derive(Debug)] +struct RouteConfig { + id: ModelId, + context_window: Option, + tool_calling: Option, + reasoning: Option, + algorithm: AlgorithmSpec, +} + +impl<'de> Deserialize<'de> for RouteConfig { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let mut table = toml::Table::deserialize(deserializer)?; + let id = take_required(&mut table, "id")?; + let context_window = take_optional(&mut table, "context_window")?; + let tool_calling = take_optional(&mut table, "tool_calling")?; + let reasoning = take_optional(&mut table, "reasoning")?; + let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table)) + .map_err(serde::de::Error::custom)?; + Ok(Self { + id, + context_window, + tool_calling, + reasoning, + algorithm, + }) + } +} + +fn take_required(table: &mut toml::Table, name: &'static str) -> Result +where + T: DeserializeOwned, + E: serde::de::Error, +{ + let value = table.remove(name).ok_or_else(|| E::missing_field(name))?; + T::deserialize(value).map_err(E::custom) +} + +fn take_optional(table: &mut toml::Table, name: &'static str) -> Result, E> +where + T: DeserializeOwned, + E: serde::de::Error, +{ + table + .remove(name) + .map(|value| T::deserialize(value).map_err(E::custom)) + .transpose() +} + +impl RouteConfig { + fn capabilities(&self) -> ModelCapabilities { + ModelCapabilities { + context_window: self.context_window, + tool_calling: self.tool_calling, + reasoning: self.reasoning, + } + } + + fn routing_target_names(&self) -> Vec<&str> { + self.algorithm.routing_target_names() + } + + fn callable_target_names(&self) -> Vec<&str> { + self.algorithm.callable_target_names() + } +} + +impl DeploymentConfig { + fn decision_target(&self, name: &str) -> Option { + let target = self.targets.get(name)?; + let client = self.llm_clients.get(&target.llm_client)?; + Some(DecisionTarget { + target: name.to_string(), + model: target.id.clone(), + format: client.format.wire_format(), + base_url: client.base_url.as_str().to_string(), + extra_body: target.extra_body.clone(), + }) + } + + fn build(self) -> RunnerResult { + if self.schema_version != SUPPORTED_SCHEMA_VERSION { + return Err(RunnerError::configuration(format!( + "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}", + self.schema_version + ))); + } + + let mut seen_client_model_ids = HashSet::new(); + for (target_name, target) in &self.targets { + validate_value("target name", target_name)?; + validate_value(&format!("target {target_name} id"), &target.id)?; + if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) { + tracing::warn!( + "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.", + target.id, + target.llm_client + ); + } + } + + let clients = self.build_clients()?; + let targets = self.build_targets(); + let mut routes = Vec::with_capacity(self.routes.len()); + for (route_name, config) in &self.routes { + validate_value("route name", route_name)?; + validate_value(&format!("route {route_name} id"), &config.id)?; + for target_name in config.callable_target_names() { + self.targets.get(target_name).ok_or_else(|| { + RunnerError::configuration(format!( + "route references unknown target {target_name}" + )) + })?; + } + let capabilities = config.capabilities(); + if capabilities.context_window == Some(0) { + return Err(RunnerError::configuration(format!( + "route {route_name} context_window must be greater than zero" + ))); + } + let algorithm = config + .algorithm + .build(route_name, &targets) + .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?; + let (route_clients, caller_auth) = + self.build_route_clients(route_name, config, &clients)?; + let count_tokens_target = self.build_count_tokens_target(config, &clients); + let decision_targets = config + .routing_target_names() + .into_iter() + .filter_map(|name| self.decision_target(name)) + .collect(); + let route = Route::new( + algorithm, + route_clients, + caller_auth, + capabilities, + count_tokens_target, + decision_targets, + ); + routes.push((config.id.clone(), route)); + } + Ok(Runner::new(routes)) + } + + fn build_clients(&self) -> RunnerResult>> { + let mut models_by_client = self + .llm_clients + .keys() + .map(|name| (name.clone(), Vec::new())) + .collect::>>(); + + for (name, client_config) in &self.llm_clients { + validate_value("llm client name", name)?; + build_backend(name, client_config, &BTreeMap::new())?; + } + for (target_name, target) in &self.targets { + let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| { + RunnerError::configuration(format!( + "target {target_name} references unknown llm client {}", + target.llm_client + )) + })?; + let model_configs = models_by_client + .get_mut(&target.llm_client) + .ok_or_else(|| { + RunnerError::configuration("validated llm client was not initialized") + })?; + model_configs.push(ModelConfig::new( + target.id.clone(), + build_backend(&target.llm_client, client_config, &target.extra_body)?, + None, + )); + } + + let mut clients = BTreeMap::new(); + for (name, model_configs) in models_by_client { + let client = Arc::new( + TranslatingLlmClient::new(&model_configs) + .map_err(|error| RunnerError::configuration(error.to_string()))?, + ); + clients.insert(name, client); + } + Ok(clients) + } + + fn build_targets(&self) -> BTreeMap { + self.targets + .iter() + .map(|(name, config)| (name.clone(), config.id.clone())) + .collect() + } + + fn build_route_clients( + &self, + route_name: &str, + route: &RouteConfig, + clients: &BTreeMap>, + ) -> RunnerResult<(ClientRouter, Option)> { + let mut by_model = HashMap::new(); + let mut caller_auth = None; + for name in route.callable_target_names() { + let target = self.targets.get(name).ok_or_else(|| { + RunnerError::configuration(format!("route references unknown target {name}")) + })?; + let client = clients.get(&target.llm_client).ok_or_else(|| { + RunnerError::configuration(format!("target {name} has no constructed llm client")) + })?; + let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| { + RunnerError::configuration(format!( + "target {name} references unknown llm client {}", + target.llm_client + )) + })?; + if client_config.forward_auth { + let target_auth = client_config.format.caller_auth_kind(); + if caller_auth.is_some_and(|kind| kind != target_auth) { + return Err(RunnerError::configuration(format!( + "route {route_name} cannot forward both Anthropic and OpenAI caller credentials" + ))); + } + caller_auth = Some(target_auth); + } + let client: Arc = client.clone(); + by_model.insert(target.id.clone(), client); + } + Ok((ClientRouter::new(by_model), caller_auth)) + } + + fn build_count_tokens_target( + &self, + route: &RouteConfig, + clients: &BTreeMap>, + ) -> Option { + route + .routing_target_names() + .into_iter() + .enumerate() + .filter_map(|(index, name)| { + let target = self.targets.get(name)?; + let client = clients.get(&target.llm_client)?; + client.supports_count_tokens(&target.id).then_some(( + count_tokens_priority(name, &target.id), + index, + target, + client, + )) + }) + .min_by_key(|(priority, index, _, _)| (*priority, *index)) + .map(|(_, _, target, client)| CountTokensTarget { + model: target.id.clone(), + client: client.clone(), + }) + } +} + +fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { + let target_name = target_name.to_ascii_lowercase(); + let model_id = model_id.to_ascii_lowercase(); + ["opus", "sonnet", "haiku"] + .iter() + .position(|hint| target_name.contains(hint) || model_id.contains(hint)) + .unwrap_or(3) +} + +/// A client endpoint, parsed when the config loads rather than checked afterwards. +/// +/// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no +/// later stage has to re-check it or can forget to. +#[derive(Clone, Debug)] +struct HttpBaseUrl(reqwest::Url); + +impl HttpBaseUrl { + fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl<'de> Deserialize<'de> for HttpBaseUrl { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + let url = reqwest::Url::parse(raw.trim()).map_err(|error| { + serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}")) + })?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err(serde::de::Error::custom( + "base_url must be an absolute HTTP(S) URL", + )); + } + Ok(Self(url)) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct LlmClientConfig { + format: ClientFormat, + base_url: HttpBaseUrl, + api_key_env: Option, + #[serde(default)] + forward_auth: bool, + #[serde(default)] + extra_headers: BTreeMap, + #[serde(default = "default_max_retries")] + max_retries: u32, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TargetConfig { + id: ModelId, + llm_client: String, + #[serde(default)] + extra_body: BTreeMap, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +enum ClientFormat { + #[serde(rename = "openai_chat")] + OpenAiChat, + #[serde(rename = "openai_responses")] + OpenAiResponses, + #[serde(rename = "anthropic_messages")] + AnthropicMessages, +} + +impl ClientFormat { + const fn wire_format(self) -> WireFormat { + match self { + Self::OpenAiChat => WireFormat::OpenAiChat, + Self::OpenAiResponses => WireFormat::OpenAiResponses, + Self::AnthropicMessages => WireFormat::AnthropicMessages, + } + } + + const fn caller_auth_kind(self) -> CallerAuthKind { + match self { + Self::AnthropicMessages => CallerAuthKind::Anthropic, + Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi, + } + } +} +fn build_backend( + client_name: &str, + config: &LlmClientConfig, + extra_body: &BTreeMap, +) -> RunnerResult { + if config.max_retries > MAX_CONFIGURED_RETRIES { + return Err(RunnerError::configuration(format!( + "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}" + ))); + } + if config.forward_auth && config.api_key_env.is_some() { + return Err(RunnerError::configuration(format!( + "llm client {client_name} cannot set both forward_auth and api_key_env" + ))); + } + let api_key = config + .api_key_env + .as_deref() + .map(|variable| { + if variable.trim().is_empty() { + return Err(RunnerError::configuration(format!( + "llm client {client_name} api_key_env must not be empty" + ))); + } + let api_key = std::env::var(variable).map_err(|error| { + RunnerError::configuration(format!( + "llm client {client_name} could not read api_key_env {variable}: {error}" + )) + })?; + if api_key.trim().is_empty() { + return Err(RunnerError::configuration(format!( + "llm client {client_name} api_key_env {variable} is empty" + ))); + } + Ok(api_key) + }) + .transpose()?; + let http = HttpBackendConfig { + base_url: config.base_url.as_str().to_string(), + api_key, + forward_auth: config.forward_auth, + extra_headers: config.extra_headers.clone(), + extra_body: extra_body.clone(), + max_retries: config.max_retries, + }; + let backend = match config.format { + ClientFormat::OpenAiChat => Backend::OpenAiChat(http), + ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http), + ClientFormat::AnthropicMessages => Backend::Anthropic(http), + }; + Ok(backend) +} + +// A function so that serde default can use it. +const fn default_max_retries() -> u32 { + DEFAULT_MAX_RETRIES +} + +fn validate_value(label: &str, value: &str) -> RunnerResult<()> { + if value.trim().is_empty() || value.trim() != value { + return Err(RunnerError::configuration(format!( + "{label} must be non-empty and have no surrounding whitespace" + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_presentation_fields_are_split_from_the_algorithm() { + let route: RouteConfig = toml::from_str( + r#" +type = "random" +id = "switchyard/random" +context_window = 128000 +tool_calling = true +reasoning = false +targets = ["fast", "strong"] +weights = [1.0, 2.0] +seed = 7 +"#, + ) + .expect("route should deserialize"); + + assert_eq!(route.id, "switchyard/random"); + assert_eq!(route.context_window, Some(128_000)); + assert_eq!(route.tool_calling, Some(true)); + assert_eq!(route.reasoning, Some(false)); + assert_eq!(route.algorithm.routing_target_names(), ["fast", "strong"]); + } + + #[test] + fn route_algorithm_unknown_fields_are_rejected() { + let error = toml::from_str::( + r#" +type = "passthrough" +id = "switchyard/fast" +target = "fast" +bogus = true +"#, + ) + .expect_err("unknown route keys must be rejected"); + + assert!(error.to_string().contains("bogus"), "{error}"); + } + + #[test] + fn deployment_errors_keep_one_path_context() { + let path = Path::new("/definitely/missing/switchyard-routes.toml"); + let error = match load_runner(path) { + Ok(_) => panic!("missing config should fail"), + Err(error) => error, + }; + let message = error.to_string(); + + assert_eq!(message.matches(&path.display().to_string()).count(), 1); + assert!(message.starts_with("failed to read server config")); + } +} +#[cfg(test)] +mod deployment_tests { + use super::*; + use serde_json::json; + + const VALID_CONFIG: &str = r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[llm_clients.responses] +format = "openai_responses" +base_url = "https://example.test/v1" + +[llm_clients.anthropic] +format = "anthropic_messages" +base_url = "https://example.test" + +[targets.classifier] +id = "classifier/model" +llm_client = "primary" + +[targets.strong] +id = "strong/model" +llm_client = "responses" + +[targets.weak] +id = "weak/model" +llm_client = "anthropic" + +[routes.noop] +id = "switchyard/noop" +type = "noop" + +[routes.random] +id = "switchyard/random" +type = "random" +targets = ["strong", "weak"] + +[routes.classifier] +id = "switchyard/classifier" +type = "llm_classifier" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +base_threshold = 0.5 + +[routes.passthrough] +id = "switchyard/passthrough" +type = "passthrough" +target = "weak" +"#; + + fn error_message(toml: &str) -> String { + match runner_from_toml(toml) { + Ok(_) => "configuration unexpectedly succeeded".to_string(), + Err(error) => error.to_string(), + } + } + + fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String { + let mut configured = config.to_string(); + configured.push_str(&format!("\n[routes.{route}.subagents]\n")); + configured.push_str( + r#"type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["strong", "weak"] +default_target = "weak" +prompt = "Select a target for this delegated task." +response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["strong","weak"]}},"required":["target"],"additionalProperties":false}' +policy = { type = "target_selector", selector = "/target" } +classify_trigger = "new_session""#, + ); + configured.push_str(extra); + configured + } + + fn with_subagent_passthrough(config: &str, route: &str) -> String { + format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"") + } + + fn stage_config() -> String { + format!( + r#"{VALID_CONFIG} +[targets.stage_judge] +id = "stage-judge/model" +llm_client = "primary" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 + +[routes.stage.classifier] +target = "stage_judge" +base_threshold = 0.5 +"# + ) + } + + #[test] + fn builds_all_supported_algorithm_types() -> RunnerResult<()> { + let state = runner_from_toml(VALID_CONFIG)?; + // The model id array is sorted alphabetically + assert_eq!( + state + .models() + .map(|model| model.id.as_str()) + .collect::>(), + [ + "switchyard/classifier", + "switchyard/noop", + "switchyard/passthrough", + "switchyard/random", + ] + ); + Ok(()) + } + + #[test] + fn passthrough_and_stage_accept_subagent_routing() -> RunnerResult<()> { + let stage = stage_config(); + let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", ""); + let parsed: DeploymentConfig = toml::from_str(&stage_with_classifier).map_err(|error| { + RunnerError::configuration(format!("failed to parse stage config: {error}")) + })?; + let Some(stage_route) = parsed.routes.get("stage") else { + return Err(RunnerError::configuration("stage route is missing")); + }; + let callable_targets = stage_route.callable_target_names(); + for expected in ["strong", "weak", "stage_judge", "classifier"] { + assert!(callable_targets.contains(&expected)); + } + + for configured in [ + with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""), + with_subagent_passthrough(VALID_CONFIG, "passthrough"), + stage_with_classifier, + with_subagent_passthrough(&stage, "stage"), + ] { + runner_from_toml(&configured)?; + } + Ok(()) + } + + #[test] + fn rejects_invalid_unreferenced_llm_client() { + let invalid = format!( + "{VALID_CONFIG}\n\ + [llm_clients.unused]\n\ + format = \"openai_chat\"\n\ + base_url = \"not a url\"\n" + ); + let message = error_message(&invalid); + assert!( + message.contains("base_url must be an absolute HTTP(S) URL"), + "unexpected error: {message}" + ); + } + + #[test] + fn an_escalation_table_switches_the_classifier_route_to_escalation() -> RunnerResult<()> { + // Present: the classifier target judges the weak tier's reply each turn instead of + // picking a tier ahead of it. The route builds either way, so the assertion is that + // the knob parses and its settings reach the algorithm's validation. + let escalating = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 2 }", + ); + runner_from_toml(&escalating)?; + + // A setting that would starve the judge is rejected here rather than on the first + // request, the same as any other unusable route configuration. + let starved = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 0 }", + ); + assert!(error_message(&starved).contains("confirmations must be at least 1")); + Ok(()) + } + + #[test] + fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { + let capability = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nmax_output_tokens = 512", + ); + runner_from_toml(&capability)?; + + let escalation = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }", + ); + runner_from_toml(&escalation)?; + Ok(()) + } + + #[test] + fn classifier_prompts_are_configurable_in_both_modes() -> RunnerResult<()> { + let capability = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nprompt = \"custom capability rubric\"", + ); + runner_from_toml(&capability)?; + + let escalation = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }", + ); + runner_from_toml(&escalation)?; + + let empty = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nprompt = \" \"", + ); + assert!(error_message(&empty).contains("classifier prompt must not be empty")); + + let schema_placeholder = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"", + ); + assert!( + error_message(&schema_placeholder) + .contains("Switchyard supplies the schema automatically") + ); + Ok(()) + } + + #[test] + fn mode_custom_rejects_capability_fields() { + let mixed = VALID_CONFIG.replace( + "base_threshold = 0.5", + "mode = \"custom\"\nbase_threshold = 0.5", + ); + + assert!( + error_message(&mixed) + .contains("mode custom cannot use capability or escalation fields") + ); + } + + #[test] + fn rejects_unknown_fields_and_algorithm_types() { + let unknown_field = + VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true"); + assert!(error_message(&unknown_field).contains("unknown field")); + + let nested_completion_cap = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }", + ); + assert!(error_message(&nested_completion_cap).contains("unknown field")); + + let unknown_classifier_field = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nclassifier_magic = true", + ); + assert!(error_message(&unknown_classifier_field).contains("unknown field")); + + let target_capability = VALID_CONFIG.replace( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ncontext_window = 1000000", + ); + assert!(error_message(&target_capability).contains("unknown field `context_window`")); + + let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\""); + assert!(error_message(&unknown_algorithm).contains("unknown variant")); + } + + #[test] + fn rejects_unknown_stage_classifier_fields() { + // Nested classifier typos must fail instead of silently using a default. + let config = format!( + r#"{VALID_CONFIG} + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 + +[routes.stage.classifier] +target = "classifier" +base_threshold = 0.5 +classifier_magic = true +"# + ); + + let error = error_message(&config); + assert!( + error.contains("unknown field `classifier_magic`"), + "{error}" + ); + } + + #[test] + fn rejects_invalid_references_and_parameters() { + let cases = [ + ( + VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""), + "unknown llm client missing", + ), + ( + VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"missing\"]", + ), + "unknown target missing", + ), + ( + VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"strong\"]", + ), + "random targets must be unique", + ), + ( + VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"weak\"]\nweights = [1]", + ), + "expected 2 weights, got 1", + ), + ( + VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"weak\"]\nweights = [0, 0]", + ), + "at least one weight must be positive", + ), + ( + VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"), + "base_threshold must be between 0 and 1", + ), + ( + VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""), + "route references unknown target", + ), + ( + VALID_CONFIG.replace( + "classifier_target = \"classifier\"", + "classifier_target = \"\"", + ), + "route references unknown target", + ), + ( + VALID_CONFIG.replace( + "classifier_target = \"classifier\"", + "classifier_target = \" \"", + ), + "route references unknown target", + ), + ( + VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nthreshold_step = -0.1", + ), + "threshold_step must be finite and greater than or equal to 0", + ), + ( + VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.8\nthreshold_step = 0.11", + ), + "base_threshold + 2 * threshold_step must be at most 1", + ), + ( + VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }", + ), + "max_output_tokens must be at least 1", + ), + ( + VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nmessage_hash_fallback = true", + ), + "message_hash_fallback requires classify_trigger = new_session", + ), + ( + with_subagent_llm_classifier( + VALID_CONFIG, + "passthrough", + "\nmessage_hash_fallback = true", + ), + "cannot use message_hash_fallback", + ), + ( + with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "") + .replace("mode = \"custom\"", "mode = \"capability\""), + "mode capability cannot use custom classifier fields", + ), + ( + VALID_CONFIG.replace( + "base_threshold = 0.5", + "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"", + ), + "mode escalation cannot use classify_trigger", + ), + ( + VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"), + "unsupported schema_version 2", + ), + ( + VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"), + "target name must be non-empty and have no surrounding whitespace", + ), + ( + VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"weak\"]\ncontext_window = 0", + ), + "route random context_window must be greater than zero", + ), + ]; + + for (toml, expected) in cases { + let error = error_message(&toml); + assert!( + error.contains(expected), + "expected error containing {expected}, got {error}" + ); + } + } + + #[test] + fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> { + // Two targets share one model id on one client. The client keeps one and drops the + // other, so the build warns and still succeeds, and both routes resolve. Serving one + // model under two route names this way is allowed; pointing both routes at one target + // is the tidier form. + const SAME_MODEL_TWO_ROUTES: &str = r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[targets.fast] +id = "gpt-4o" +llm_client = "primary" + +[targets.smart] +id = "gpt-4o" +llm_client = "primary" + +[routes.fast] +id = "switchyard/fast" +type = "passthrough" +target = "fast" + +[routes.smart] +id = "switchyard/smart" +type = "passthrough" +target = "smart" +"#; + let state = runner_from_toml(SAME_MODEL_TWO_ROUTES)?; + assert_eq!( + state + .models() + .map(|model| model.id.as_str()) + .collect::>(), + ["switchyard/fast", "switchyard/smart"] + ); + Ok(()) + } + + #[test] + fn accepts_same_model_id_on_different_llm_clients() -> RunnerResult<()> { + // The same model id served by two llm clients never collides (each client keys its own + // models), so cross-provider A/B builds with no warning; only a repeat within one client + // warns. + const CROSS_PROVIDER: &str = r#" +schema_version = 1 + +[llm_clients.openai] +format = "openai_chat" +base_url = "https://example.test/v1" + +[llm_clients.azure] +format = "openai_chat" +base_url = "https://azure.test/v1" + +[targets.openai] +id = "gpt-4o" +llm_client = "openai" + +[targets.azure] +id = "gpt-4o" +llm_client = "azure" + +[routes.openai] +id = "switchyard/openai-gpt4o" +type = "passthrough" +target = "openai" + +[routes.azure] +id = "switchyard/azure-gpt4o" +type = "passthrough" +target = "azure" +"#; + runner_from_toml(CROSS_PROVIDER)?; + Ok(()) + } + + #[test] + fn accepts_relative_weights_and_seed() -> RunnerResult<()> { + let weighted = VALID_CONFIG.replace( + "targets = [\"strong\", \"weak\"]", + "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42", + ); + runner_from_toml(&weighted)?; + Ok(()) + } + + #[test] + fn accepts_new_session_trigger_with_message_hash_fallback() -> RunnerResult<()> { + let configured = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true", + ); + runner_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn target_extra_body_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> { + let configured = VALID_CONFIG.replacen( + "llm_client = \"primary\"", + "llm_client = \"primary\"\n\ + extra_body = { service_tier = \"priority\", \ + chat_template_kwargs = { enable_thinking = false } }", + 1, + ); + let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let Some(target) = config.targets.get("classifier") else { + return Err(RunnerError::configuration("classifier target is missing")); + }; + let Some(client) = config.llm_clients.get("primary") else { + return Err(RunnerError::configuration("primary llm client is missing")); + }; + let backend = build_backend("primary", client, &target.extra_body)?; + + assert_eq!( + backend.extra_body().get("service_tier"), + Some(&json!("priority")) + ); + assert_eq!( + backend + .extra_body() + .get("chat_template_kwargs") + .and_then(|value| value.get("enable_thinking")), + Some(&json!(false)) + ); + Ok(()) + } + + #[test] + fn retry_budget_defaults_and_accepts_an_override() -> RunnerResult<()> { + let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| { + RunnerError::configuration(format!("failed to parse default config: {error}")) + })?; + let Some(primary) = default.llm_clients.get("primary") else { + return Err(RunnerError::configuration("primary llm client is missing")); + }; + assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES); + + let explicit = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\nmax_retries = 0", + 1, + ); + let config: DeploymentConfig = toml::from_str(&explicit).map_err(|error| { + RunnerError::configuration(format!("failed to parse explicit retry config: {error}")) + })?; + let Some(primary) = config.llm_clients.get("primary") else { + return Err(RunnerError::configuration("primary llm client is missing")); + }; + assert_eq!(primary.max_retries, 0); + + let maximum = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + &format!( + "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}" + ), + 1, + ); + let config: DeploymentConfig = toml::from_str(&maximum).map_err(|error| { + RunnerError::configuration(format!("failed to parse maximum retry config: {error}")) + })?; + let Some(primary) = config.llm_clients.get("primary") else { + return Err(RunnerError::configuration("primary llm client is missing")); + }; + assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES); + Ok(()) + } + + #[test] + fn rejects_headers_that_switchyard_sets() { + let cases = [ + ( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\n\ + extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }", + "AUTHORIZATION", + ), + ( + "base_url = \"https://example.test\"", + "base_url = \"https://example.test\"\n\ + extra_headers = { \"X-Api-Key\" = \"custom-key\" }", + "X-Api-Key", + ), + ( + "base_url = \"https://example.test\"", + "base_url = \"https://example.test\"\n\ + extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }", + "ANTHROPIC-VERSION", + ), + ]; + + for (original, replacement, header) in cases { + let configured = VALID_CONFIG.replacen(original, replacement, 1); + let error = error_message(&configured); + assert!( + error.contains(&format!("extra_headers cannot set {header:?}")), + "expected {header} to be rejected, got: {error}" + ); + } + } + + #[test] + fn accepts_additional_headers() -> RunnerResult<()> { + let configured = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\n\ + extra_headers = { X-Inference-Priority = \"batch\" }", + 1, + ); + + runner_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn retry_budget_rejects_negative_values() { + let invalid = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\nmax_retries = -1", + 1, + ); + assert!(error_message(&invalid).contains("max_retries")); + } + + #[test] + fn retry_budget_rejects_excessive_values() { + let invalid = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\nmax_retries = 11", + 1, + ); + assert!( + error_message(&invalid).contains("llm client primary max_retries must be at most 10") + ); + } + + #[test] + fn api_key_environment_reference_is_validated() { + let missing = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"", + 1, + ); + assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET")); + + const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY"; + unsafe { + // "unsafe" is for concurrent reads and writes, very rare + std::env::set_var(EMPTY_KEY_ENV, ""); + } + let empty = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""), + 1, + ); + let message = error_message(&empty); + unsafe { + std::env::remove_var(EMPTY_KEY_ENV); + } + assert!(message.contains("is empty")); + } + + #[test] + fn forward_auth_rejects_conflicting_credentials() { + let competing_auth = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + "base_url = \"https://example.test/v1\"\n\ + forward_auth = true\n\ + api_key_env = \"UNUSED_TEST_KEY\"", + 1, + ); + assert!( + error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env") + ); + + let static_auth = VALID_CONFIG.replacen( + "base_url = \"https://example.test\"", + "base_url = \"https://example.test\"\n\ + forward_auth = true\n\ + extra_headers = { Authorization = \"static-value\" }", + 1, + ); + assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\"")); + + let static_beta = static_auth.replace("Authorization", "anthropic-beta"); + assert!( + error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"") + ); + + for header in ["chatgpt-account-id", "x-openai-fedramp"] { + let static_context = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + &format!( + "base_url = \"https://example.test/v1\"\n\ + forward_auth = true\n\ + extra_headers = {{ \"{header}\" = \"static-value\" }}" + ), + 1, + ); + assert!( + error_message(&static_context) + .contains(&format!("extra_headers cannot set \"{header}\"")) + ); + } + } + + const ADVISOR_CONFIG: &str = r#" +schema_version = 1 + +[llm_clients.anthropic] +format = "anthropic_messages" +base_url = "https://example.test" + +[targets.executor] +id = "executor/model" +llm_client = "anthropic" + +[targets.advisor] +id = "advisor/model" +llm_client = "anthropic" + +[routes.gated] +id = "switchyard/advisor" +type = "advisor" +executor_target = "executor" +advisor_target = "advisor" +"#; + + #[test] + fn advisor_route_parses_with_defaults_and_builds() -> RunnerResult<()> { + let state = runner_from_toml(ADVISOR_CONFIG)?; + assert_eq!( + state + .models() + .map(|model| model.id.as_str()) + .collect::>(), + ["switchyard/advisor"] + ); + Ok(()) + } + + #[test] + fn advisor_route_accepts_every_gate_knob() -> RunnerResult<()> { + let tuned = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + concat!( + "advisor_target = \"advisor\"\n", + "reviewer_system_prompt = \"review it\"\n", + "redo_feedback_prefix = \"REVIEWER SAYS: \"\n", + "gate_trigger = \"pattern\"\n", + "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n", + "max_reviews = 2\n", + "gate_stall_turns = 40\n", + "gate_min_tool_results = 1\n", + "advisor_max_tokens = 1024\n", + "advisor_temperature = 0.0\n", + "transcript_max_chars = 100000\n", + "fail_open = false\n", + "context_window = 200000\n", + "tool_calling = true\n", + "reasoning = true", + ), + ); + runner_from_toml(&tuned)?; + Ok(()) + } + + #[test] + fn advisor_route_rejects_unknown_keys() { + let invalid = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + "advisor_target = \"advisor\"\nbogus_field = 1", + ); + assert!(error_message(&invalid).contains("bogus_field")); + } + + #[test] + fn advisor_route_requires_both_targets() { + let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", ""); + assert!(error_message(&missing).contains("advisor_target")); + } + + #[test] + fn advisor_route_rejects_unknown_target() { + let invalid = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + "advisor_target = \"missing\"", + ); + assert!(error_message(&invalid).contains("missing")); + } + + #[test] + fn advisor_route_rejects_invalid_pattern() { + let invalid = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"", + ); + assert!(error_message(&invalid).contains("not a valid regex")); + } + + #[test] + fn advisor_route_rejects_pattern_without_pattern_trigger() { + let invalid = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"", + ); + assert!( + error_message(&invalid) + .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"") + ); + } + + #[test] + fn advisor_route_pattern_trigger_requires_pattern() { + let invalid = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"", + ); + assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern")); + } + + #[test] + fn advisor_route_rejects_zero_max_reviews() { + let invalid = ADVISOR_CONFIG.replace( + "advisor_target = \"advisor\"", + "advisor_target = \"advisor\"\nmax_reviews = 0", + ); + assert!(error_message(&invalid).contains("max_reviews must be at least 1")); + } +} diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs new file mode 100644 index 000000000..d69f2da01 --- /dev/null +++ b/crates/switchyard-runner/src/lib.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared configured routing for Switchyard serving surfaces. + +mod algorithm; +mod config; +mod route; +mod runner; + +pub use algorithm::{ + AdvisorTriggerConfig, AlgorithmConfigError, AlgorithmSpec, ClassifierMode, + ClassifierPolicyConfig, LlmClassifierRouteConfig, StageClassifierConfig, SubagentRouteConfig, +}; +pub use route::{ + CallerAuthKind, CountTokensTarget, ModelCapabilities, Route, RunOutput, RunnerError, +}; +pub use runner::{DecisionDescription, DecisionTarget, ModelInfo, Runner}; diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs new file mode 100644 index 000000000..dc4db13ee --- /dev/null +++ b/crates/switchyard-runner/src/route.rs @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One configured algorithm and the clients that serve its targets. + +use std::error::Error; +use std::sync::Arc; + +use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive}; +use serde_json::Value; +use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient}; +use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat}; +use thiserror::Error; + +use crate::DecisionTarget; + +/// Capabilities that one route advertises on `GET /v1/models`. +/// +/// An unset capability is undeclared: it serializes as `null` in the OpenAI +/// `data` entry, and the Codex entry falls back to a safe default for it. +#[derive(Clone, Copy, Default)] +pub struct ModelCapabilities { + pub context_window: Option, + pub tool_calling: Option, + /// Whether the routed model takes reasoning controls. A serving surface cannot + /// probe this, so a route opts in via config; undeclared routes advertise as + /// non-reasoning to Codex (fail closed). + pub reasoning: Option, +} + +/// Caller credential family required by a forwarded-auth route. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CallerAuthKind { + Anthropic, + OpenAi, +} + +impl CallerAuthKind { + /// Stable provider name used by the server compatibility API. + pub const fn as_str(self) -> &'static str { + match self { + Self::Anthropic => "anthropic", + Self::OpenAi => "openai", + } + } + + const fn accepts(self, wire_format: WireFormat) -> bool { + matches!( + (self, wire_format), + (Self::Anthropic, WireFormat::AnthropicMessages) + | ( + Self::OpenAi, + WireFormat::OpenAiChat | WireFormat::OpenAiResponses + ) + ) + } +} + +/// Exact upstream model used for Anthropic token counting. +#[derive(Clone)] +pub struct CountTokensTarget { + pub model: ModelId, + pub client: Arc, +} + +/// Error returned while loading or executing configured routes. +#[derive(Debug, Error)] +pub enum RunnerError { + #[error("{message}")] + Configuration { + message: String, + #[source] + source: Option>, + }, + #[error("unknown route model {0:?}")] + UnknownRouteModel(String), + #[error("caller format is incompatible with {} credentials", .0.as_str())] + IncompatibleCallerFormat(CallerAuthKind), + #[error("route has no Anthropic target for token counting")] + CountTokensUnsupported, + #[error(transparent)] + Algorithm(#[from] LibsyError), + #[error(transparent)] + Client(#[from] LlmClientError), +} + +impl RunnerError { + pub(crate) fn configuration(message: impl Into) -> Self { + Self::Configuration { + message: message.into(), + source: None, + } + } + + pub(crate) fn configuration_source( + message: impl Into, + source: impl Error + Send + Sync + 'static, + ) -> Self { + Self::Configuration { + message: message.into(), + source: Some(Box::new(source)), + } + } +} + +/// A configured algorithm and the per-target clients its calls resolve through. +pub struct Route { + algorithm: Arc, + // Resolves each offloaded call to the client configured for the target the algorithm + // selected. A route is a synthetic model with no upstream of its own, so this is a + // per-target lookup, never one client serving the whole route. + clients: ClientRouter, + caller_auth: Option, + capabilities: ModelCapabilities, + count_tokens_target: Option, + decision_targets: Vec, +} + +/// The selected model and untouched response produced by a route execution. +pub struct RunOutput { + pub selected_model: ModelId, + pub response: Response, +} + +impl Route { + /// Creates a fully configured execution route. + pub fn new( + algorithm: Arc, + clients: ClientRouter, + caller_auth: Option, + capabilities: ModelCapabilities, + count_tokens_target: Option, + decision_targets: Vec, + ) -> Self { + Self { + algorithm, + clients, + caller_auth, + capabilities, + count_tokens_target, + decision_targets, + } + } + + /// Returns the configured libsy algorithm name. + pub fn algorithm_name(&self) -> &str { + self.algorithm.name() + } + + /// Returns model-list capability metadata. + pub fn capabilities(&self) -> ModelCapabilities { + self.capabilities + } + + /// Returns the forwarded caller credential family. + pub fn caller_auth(&self) -> Option { + self.caller_auth + } + + /// Resolves a selected model to this route's non-secret target metadata. + pub(crate) fn decision_target(&self, model: &ModelId) -> Option { + self.decision_targets + .iter() + .find(|target| target.model == *model) + .cloned() + } + + /// Rejects a caller format incompatible with forwarded credentials. + pub fn check_caller_format(&self, input_format: WireFormat) -> Result<(), RunnerError> { + if let Some(kind) = self.caller_auth + && !kind.accepts(input_format) + { + return Err(RunnerError::IncompatibleCallerFormat(kind)); + } + Ok(()) + } + + /// Executes the configured route without consuming or proxying streamed responses. + pub async fn execute( + &self, + request: Request, + observer: Option, + ) -> Result { + let (selected_model, response) = switchyard_llm_client::run( + Arc::clone(&self.algorithm), + self.clients.clone(), + request, + observer, + ) + .await?; + Ok(RunOutput { + selected_model, + response, + }) + } + + /// Completes routing-time calls without serving the answer target. + pub async fn decide(&self, request: Request) -> Result { + drive(Arc::clone(&self.algorithm), request, |call| { + serve_decision_dependency(self.clients.clone(), call) + }) + .await + .map_err(Into::into) + } + + /// Counts tokens using the configured Anthropic-capable target. + pub async fn count_tokens(&self, request: Request) -> Result { + let target = self + .count_tokens_target + .as_ref() + .ok_or(RunnerError::CountTokensUnsupported)?; + target + .client + .count_tokens(&target.model, request) + .await + .map_err(Into::into) + } +} + +async fn serve_decision_dependency(clients: ClientRouter, call: CallModel) -> libsy::Result<()> { + let mut result = Err(LibsyError::NoTargets); + for (index, model) in call.models.iter().enumerate() { + // The driver stamps only the first candidate, so every fallback must replace it. + let mut request = call.request.clone(); + request.llm_request.model = Some(model.to_string()); + let response = match clients.route(model) { + Ok(client) => client.call(request).await, + Err(source) => Err(source), + }; + match response { + Ok(response) => { + result = Ok(response); + break; + } + Err(source) => { + let try_next = index + 1 < call.models.len() && eligible_routing_fallback(&source); + result = Err(LibsyError::client_call(model.clone(), source)); + if !try_next { + break; + } + } + } + } + call.respond(result) +} + +/// Whether a routing-time candidate failure may fall through to the next model. +fn eligible_routing_fallback(error: &LlmClientError) -> bool { + match error { + LlmClientError::ContextWindowExceeded { .. } + | LlmClientError::Transport { .. } + | LlmClientError::Timeout { .. } => true, + LlmClientError::UpstreamHttp { status, .. } => { + matches!( + *status, + reqwest::StatusCode::FORBIDDEN + | reqwest::StatusCode::REQUEST_TIMEOUT + | reqwest::StatusCode::TOO_MANY_REQUESTS + ) || status.is_server_error() + } + _ => false, + } +} diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs new file mode 100644 index 000000000..55760f3a3 --- /dev/null +++ b/crates/switchyard-runner/src/runner.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Named route table and server-facing route metadata. + +use std::collections::BTreeMap; +use std::path::Path; + +use libsy::RoutingOutcome; +use serde_json::Value; +use switchyard_protocol::{ModelId, WireFormat}; + +use crate::config; +use crate::{ModelCapabilities, Route, RunnerError}; + +/// Immutable named route table. +pub struct Runner { + routes: Vec<(ModelId, Route)>, +} + +/// Borrowed model metadata returned while listing routes. +pub struct ModelInfo<'a> { + pub id: &'a ModelId, + pub algorithm: &'a str, + pub capabilities: ModelCapabilities, +} + +/// Fully resolved routing decision. +pub struct DecisionDescription { + pub selected: DecisionTarget, + pub fallbacks: Vec, +} + +/// Non-secret configured target details returned by the decision endpoint. +#[derive(Clone)] +pub struct DecisionTarget { + pub target: String, + pub model: ModelId, + pub format: WireFormat, + pub base_url: String, + pub extra_body: BTreeMap, +} + +impl Runner { + /// Loads and validates a version-1 deployment TOML file. + pub fn load(path: impl AsRef) -> Result { + config::load_runner(path) + } + + /// Builds a runner from named routes in caller-provided order. + /// Pre-condition: There must be at least one route. + pub fn new(routes: Vec<(ModelId, Route)>) -> Self { + Self { routes } + } + + /// Returns the route registered for a model. + pub fn route(&self, model: &str) -> Option<&Route> { + self.routes + .iter() + .find(|(id, _)| id.as_str() == model) + .map(|(_, route)| route) + } + + /// Iterates over configured routes in caller-provided order. + pub fn models(&self) -> impl Iterator> { + self.routes.iter().map(|(id, route)| ModelInfo { + id, + algorithm: route.algorithm_name(), + capabilities: route.capabilities(), + }) + } + + /// Resolves an outcome to configured target names and non-secret client settings. + pub fn describe_decision( + &self, + model: &ModelId, + outcome: &RoutingOutcome, + ) -> Option { + let route = self.route(model.as_str())?; + let resolve = |selected: &ModelId| route.decision_target(selected); + Some(DecisionDescription { + selected: resolve(&outcome.selected_model_id)?, + fallbacks: outcome + .fallback_models + .iter() + .map(resolve) + .collect::>>()?, + }) + } +} diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs new file mode 100644 index 000000000..71ce57c7e --- /dev/null +++ b/crates/switchyard-runner/tests/route.rs @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use futures_util::StreamExt; +use switchyard_llm_client::{ClientRouter, RunObservation}; +use switchyard_protocol::{ + LlmClientError, LlmResponse, ModelId, Request, Response, RoutedLlmClient, text_request, + text_response, +}; +use switchyard_runner::{AlgorithmSpec, ModelCapabilities, Route}; + +struct StubClient; + +#[async_trait] +impl RoutedLlmClient for StubClient { + async fn call(&self, request: Request) -> Result { + Ok(Response { + llm_response: LlmResponse::Agg(text_response( + request.llm_request.model.clone(), + "plugin response", + )), + metadata: None, + }) + } +} + +fn plugin_route(client: Arc) -> Route { + let spec = AlgorithmSpec::Passthrough { + target: "semantic-target".to_string(), + subagents: None, + }; + let targets = BTreeMap::from([( + "semantic-target".to_string(), + ModelId::from("semantic-target"), + )]); + let algorithm = spec + .build("switchyard", &targets) + .expect("identity target map should build"); + let clients = ClientRouter::new( + BTreeMap::from([(ModelId::from("semantic-target"), client)]) + .into_iter() + .collect(), + ); + Route::new( + algorithm, + clients, + None, + ModelCapabilities::default(), + None, + Vec::new(), + ) +} + +#[tokio::test] +async fn plugin_shaped_route_executes_without_runner_model_or_toml() { + let route = plugin_route(Arc::new(StubClient)); + let observations = Arc::new(Mutex::new(Vec::new())); + let observer = { + let observations = Arc::clone(&observations); + Arc::new(move |observation| observations.lock().unwrap().push(observation)) + }; + let request = Request { + llm_request: text_request(Some("arbitrary-upstream-model".to_string()), "hello"), + ..Request::default() + }; + + let output = route + .execute(request, Some(observer)) + .await + .expect("route should execute"); + + assert_eq!(output.selected_model, "semantic-target"); + assert_eq!( + output + .response + .llm_response + .as_agg() + .unwrap() + .model + .as_deref(), + Some("semantic-target") + ); + assert!( + observations + .lock() + .unwrap() + .iter() + .any(|observation| { matches!(observation, RunObservation::AnswerCall(_)) }) + ); + assert!( + observations + .lock() + .unwrap() + .iter() + .any(|observation| { matches!(observation, RunObservation::RoutingOverhead(_)) }) + ); +} + +struct LazyStreamClient { + polls: Arc, +} + +#[async_trait] +impl RoutedLlmClient for LazyStreamClient { + async fn call(&self, _request: Request) -> Result { + let polls = Arc::clone(&self.polls); + let stream = futures_util::stream::poll_fn(move |_| { + polls.fetch_add(1, Ordering::SeqCst); + std::task::Poll::Ready(None) + }) + .boxed(); + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }) + } +} + +#[tokio::test] +async fn route_returns_stream_without_polling_it() { + let polls = Arc::new(AtomicUsize::new(0)); + let route = plugin_route(Arc::new(LazyStreamClient { + polls: Arc::clone(&polls), + })); + let request = Request { + llm_request: text_request(None, "hello"), + ..Request::default() + }; + + let output = route + .execute(request, None) + .await + .expect("stream handle should be returned"); + + assert!(matches!( + output.response.llm_response, + LlmResponse::Stream(_) + )); + assert_eq!(polls.load(Ordering::SeqCst), 0); +} + +#[test] +fn algorithm_build_reports_unknown_configured_target() { + let spec = AlgorithmSpec::Random { + targets: vec!["missing".to_string()], + weights: None, + seed: None, + }; + + let error = match spec.build("plugin", &BTreeMap::new()) { + Ok(_) => panic!("unknown target should fail"), + Err(error) => error, + }; + + assert_eq!( + error.to_string(), + "route plugin references unknown target missing" + ); +} diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md index efd438109..ae40d38eb 100644 --- a/crates/switchyard-server/CONFIGURATION.md +++ b/crates/switchyard-server/CONFIGURATION.md @@ -26,11 +26,11 @@ target only when the upstream supports it and would otherwise return the verdict outside normal assistant `content`. To support another wire format, add its `ClientFormat` variant and explicit construction match in -`src/config.rs`. Add a client type only when a second implementation exists. +`../switchyard-runner/src/config.rs`. Add a client type only when a second implementation exists. ## Add an algorithm 1. Implement and export the algorithm from `libsy`. -2. Add its TOML fields as an `AlgorithmConfig` variant in `src/config.rs`. -3. Construct it in the `build_algorithm` match, resolving target names with `resolve_targets`. +2. Add its fields as an `AlgorithmSpec` variant in `../switchyard-runner/src/algorithm.rs`. +3. Construct it in that module's builder, resolving names through the caller-supplied target map. 4. Add a parsing test and an end-to-end server test when the algorithm makes LLM calls. diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index 5502a0e8c..0c4b47e24 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -32,11 +32,10 @@ opentelemetry-prometheus = "0.32" opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "trace"] } parking_lot.workspace = true prometheus = "0.14" -reqwest.workspace = true serde.workspace = true -toml = "1.1" switchyard-llm-client.workspace = true switchyard-protocol.workspace = true +switchyard-runner.workspace = true serde_json.workspace = true switchyard-translation.workspace = true humantime = "2.4" diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 77373959b..ba88b3445 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -1,2262 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Typed TOML configuration and explicit construction for the Rust server. +//! Compatibility wrapper for loading shared runner configuration. -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs; use std::path::Path; -use std::sync::Arc; -use libsy::{ - AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, - ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, - GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, - Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, - SubagentRouterConfig, TargetPrompts, TaskClassifierConfig, -}; -use serde::Deserialize; -use serde_json::Value; -use switchyard_llm_client::{ - Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, -}; -use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; +use switchyard_runner::Runner; -use crate::{ - CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState, -}; - -const SUPPORTED_SCHEMA_VERSION: u32 = 1; -const MAX_CONFIGURED_RETRIES: u32 = 10; +use crate::{ServerError, ServerResult, ServerState}; /// Loads a TOML deployment file and constructs the complete server state. pub fn load_server_state(path: impl AsRef) -> ServerResult { - let path = path.as_ref(); - let toml = fs::read_to_string(path).map_err(|error| { - ServerError::new(format!( - "failed to read server config {}: {error}", - path.display() - )) - })?; - server_state_from_toml(&toml).map_err(|error| { - ServerError::new(format!("invalid server config {}: {error}", path.display())) - }) -} - -fn server_state_from_toml(toml: &str) -> ServerResult { - let config: Arc = Arc::new( - toml::from_str(toml) - .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?, - ); - let state = config.build()?; - Ok(state.with_config(config)) -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ServerConfig { - schema_version: u32, - #[serde(default)] - pub(crate) llm_clients: BTreeMap, - pub(crate) targets: BTreeMap, - routes: BTreeMap, -} - -impl ServerConfig { - pub(crate) fn routing_target_names(&self, route_name: &str) -> Option> { - self.routes - .get(route_name) - .map(RouteConfig::routing_target_names) - } - - fn build(&self) -> ServerResult { - if self.schema_version != SUPPORTED_SCHEMA_VERSION { - return Err(ServerError::new(format!( - "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}", - self.schema_version - ))); - } - - // An llm client keys its models by id, so two targets that share an id on one - // client end up as one entry: the client keeps one target and drops the other. Warn - // so the drop is visible at startup instead of silent, and still build both routes. - // The set only tells us the pair was already seen, not whether the two targets - // differ, so it warns for a harmless duplicate and for one whose extra_body would be - // dropped alike. The same id on two different clients never collides: each client - // keeps its own model. - let mut seen_client_model_ids = HashSet::new(); - for (target_name, target) in &self.targets { - validate_value("target name", target_name)?; - validate_value(&format!("target {target_name} id"), &target.id)?; - if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) { - tracing::warn!( - "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.", - target.id, - target.llm_client - ); - } - } - - let clients = self.build_clients()?; - let targets = self.build_targets()?; - let mut routes = Vec::with_capacity(self.routes.len()); - for (route_name, config) in &self.routes { - validate_value("route name", route_name)?; - validate_value(&format!("route {route_name} id"), config.id())?; - for target_name in config.callable_target_names() { - self.targets.get(target_name).ok_or_else(|| { - ServerError::new(format!("route references unknown target {target_name}")) - })?; - } - let capabilities = config.capabilities(); - if capabilities.context_window == Some(0) { - return Err(ServerError::new(format!( - "route {route_name} context_window must be greater than zero" - ))); - } - let algorithm = build_algorithm(route_name, config, &targets)?; - let (client, caller_auth) = self.build_route_clients(route_name, config, &clients)?; - let count_tokens_target = self.build_count_tokens_target(config, &clients); - routes.push(( - config.id().clone(), - algorithm, - client, - caller_auth, - capabilities, - count_tokens_target, - Some(route_name.clone()), - )); - } - ServerState::new_with_capabilities(routes) - } - - fn build_clients(&self) -> ServerResult>> { - let mut models_by_client = self - .llm_clients - .keys() - .map(|name| (name.clone(), Vec::new())) - .collect::>>(); - - // Validate every declared client even when no target currently references it. - for (name, client_config) in &self.llm_clients { - validate_value("llm client name", name)?; - build_backend(name, client_config, &BTreeMap::new())?; - } - for (target_name, target) in &self.targets { - let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| { - ServerError::new(format!( - "target {target_name} references unknown llm client {}", - target.llm_client - )) - })?; - let model_configs = models_by_client - .get_mut(&target.llm_client) - .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?; - model_configs.push(ModelConfig::new( - target.id.clone(), - build_backend(&target.llm_client, client_config, &target.extra_body)?, - None, - )); - } - - let mut clients = BTreeMap::new(); - for (name, model_configs) in models_by_client { - let client = Arc::new( - TranslatingLlmClient::new(&model_configs) - .map_err(|error| ServerError::new(error.to_string()))?, - ); - clients.insert(name, client); - } - Ok(clients) - } - - fn build_targets(&self) -> ServerResult> { - Ok(self - .targets - .iter() - .map(|(name, config)| (name.clone(), config.id.clone())) - .collect()) - } - - /// Maps every model this route can select to the client configured for that target. - /// - /// A route is a synthetic model (`switchyard/random`) with no upstream of its own — the - /// algorithm picks a real target, and the *target* names the `[llm_clients.*]` section - /// that serves it. Two targets in one route may therefore sit on different clients, and - /// one request can emit calls for both. - /// - /// The map is built per route because targets are only reachable through the routes that - /// name them, so the same model id may resolve to different clients in two routes without - /// colliding. - fn build_route_clients( - &self, - route_name: &str, - route: &RouteConfig, - clients: &BTreeMap>, - ) -> ServerResult<(ClientRouter, Option)> { - let mut by_model = HashMap::new(); - let mut caller_auth = None; - for name in route.callable_target_names() { - let target = self.targets.get(name).ok_or_else(|| { - ServerError::new(format!("route references unknown target {name}")) - })?; - let client = clients.get(&target.llm_client).ok_or_else(|| { - ServerError::new(format!("target {name} has no constructed llm client")) - })?; - let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| { - ServerError::new(format!( - "target {name} references unknown llm client {}", - target.llm_client - )) - })?; - if client_config.forward_auth { - let target_auth = client_config.format.caller_auth_kind(); - if caller_auth.is_some_and(|kind| kind != target_auth) { - return Err(ServerError::new(format!( - "route {route_name} cannot forward both Anthropic and OpenAI caller credentials" - ))); - } - caller_auth = Some(target_auth); - } - let client: Arc = client.clone(); - by_model.insert(target.id.clone(), client); - } - Ok((ClientRouter::new(by_model), caller_auth)) - } - - fn build_count_tokens_target( - &self, - route_config: &RouteConfig, - clients: &BTreeMap>, - ) -> Option { - route_config - .routing_target_names() - .into_iter() - .enumerate() - .filter_map(|(index, name)| { - let target = self.targets.get(name)?; - let client = clients.get(&target.llm_client)?; - client.supports_count_tokens(&target.id).then_some(( - count_tokens_priority(name, &target.id), - index, - target, - client, - )) - }) - .min_by_key(|(priority, index, _, _)| (*priority, *index)) - .map(|(_, _, target, client)| CountTokensTarget { - model: target.id.clone(), - client: client.clone(), - }) - } -} - -// Prefer known Claude families, then preserve the route's target order. -fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { - let target_name = target_name.to_ascii_lowercase(); - let model_id = model_id.to_ascii_lowercase(); - ["opus", "sonnet", "haiku"] - .iter() - .position(|hint| target_name.contains(hint) || model_id.contains(hint)) - .unwrap_or(3) -} - -/// A client endpoint, parsed when the config loads rather than checked afterwards. -/// -/// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no -/// later stage has to re-check it or can forget to. -#[derive(Clone, Debug)] -pub(crate) struct HttpBaseUrl(reqwest::Url); - -impl HttpBaseUrl { - pub(crate) fn as_str(&self) -> &str { - self.0.as_str() - } -} - -impl<'de> Deserialize<'de> for HttpBaseUrl { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let raw = String::deserialize(deserializer)?; - let url = reqwest::Url::parse(raw.trim()).map_err(|error| { - serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}")) - })?; - if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { - return Err(serde::de::Error::custom( - "base_url must be an absolute HTTP(S) URL", - )); - } - Ok(Self(url)) - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct LlmClientConfig { - pub(crate) format: ClientFormat, - pub(crate) base_url: HttpBaseUrl, - api_key_env: Option, - #[serde(default)] - forward_auth: bool, - #[serde(default)] - extra_headers: BTreeMap, - #[serde(default = "default_max_retries")] - max_retries: u32, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct TargetConfig { - pub(crate) id: ModelId, - pub(crate) llm_client: String, - #[serde(default)] - pub(crate) extra_body: BTreeMap, -} - -#[derive(Clone, Copy, Debug, Deserialize)] -pub(crate) enum ClientFormat { - #[serde(rename = "openai_chat")] - OpenAiChat, - #[serde(rename = "openai_responses")] - OpenAiResponses, - #[serde(rename = "anthropic_messages")] - AnthropicMessages, -} - -impl ClientFormat { - pub(crate) const fn wire_format(self) -> WireFormat { - match self { - Self::OpenAiChat => WireFormat::OpenAiChat, - Self::OpenAiResponses => WireFormat::OpenAiResponses, - Self::AnthropicMessages => WireFormat::AnthropicMessages, - } - } - - const fn caller_auth_kind(self) -> CallerAuthKind { - match self { - Self::AnthropicMessages => CallerAuthKind::Anthropic, - Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi, - } - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum ClassifierPolicyConfig { - TargetSelector { selector: String }, -} - -#[derive(Clone, Copy, Debug, Deserialize)] -#[serde(rename_all = "snake_case")] -enum ClassifierMode { - Capability, - Escalation, - Custom, -} - -impl ClassifierPolicyConfig { - fn into_libsy(self) -> CustomClassifierPolicy { - match self { - Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector), - } - } -} - -#[derive(Debug)] -enum LlmClassifierModeConfig { - Capability(CapabilityClassifierRouteConfig), - Escalation(EscalationClassifierRouteConfig), - Custom(CustomClassifierRouteConfig), -} - -#[derive(Debug)] -struct CapabilityClassifierRouteConfig { - strong_target: String, - weak_target: String, - base_threshold: f64, - threshold_step: f64, - classify_trigger: ClassifyTrigger, - message_hash_fallback: bool, - recent_turn_window: Option, - prompt: Option, - response_format_type: ClassifierResponseFormat, - max_output_tokens: u64, -} - -#[derive(Debug)] -struct EscalationClassifierRouteConfig { - strong_target: String, - weak_target: String, - prompt: Option, - response_format_type: ClassifierResponseFormat, - max_output_tokens: u64, - judge: EscalationJudgeConfig, -} - -#[derive(Debug)] -struct CustomClassifierRouteConfig { - classifier_target: String, - targets: Vec, - default_target: String, - prompt: String, - response_schema: String, - policy: ClassifierPolicyConfig, - classify_trigger: ClassifyTrigger, - message_hash_fallback: bool, - recent_turn_window: Option, - max_output_tokens: u64, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct LlmClassifierRouteConfig { - classifier_target: String, - mode: Option, - strong_target: Option, - weak_target: Option, - base_threshold: Option, - threshold_step: Option, - classify_trigger: ClassifyTrigger, - message_hash_fallback: bool, - recent_turn_window: Option, - prompt: Option, - response_format_type: ClassifierResponseFormat, - #[serde(default = "default_classifier_max_output_tokens")] - max_output_tokens: u64, - escalation: Option, - targets: Option>, - default_target: Option, - response_schema: Option, - policy: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum SubagentRouteConfig { - Passthrough { target: String }, - LlmClassifier(Box), -} - -impl SubagentRouteConfig { - fn routing_target_names(&self) -> Vec<&str> { - match self { - Self::Passthrough { target } => vec![target], - Self::LlmClassifier(classifier) => classifier - .targets - .iter() - .flatten() - .map(String::as_str) - .collect(), - } - } - - fn classifier_target_name(&self) -> Option<&str> { - match self { - Self::LlmClassifier(classifier) => Some(&classifier.classifier_target), - Self::Passthrough { .. } => None, - } - } -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum RouteConfig { - Noop { - id: ModelId, - #[serde(default)] - context_window: Option, - #[serde(default)] - tool_calling: Option, - #[serde(default)] - reasoning: Option, - }, - Random { - id: ModelId, - #[serde(default)] - context_window: Option, - #[serde(default)] - tool_calling: Option, - #[serde(default)] - reasoning: Option, - targets: Vec, - weights: Option>, - seed: Option, - }, - Passthrough { - id: ModelId, - #[serde(default)] - context_window: Option, - #[serde(default)] - tool_calling: Option, - #[serde(default)] - reasoning: Option, - target: String, - #[serde(default)] - subagents: Option, - }, - LlmClassifier { - id: ModelId, - #[serde(default)] - context_window: Option, - #[serde(default)] - tool_calling: Option, - #[serde(default)] - reasoning: Option, - #[serde(flatten)] - config: LlmClassifierRouteConfig, - }, - StageRouter { - id: ModelId, - #[serde(default)] - context_window: Option, - #[serde(default)] - tool_calling: Option, - #[serde(default)] - reasoning: Option, - capable_target: String, - efficient_target: String, - /// Tier a turn falls back to when the signals are not confident. - picker: PickerMode, - confidence_threshold: f64, - /// Trailing tool results the signals are computed over. - #[serde(default)] - recent_turn_window: Option, - /// Note handed to the model a signal-driven switch routes to. - #[serde(default)] - handoff_notes: Option, - /// System prompt handed to each tier on every turn it serves. - #[serde(default)] - capable_system_prompt: Option, - #[serde(default)] - efficient_system_prompt: Option, - /// Capability judge consulted on turns the signals leave undecided. - #[serde(default)] - classifier: Option, - /// Optional routing applied only to delegated sub-agent work. - #[serde(default)] - subagents: Option, - }, - Advisor { - id: ModelId, - #[serde(default)] - context_window: Option, - #[serde(default)] - tool_calling: Option, - #[serde(default)] - reasoning: Option, - /// Serves every client-visible turn; also the count_tokens target. - executor_target: String, - /// Reviews the executor's first terminal turn. Judge-only, never a - /// routing destination. - advisor_target: String, - #[serde(default)] - reviewer_system_prompt: Option, - #[serde(default)] - redo_feedback_prefix: Option, - #[serde(default)] - gate_trigger: AdvisorTriggerConfig, - #[serde(default)] - gate_trigger_pattern: Option, - #[serde(default = "default_max_reviews")] - max_reviews: u32, - #[serde(default)] - gate_stall_turns: u32, - #[serde(default)] - gate_min_tool_results: u32, - #[serde(default = "default_advisor_max_tokens")] - advisor_max_tokens: u64, - #[serde(default)] - advisor_temperature: Option, - #[serde(default = "default_transcript_max_chars")] - transcript_max_chars: usize, - #[serde(default = "default_fail_open")] - fail_open: bool, - }, -} - -/// What fires an advisor route's review. -#[derive(Debug, Default, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -enum AdvisorTriggerConfig { - /// The executor's first turn without tool calls. - #[default] - NoToolCall, - /// The first turn whose text matches `gate_trigger_pattern`. - Pattern, -} - -/// The judge a `stage_router` route falls through to, and how it routes. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct StageClassifierConfig { - /// Target the judge is called through. Not a routing destination. - target: String, - base_threshold: f64, - #[serde(default)] - threshold_step: f64, - #[serde(default)] - classify_trigger: ClassifyTrigger, - #[serde(default)] - message_hash_fallback: bool, - #[serde(default)] - recent_turn_window: Option, - #[serde(default)] - prompt: Option, - #[serde(default)] - response_format_type: ClassifierResponseFormat, - #[serde(default = "default_classifier_max_output_tokens")] - max_output_tokens: u64, -} - -impl StageClassifierConfig { - fn task_classifier_config(&self) -> TaskClassifierConfig { - TaskClassifierConfig { - base_threshold: self.base_threshold, - threshold_step: self.threshold_step, - classify_trigger: self.classify_trigger, - message_hash_fallback: self.message_hash_fallback, - recent_turn_window: self.recent_turn_window, - contract: classifier_contract(self.prompt.as_deref()) - .with_response_format_type(self.response_format_type), - max_output_tokens: self.max_output_tokens, - } - } -} - -impl RouteConfig { - fn id(&self) -> &ModelId { - use RouteConfig::*; - match self { - Noop { id, .. } - | Random { id, .. } - | LlmClassifier { id, .. } - | Passthrough { id, .. } - | StageRouter { id, .. } - | Advisor { id, .. } => id, - } - } - - // Completion targets in algorithm order; judge-only targets are excluded. - fn routing_target_names(&self) -> Vec<&str> { - match self { - Self::Noop { .. } => Vec::new(), - Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(), - Self::Passthrough { - target, subagents, .. - } => { - let mut names = vec![target.as_str()]; - if let Some(subagents) = subagents { - names.extend(subagents.routing_target_names()); - } - names - } - Self::LlmClassifier { config, .. } => { - match config.mode.unwrap_or(if config.escalation.is_some() { - ClassifierMode::Escalation - } else { - ClassifierMode::Capability - }) { - ClassifierMode::Capability => config - .weak_target - .iter() - .chain(&config.strong_target) - .map(String::as_str) - .collect(), - ClassifierMode::Escalation => config - .strong_target - .iter() - .chain(&config.weak_target) - .map(String::as_str) - .collect(), - ClassifierMode::Custom => config - .targets - .iter() - .flatten() - .map(String::as_str) - .collect(), - } - } - Self::StageRouter { - capable_target, - efficient_target, - subagents, - .. - } => { - let mut names = vec![capable_target.as_str(), efficient_target.as_str()]; - if let Some(subagents) = subagents { - names.extend(subagents.routing_target_names()); - } - names - } - // The advisor is judge-only: reviews go through its own client, - // so it is not a completion (or count_tokens) destination. - Self::Advisor { - executor_target, .. - } => vec![executor_target], - } - } - - /// Every target the algorithm may call, including judge-only targets. - /// - /// [`routing_target_names`](Self::routing_target_names) covers completion destinations; - /// a classifier also calls its judge, and that call needs a client too. - fn callable_target_names(&self) -> Vec<&str> { - let mut names = self.routing_target_names(); - match self { - Self::LlmClassifier { config, .. } => names.push(&config.classifier_target), - Self::Passthrough { - subagents: Some(subagents), - .. - } => names.extend(subagents.classifier_target_name()), - Self::StageRouter { - classifier, - subagents, - .. - } => { - if let Some(classifier) = classifier { - names.push(&classifier.target); - } - if let Some(subagents) = subagents { - names.extend(subagents.classifier_target_name()); - } - } - Self::Advisor { advisor_target, .. } => names.push(advisor_target), - _ => {} - } - names - } - - fn capabilities(&self) -> ModelCapabilities { - use RouteConfig::*; - match self { - Noop { - context_window, - tool_calling, - reasoning, - .. - } - | Random { - context_window, - tool_calling, - reasoning, - .. - } - | Passthrough { - context_window, - tool_calling, - reasoning, - .. - } - | LlmClassifier { - context_window, - tool_calling, - reasoning, - .. - } - | StageRouter { - context_window, - tool_calling, - reasoning, - .. - } - | Advisor { - context_window, - tool_calling, - reasoning, - .. - } => ModelCapabilities { - context_window: *context_window, - tool_calling: *tool_calling, - reasoning: *reasoning, - }, - } - } -} - -impl LlmClassifierRouteConfig { - fn classifier_mode(&self, route_name: &str) -> ServerResult { - let Self { - classifier_target, - mode, - strong_target, - weak_target, - base_threshold, - threshold_step, - classify_trigger, - message_hash_fallback, - recent_turn_window, - prompt, - response_format_type, - max_output_tokens, - escalation, - targets, - default_target, - response_schema, - policy, - } = self; - - let selected_mode = match (mode, escalation.is_some()) { - (Some(mode), _) => *mode, - (None, true) => ClassifierMode::Escalation, - (None, false) => ClassifierMode::Capability, - }; - - match selected_mode { - ClassifierMode::Capability => { - if escalation.is_some() { - return Err(classifier_field_error( - route_name, - "escalation", - "capability", - )); - } - reject_custom_fields( - route_name, - "capability", - targets, - default_target, - response_schema, - policy, - )?; - Ok(LlmClassifierModeConfig::Capability( - CapabilityClassifierRouteConfig { - strong_target: required_classifier_field( - route_name, - "strong_target", - strong_target, - )?, - weak_target: required_classifier_field( - route_name, - "weak_target", - weak_target, - )?, - base_threshold: required_classifier_field( - route_name, - "base_threshold", - base_threshold, - )?, - threshold_step: threshold_step.unwrap_or_default(), - classify_trigger: *classify_trigger, - message_hash_fallback: *message_hash_fallback, - recent_turn_window: *recent_turn_window, - prompt: prompt.clone(), - response_format_type: *response_format_type, - max_output_tokens: *max_output_tokens, - }, - )) - } - ClassifierMode::Escalation => { - reject_custom_fields( - route_name, - "escalation", - targets, - default_target, - response_schema, - policy, - )?; - if *classify_trigger != ClassifyTrigger::EveryRequest { - return Err(ServerError::new(format!( - "llm_classifier route {route_name} mode escalation cannot use classify_trigger" - ))); - } - if mode.is_some() - && (base_threshold.is_some() - || threshold_step.is_some() - || *message_hash_fallback - || recent_turn_window.is_some()) - { - return Err(ServerError::new(format!( - "llm_classifier route {route_name} mode escalation cannot use capability routing settings" - ))); - } - Ok(LlmClassifierModeConfig::Escalation( - EscalationClassifierRouteConfig { - strong_target: required_classifier_field( - route_name, - "strong_target", - strong_target, - )?, - weak_target: required_classifier_field( - route_name, - "weak_target", - weak_target, - )?, - prompt: prompt.clone(), - response_format_type: *response_format_type, - max_output_tokens: *max_output_tokens, - judge: required_classifier_field(route_name, "escalation", escalation)?, - }, - )) - } - ClassifierMode::Custom => { - if strong_target.is_some() - || weak_target.is_some() - || base_threshold.is_some() - || threshold_step.is_some() - || escalation.is_some() - || *response_format_type != ClassifierResponseFormat::JsonSchema - { - return Err(ServerError::new(format!( - "llm_classifier route {route_name} mode custom cannot use capability or escalation fields" - ))); - } - Ok(LlmClassifierModeConfig::Custom( - CustomClassifierRouteConfig { - classifier_target: classifier_target.clone(), - targets: required_classifier_field(route_name, "targets", targets)?, - default_target: required_classifier_field( - route_name, - "default_target", - default_target, - )?, - prompt: required_classifier_field(route_name, "prompt", prompt)?, - response_schema: required_classifier_field( - route_name, - "response_schema", - response_schema, - )?, - policy: required_classifier_field(route_name, "policy", policy)?, - classify_trigger: *classify_trigger, - message_hash_fallback: *message_hash_fallback, - recent_turn_window: *recent_turn_window, - max_output_tokens: *max_output_tokens, - }, - )) - } - } - } -} - -fn reject_custom_fields( - route_name: &str, - mode: &str, - targets: &Option>, - default_target: &Option, - response_schema: &Option, - policy: &Option, -) -> ServerResult<()> { - if targets.is_some() - || default_target.is_some() - || response_schema.is_some() - || policy.is_some() - { - return Err(ServerError::new(format!( - "llm_classifier route {route_name} mode {mode} cannot use custom classifier fields" - ))); - } - Ok(()) -} - -fn classifier_field_error(route_name: &str, field: &str, mode: &str) -> ServerError { - ServerError::new(format!( - "llm_classifier route {route_name} mode {mode} cannot use {field}" - )) -} - -fn required_classifier_field( - route_name: &str, - field: &str, - value: &Option, -) -> ServerResult { - value.clone().ok_or_else(|| { - ServerError::new(format!( - "llm_classifier route {route_name} requires {field}" - )) - }) -} - -fn build_backend( - client_name: &str, - config: &LlmClientConfig, - extra_body: &BTreeMap, -) -> ServerResult { - if config.max_retries > MAX_CONFIGURED_RETRIES { - return Err(ServerError::new(format!( - "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}" - ))); - } - if config.forward_auth && config.api_key_env.is_some() { - return Err(ServerError::new(format!( - "llm client {client_name} cannot set both forward_auth and api_key_env" - ))); - } - let api_key = config - .api_key_env - .as_deref() - .map(|variable| { - if variable.trim().is_empty() { - return Err(ServerError::new(format!( - "llm client {client_name} api_key_env must not be empty" - ))); - } - let api_key = std::env::var(variable).map_err(|error| { - ServerError::new(format!( - "llm client {client_name} could not read api_key_env {variable}: {error}" - )) - })?; - if api_key.trim().is_empty() { - return Err(ServerError::new(format!( - "llm client {client_name} api_key_env {variable} is empty" - ))); - } - Ok(api_key) - }) - .transpose()?; - let http = HttpBackendConfig { - base_url: config.base_url.as_str().to_string(), - api_key, - forward_auth: config.forward_auth, - extra_headers: config.extra_headers.clone(), - extra_body: extra_body.clone(), - max_retries: config.max_retries, - }; - let backend = match config.format { - ClientFormat::OpenAiChat => Backend::OpenAiChat(http), - ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http), - ClientFormat::AnthropicMessages => Backend::Anthropic(http), - }; - Ok(backend) -} - -const fn default_max_retries() -> u32 { - DEFAULT_MAX_RETRIES -} - -// Resolve nested policy once so every supported parent builds the same child route. -fn build_subagent_router_config( - route_name: &str, - config: &SubagentRouteConfig, - targets: &BTreeMap, -) -> ServerResult { - match config { - SubagentRouteConfig::Passthrough { target } => Ok(SubagentRouterConfig::fixed_target( - resolve_target_model_id(route_name, target, targets)?, - )), - SubagentRouteConfig::LlmClassifier(config) => { - let LlmClassifierModeConfig::Custom(config) = config.classifier_mode(route_name)? - else { - return Err(ServerError::new(format!( - "route {route_name}: subagents llm_classifier only supports mode custom" - ))); - }; - let judge_target = - resolve_target_model_id(route_name, &config.classifier_target, targets)?; - let resolved_targets = config - .targets - .iter() - .map(|name| { - resolve_target_model_id(route_name, name, targets) - .map(|target| (name.clone(), target)) - }) - .collect::>>()?; - let default_target = resolved_targets - .iter() - .find(|(name, _)| *name == config.default_target) - .map(|(_, target)| target.clone()) - .ok_or_else(|| { - ServerError::new(format!( - "route {route_name}: subagents llm_classifier default_target {:?} must be one of its configured targets", - config.default_target - )) - })?; - let response_schema = - serde_json::from_str(&config.response_schema).map_err(|error| { - ServerError::new(format!( - "route {route_name}: subagents llm_classifier response_schema is invalid JSON: {error}" - )) - })?; - let mut classifier_config = CustomClassifierConfig::new( - config.prompt, - response_schema, - config.policy.into_libsy(), - ); - classifier_config.recent_turn_window = config.recent_turn_window; - classifier_config.max_output_tokens = config.max_output_tokens; - let subagent_targets = resolved_targets - .iter() - .map(|(_, target)| target.clone()) - .collect(); - let classifier = Arc::new( - LlmTaskClassifier::new(LlmClassifierConfig::Custom { - judge_target, - targets: resolved_targets, - default_target: config.default_target, - config: classifier_config, - }) - .map_err(|error| { - ServerError::new(format!( - "route {route_name}: subagents llm_classifier: {error}" - )) - })?, - ); - Ok(SubagentRouterConfig { - targets: subagent_targets, - classifier, - default_target, - classify_trigger: config.classify_trigger, - message_hash_fallback: config.message_hash_fallback, - }) - } - } -} - -fn attach_subagent_router( - route_name: &str, - parent: Arc, - config: Option<&SubagentRouteConfig>, - targets: &BTreeMap, -) -> ServerResult> { - let Some(config) = config else { - return Ok(parent); - }; - let config = build_subagent_router_config(route_name, config, targets)?; - let algorithm = SubagentRouter::new(parent, config).map_err(|error| { - ServerError::new(format!("route {route_name}: subagent routing: {error}")) - })?; - Ok(Arc::new(algorithm)) -} - -fn build_algorithm( - route_name: &str, - config: &RouteConfig, - targets: &BTreeMap, -) -> ServerResult> { - match config { - RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})), - RouteConfig::Random { - targets: names, - weights, - seed, - .. - } => { - let target_set = - resolve_targets(route_name, names.iter().map(String::as_str), targets)?; - let algorithm = Random::new(target_set, weights.clone(), *seed) - .map_err(|error| ServerError::new(format!("random route {route_name}: {error}")))?; - Ok(Arc::new(algorithm)) - } - RouteConfig::Passthrough { - target, subagents, .. - } => { - let parent_target = resolve_target_model_id(route_name, target, targets)?; - let algorithm = Passthrough::new(parent_target); - let parent: Arc = Arc::new(algorithm); - attach_subagent_router(route_name, parent, subagents.as_ref(), targets) - } - RouteConfig::LlmClassifier { - config: classifier_config, - .. - } => { - let classifier = - resolve_target_model_id(route_name, &classifier_config.classifier_target, targets)?; - let mode = classifier_config.classifier_mode(route_name)?; - let algorithm = match mode { - LlmClassifierModeConfig::Capability(config) => { - let strong = - resolve_target_model_id(route_name, &config.strong_target, targets)?; - let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; - let classifier_config = TaskClassifierConfig { - base_threshold: config.base_threshold, - threshold_step: config.threshold_step, - classify_trigger: config.classify_trigger, - message_hash_fallback: config.message_hash_fallback, - recent_turn_window: config.recent_turn_window, - contract: classifier_contract(config.prompt.as_deref()) - .with_response_format_type(config.response_format_type), - max_output_tokens: config.max_output_tokens, - }; - LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: classifier, - efficient_target: weak, - capable_target: strong, - config: classifier_config, - }) - } - LlmClassifierModeConfig::Escalation(config) => { - let strong = - resolve_target_model_id(route_name, &config.strong_target, targets)?; - let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; - LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: classifier, - efficient_target: weak, - capable_target: strong, - contract: classifier_contract(config.prompt.as_deref()) - .with_response_format_type(config.response_format_type), - config: config.judge, - max_output_tokens: config.max_output_tokens, - }) - } - LlmClassifierModeConfig::Custom(config) => { - let resolved_targets = config - .targets - .iter() - .map(|name| { - resolve_target_model_id(route_name, name, targets) - .map(|target| (name.clone(), target)) - }) - .collect::>>()?; - let response_schema = serde_json::from_str(&config.response_schema).map_err( - |error| { - ServerError::new(format!( - "llm_classifier route {route_name}: response_schema is invalid JSON: {error}" - )) - }, - )?; - let mut classifier_config = CustomClassifierConfig::new( - config.prompt, - response_schema, - config.policy.into_libsy(), - ); - classifier_config.classify_trigger = config.classify_trigger; - classifier_config.message_hash_fallback = config.message_hash_fallback; - classifier_config.recent_turn_window = config.recent_turn_window; - classifier_config.max_output_tokens = config.max_output_tokens; - LlmTaskClassifier::new(LlmClassifierConfig::Custom { - judge_target: classifier, - targets: resolved_targets, - default_target: config.default_target, - config: classifier_config, - }) - } - } - .map_err(|error| { - ServerError::new(format!("llm_classifier route {route_name}: {error}")) - })?; - Ok(Arc::new(algorithm)) - } - RouteConfig::StageRouter { - capable_target, - efficient_target, - picker, - confidence_threshold, - recent_turn_window, - handoff_notes, - capable_system_prompt, - efficient_system_prompt, - classifier, - subagents, - .. - } => { - if matches!(picker, PickerMode::CapableFirst) { - tracing::warn!( - "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration." - ); - } - let capable = resolve_target_model_id(route_name, capable_target, targets)?; - let efficient = resolve_target_model_id(route_name, efficient_target, targets)?; - let mut config = StageRouterConfig::new(*picker, *confidence_threshold); - config.recent_window = *recent_turn_window; - config.handoff_notes = handoff_notes.clone(); - config.tier_prompts = tier_prompts( - &capable, - capable_system_prompt.as_deref(), - &efficient, - efficient_system_prompt.as_deref(), - ); - // The judge is called through its own target, so it is not a routing - // destination and stays out of the tier pair. - config.llm_fallback = classifier - .as_ref() - .map(|classifier| { - resolve_target_model_id(route_name, &classifier.target, targets).map( - |judge_target| LlmFallback { - judge_target, - config: classifier.task_classifier_config(), - }, - ) - }) - .transpose()?; - let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { - ServerError::new(format!("stage_router route {route_name}: {error}")) - })?; - let parent: Arc = Arc::new(algorithm); - attach_subagent_router(route_name, parent, subagents.as_ref(), targets) - } - RouteConfig::Advisor { - executor_target, - advisor_target, - reviewer_system_prompt, - redo_feedback_prefix, - gate_trigger, - gate_trigger_pattern, - max_reviews, - gate_stall_turns, - gate_min_tool_results, - advisor_max_tokens, - advisor_temperature, - transcript_max_chars, - fail_open, - .. - } => { - let executor = resolve_target_model_id(route_name, executor_target, targets)?; - let advisor = resolve_target_model_id(route_name, advisor_target, targets)?; - // A pattern set under the default trigger would be silently - // ignored; reject the misconfiguration instead. - if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() { - return Err(ServerError::new(format!( - "advisor route {route_name}: gate_trigger_pattern requires \ - gate_trigger = \"pattern\"" - ))); - } - let mut config = AdvisorGateConfig::default(); - if let Some(prompt) = reviewer_system_prompt { - config.reviewer_system_prompt = prompt.clone(); - } - if let Some(prefix) = redo_feedback_prefix { - config.redo_feedback_prefix = prefix.clone(); - } - config.gate_trigger = match gate_trigger { - AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall, - AdvisorTriggerConfig::Pattern => { - GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default()) - } - }; - config.max_reviews = *max_reviews; - config.gate_stall_turns = *gate_stall_turns; - config.gate_min_tool_results = *gate_min_tool_results; - config.advisor_max_tokens = *advisor_max_tokens; - config.advisor_temperature = *advisor_temperature; - config.transcript_max_chars = *transcript_max_chars; - config.fail_open = *fail_open; - let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| { - ServerError::new(format!("advisor route {route_name}: {error}")) - })?; - Ok(Arc::new(algorithm)) - } - } -} - -const fn default_max_reviews() -> u32 { - 1 -} - -const fn default_advisor_max_tokens() -> u64 { - 2048 -} - -const fn default_transcript_max_chars() -> usize { - 200_000 -} - -const fn default_fail_open() -> bool { - true -} - -fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig { - prompt.map_or_else(ClassifierContractConfig::default, |prompt| { - ClassifierContractConfig::default().with_prompt(prompt) - }) -} - -fn default_classifier_max_output_tokens() -> u64 { - TaskClassifierConfig::default().max_output_tokens -} - -/// Keys each configured system prompt by the target it belongs to. -fn tier_prompts( - capable: &str, - capable_prompt: Option<&str>, - efficient: &str, - efficient_prompt: Option<&str>, -) -> TargetPrompts { - let mut prompts = TargetPrompts::default(); - if let Some(prompt) = capable_prompt { - prompts = prompts.with(capable, prompt); - } - if let Some(prompt) = efficient_prompt { - prompts = prompts.with(efficient, prompt); - } - prompts -} - -fn resolve_targets<'a>( - route_name: &str, - names: impl IntoIterator, - targets: &BTreeMap, -) -> ServerResult> { - names - .into_iter() - .map(|name| resolve_target_model_id(route_name, name, targets)) - .collect() -} - -fn resolve_target_model_id( - route_name: &str, - name: &str, - targets: &BTreeMap, -) -> ServerResult { - targets.get(name).cloned().ok_or_else(|| { - ServerError::new(format!( - "route {route_name} references unknown target {name}" - )) - }) -} - -fn validate_value(label: &str, value: &str) -> ServerResult<()> { - if value.trim().is_empty() || value.trim() != value { - return Err(ServerError::new(format!( - "{label} must be non-empty and have no surrounding whitespace" - ))); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - const VALID_CONFIG: &str = r#" -schema_version = 1 - -[llm_clients.primary] -format = "openai_chat" -base_url = "https://example.test/v1" - -[llm_clients.responses] -format = "openai_responses" -base_url = "https://example.test/v1" - -[llm_clients.anthropic] -format = "anthropic_messages" -base_url = "https://example.test" - -[targets.classifier] -id = "classifier/model" -llm_client = "primary" - -[targets.strong] -id = "strong/model" -llm_client = "responses" - -[targets.weak] -id = "weak/model" -llm_client = "anthropic" - -[routes.noop] -id = "switchyard/noop" -type = "noop" - -[routes.random] -id = "switchyard/random" -type = "random" -targets = ["strong", "weak"] - -[routes.classifier] -id = "switchyard/classifier" -type = "llm_classifier" -classifier_target = "classifier" -strong_target = "strong" -weak_target = "weak" -base_threshold = 0.5 - -[routes.passthrough] -id = "switchyard/passthrough" -type = "passthrough" -target = "weak" -"#; - - fn error_message(toml: &str) -> String { - match server_state_from_toml(toml) { - Ok(_) => "configuration unexpectedly succeeded".to_string(), - Err(error) => error.to_string(), - } - } - - fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String { - let mut configured = config.to_string(); - configured.push_str(&format!("\n[routes.{route}.subagents]\n")); - configured.push_str( - r#"type = "llm_classifier" -mode = "custom" -classifier_target = "classifier" -targets = ["strong", "weak"] -default_target = "weak" -prompt = "Select a target for this delegated task." -response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["strong","weak"]}},"required":["target"],"additionalProperties":false}' -policy = { type = "target_selector", selector = "/target" } -classify_trigger = "new_session""#, - ); - configured.push_str(extra); - configured - } - - fn with_subagent_passthrough(config: &str, route: &str) -> String { - format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"") - } - - fn stage_config() -> String { - format!( - r#"{VALID_CONFIG} -[targets.stage_judge] -id = "stage-judge/model" -llm_client = "primary" - -[routes.stage] -id = "switchyard/stage" -type = "stage_router" -capable_target = "strong" -efficient_target = "weak" -picker = "efficient_first" -confidence_threshold = 1.0 - -[routes.stage.classifier] -target = "stage_judge" -base_threshold = 0.5 -"# - ) - } - - #[test] - fn builds_all_supported_algorithm_types() -> ServerResult<()> { - let state = server_state_from_toml(VALID_CONFIG)?; - // The model id array is sorted alphabetically - assert_eq!( - state.models().collect::>(), - [ - "switchyard/classifier", - "switchyard/noop", - "switchyard/passthrough", - "switchyard/random", - ] - ); - Ok(()) - } - - #[test] - fn passthrough_and_stage_accept_subagent_routing() -> ServerResult<()> { - let stage = stage_config(); - let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", ""); - let parsed: ServerConfig = toml::from_str(&stage_with_classifier) - .map_err(|error| ServerError::new(format!("failed to parse stage config: {error}")))?; - let Some(stage_route) = parsed.routes.get("stage") else { - return Err(ServerError::new("stage route is missing")); - }; - let callable_targets = stage_route.callable_target_names(); - for expected in ["strong", "weak", "stage_judge", "classifier"] { - assert!(callable_targets.contains(&expected)); - } - - for configured in [ - with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""), - with_subagent_passthrough(VALID_CONFIG, "passthrough"), - stage_with_classifier, - with_subagent_passthrough(&stage, "stage"), - ] { - server_state_from_toml(&configured)?; - } - Ok(()) - } - - #[test] - fn rejects_invalid_unreferenced_llm_client() { - let invalid = format!( - "{VALID_CONFIG}\n\ - [llm_clients.unused]\n\ - format = \"openai_chat\"\n\ - base_url = \"not a url\"\n" - ); - let message = error_message(&invalid); - assert!( - message.contains("base_url must be an absolute HTTP(S) URL"), - "unexpected error: {message}" - ); - } - - #[test] - fn an_escalation_table_switches_the_classifier_route_to_escalation() -> ServerResult<()> { - // Present: the classifier target judges the weak tier's reply each turn instead of - // picking a tier ahead of it. The route builds either way, so the assertion is that - // the knob parses and its settings reach the algorithm's validation. - let escalating = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nescalation = { confirmations = 2 }", - ); - server_state_from_toml(&escalating)?; - - // A setting that would starve the judge is rejected here rather than on the first - // request, the same as any other unusable route configuration. - let starved = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nescalation = { confirmations = 0 }", - ); - assert!(error_message(&starved).contains("confirmations must be at least 1")); - Ok(()) - } - - #[test] - fn classifier_judge_completion_caps_are_configurable() -> ServerResult<()> { - let capability = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nmax_output_tokens = 512", - ); - server_state_from_toml(&capability)?; - - let escalation = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }", - ); - server_state_from_toml(&escalation)?; - Ok(()) - } - - #[test] - fn classifier_prompts_are_configurable_in_both_modes() -> ServerResult<()> { - let capability = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nprompt = \"custom capability rubric\"", - ); - server_state_from_toml(&capability)?; - - let escalation = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }", - ); - server_state_from_toml(&escalation)?; - - let empty = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nprompt = \" \"", - ); - assert!(error_message(&empty).contains("classifier prompt must not be empty")); - - let schema_placeholder = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"", - ); - assert!( - error_message(&schema_placeholder) - .contains("Switchyard supplies the schema automatically") - ); - Ok(()) - } - - #[test] - fn mode_custom_rejects_capability_fields() { - let mixed = VALID_CONFIG.replace( - "base_threshold = 0.5", - "mode = \"custom\"\nbase_threshold = 0.5", - ); - - assert!( - error_message(&mixed) - .contains("mode custom cannot use capability or escalation fields") - ); - } - - #[test] - fn rejects_unknown_fields_and_algorithm_types() { - let unknown_field = - VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true"); - assert!(error_message(&unknown_field).contains("unknown field")); - - let nested_completion_cap = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }", - ); - assert!(error_message(&nested_completion_cap).contains("unknown field")); - - let unknown_classifier_field = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nclassifier_magic = true", - ); - assert!(error_message(&unknown_classifier_field).contains("unknown field")); - - let target_capability = VALID_CONFIG.replace( - "llm_client = \"responses\"", - "llm_client = \"responses\"\ncontext_window = 1000000", - ); - assert!(error_message(&target_capability).contains("unknown field `context_window`")); - - let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\""); - assert!(error_message(&unknown_algorithm).contains("unknown variant")); - } - - #[test] - fn rejects_unknown_stage_classifier_fields() { - // Nested classifier typos must fail instead of silently using a default. - let config = format!( - r#"{VALID_CONFIG} - -[routes.stage] -id = "switchyard/stage" -type = "stage_router" -capable_target = "strong" -efficient_target = "weak" -picker = "efficient_first" -confidence_threshold = 1.0 - -[routes.stage.classifier] -target = "classifier" -base_threshold = 0.5 -classifier_magic = true -"# - ); - - let error = error_message(&config); - assert!( - error.contains("unknown field `classifier_magic`"), - "{error}" - ); - } - - #[test] - fn rejects_invalid_references_and_parameters() { - let cases = [ - ( - VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""), - "unknown llm client missing", - ), - ( - VALID_CONFIG.replace( - "targets = [\"strong\", \"weak\"]", - "targets = [\"strong\", \"missing\"]", - ), - "unknown target missing", - ), - ( - VALID_CONFIG.replace( - "targets = [\"strong\", \"weak\"]", - "targets = [\"strong\", \"strong\"]", - ), - "random targets must be unique", - ), - ( - VALID_CONFIG.replace( - "targets = [\"strong\", \"weak\"]", - "targets = [\"strong\", \"weak\"]\nweights = [1]", - ), - "expected 2 weights, got 1", - ), - ( - VALID_CONFIG.replace( - "targets = [\"strong\", \"weak\"]", - "targets = [\"strong\", \"weak\"]\nweights = [0, 0]", - ), - "at least one weight must be positive", - ), - ( - VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"), - "base_threshold must be between 0 and 1", - ), - ( - VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""), - "route references unknown target", - ), - ( - VALID_CONFIG.replace( - "classifier_target = \"classifier\"", - "classifier_target = \"\"", - ), - "route references unknown target", - ), - ( - VALID_CONFIG.replace( - "classifier_target = \"classifier\"", - "classifier_target = \" \"", - ), - "route references unknown target", - ), - ( - VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nthreshold_step = -0.1", - ), - "threshold_step must be finite and greater than or equal to 0", - ), - ( - VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.8\nthreshold_step = 0.11", - ), - "base_threshold + 2 * threshold_step must be at most 1", - ), - ( - VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }", - ), - "max_output_tokens must be at least 1", - ), - ( - VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.5\nmessage_hash_fallback = true", - ), - "message_hash_fallback requires classify_trigger = new_session", - ), - ( - with_subagent_llm_classifier( - VALID_CONFIG, - "passthrough", - "\nmessage_hash_fallback = true", - ), - "cannot use message_hash_fallback", - ), - ( - with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "") - .replace("mode = \"custom\"", "mode = \"capability\""), - "mode capability cannot use custom classifier fields", - ), - ( - VALID_CONFIG.replace( - "base_threshold = 0.5", - "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"", - ), - "mode escalation cannot use classify_trigger", - ), - ( - VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"), - "unsupported schema_version 2", - ), - ( - VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"), - "target name must be non-empty and have no surrounding whitespace", - ), - ( - VALID_CONFIG.replace( - "targets = [\"strong\", \"weak\"]", - "targets = [\"strong\", \"weak\"]\ncontext_window = 0", - ), - "route random context_window must be greater than zero", - ), - ]; - - for (toml, expected) in cases { - let error = error_message(&toml); - assert!( - error.contains(expected), - "expected error containing {expected}, got {error}" - ); - } - } - - #[test] - fn accepts_duplicate_target_model_ids_on_one_client() -> ServerResult<()> { - // Two targets share one model id on one client. The client keeps one and drops the - // other, so the build warns and still succeeds, and both routes resolve. Serving one - // model under two route names this way is allowed; pointing both routes at one target - // is the tidier form. - const SAME_MODEL_TWO_ROUTES: &str = r#" -schema_version = 1 - -[llm_clients.primary] -format = "openai_chat" -base_url = "https://example.test/v1" - -[targets.fast] -id = "gpt-4o" -llm_client = "primary" - -[targets.smart] -id = "gpt-4o" -llm_client = "primary" - -[routes.fast] -id = "switchyard/fast" -type = "passthrough" -target = "fast" - -[routes.smart] -id = "switchyard/smart" -type = "passthrough" -target = "smart" -"#; - let state = server_state_from_toml(SAME_MODEL_TWO_ROUTES)?; - assert_eq!( - state.models().collect::>(), - ["switchyard/fast", "switchyard/smart"] - ); - Ok(()) - } - - #[test] - fn accepts_same_model_id_on_different_llm_clients() -> ServerResult<()> { - // The same model id served by two llm clients never collides (each client keys its own - // models), so cross-provider A/B builds with no warning; only a repeat within one client - // warns. - const CROSS_PROVIDER: &str = r#" -schema_version = 1 - -[llm_clients.openai] -format = "openai_chat" -base_url = "https://example.test/v1" - -[llm_clients.azure] -format = "openai_chat" -base_url = "https://azure.test/v1" - -[targets.openai] -id = "gpt-4o" -llm_client = "openai" - -[targets.azure] -id = "gpt-4o" -llm_client = "azure" - -[routes.openai] -id = "switchyard/openai-gpt4o" -type = "passthrough" -target = "openai" - -[routes.azure] -id = "switchyard/azure-gpt4o" -type = "passthrough" -target = "azure" -"#; - server_state_from_toml(CROSS_PROVIDER)?; - Ok(()) - } - - #[test] - fn accepts_relative_weights_and_seed() -> ServerResult<()> { - let weighted = VALID_CONFIG.replace( - "targets = [\"strong\", \"weak\"]", - "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42", - ); - server_state_from_toml(&weighted)?; - Ok(()) - } - - #[test] - fn accepts_new_session_trigger_with_message_hash_fallback() -> ServerResult<()> { - let configured = VALID_CONFIG.replace( - "base_threshold = 0.5", - "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true", - ); - server_state_from_toml(&configured)?; - Ok(()) - } - - #[test] - fn target_extra_body_is_parsed_and_applied_to_its_backend() -> ServerResult<()> { - let configured = VALID_CONFIG.replacen( - "llm_client = \"primary\"", - "llm_client = \"primary\"\n\ - extra_body = { service_tier = \"priority\", \ - chat_template_kwargs = { enable_thinking = false } }", - 1, - ); - let config: ServerConfig = toml::from_str(&configured) - .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; - let Some(target) = config.targets.get("classifier") else { - return Err(ServerError::new("classifier target is missing")); - }; - let Some(client) = config.llm_clients.get("primary") else { - return Err(ServerError::new("primary llm client is missing")); - }; - let backend = build_backend("primary", client, &target.extra_body)?; - - assert_eq!( - backend.extra_body().get("service_tier"), - Some(&json!("priority")) - ); - assert_eq!( - backend - .extra_body() - .get("chat_template_kwargs") - .and_then(|value| value.get("enable_thinking")), - Some(&json!(false)) - ); - Ok(()) - } - - #[test] - fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> { - let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| { - ServerError::new(format!("failed to parse default config: {error}")) - })?; - let Some(primary) = default.llm_clients.get("primary") else { - return Err(ServerError::new("primary llm client is missing")); - }; - assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES); - - let explicit = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\nmax_retries = 0", - 1, - ); - let config: ServerConfig = toml::from_str(&explicit).map_err(|error| { - ServerError::new(format!("failed to parse explicit retry config: {error}")) - })?; - let Some(primary) = config.llm_clients.get("primary") else { - return Err(ServerError::new("primary llm client is missing")); - }; - assert_eq!(primary.max_retries, 0); - - let maximum = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - &format!( - "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}" - ), - 1, - ); - let config: ServerConfig = toml::from_str(&maximum).map_err(|error| { - ServerError::new(format!("failed to parse maximum retry config: {error}")) - })?; - let Some(primary) = config.llm_clients.get("primary") else { - return Err(ServerError::new("primary llm client is missing")); - }; - assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES); - Ok(()) - } - - #[test] - fn rejects_headers_that_switchyard_sets() { - let cases = [ - ( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\n\ - extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }", - "AUTHORIZATION", - ), - ( - "base_url = \"https://example.test\"", - "base_url = \"https://example.test\"\n\ - extra_headers = { \"X-Api-Key\" = \"custom-key\" }", - "X-Api-Key", - ), - ( - "base_url = \"https://example.test\"", - "base_url = \"https://example.test\"\n\ - extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }", - "ANTHROPIC-VERSION", - ), - ]; - - for (original, replacement, header) in cases { - let configured = VALID_CONFIG.replacen(original, replacement, 1); - let error = error_message(&configured); - assert!( - error.contains(&format!("extra_headers cannot set {header:?}")), - "expected {header} to be rejected, got: {error}" - ); - } - } - - #[test] - fn accepts_additional_headers() -> ServerResult<()> { - let configured = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\n\ - extra_headers = { X-Inference-Priority = \"batch\" }", - 1, - ); - - server_state_from_toml(&configured)?; - Ok(()) - } - - #[test] - fn retry_budget_rejects_negative_values() { - let invalid = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\nmax_retries = -1", - 1, - ); - assert!(error_message(&invalid).contains("max_retries")); - } - - #[test] - fn retry_budget_rejects_excessive_values() { - let invalid = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\nmax_retries = 11", - 1, - ); - assert!( - error_message(&invalid).contains("llm client primary max_retries must be at most 10") - ); - } - - #[test] - fn api_key_environment_reference_is_validated() { - let missing = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"", - 1, - ); - assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET")); - - const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY"; - unsafe { - // "unsafe" is for concurrent reads and writes, very rare - std::env::set_var(EMPTY_KEY_ENV, ""); - } - let empty = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""), - 1, - ); - let message = error_message(&empty); - unsafe { - std::env::remove_var(EMPTY_KEY_ENV); - } - assert!(message.contains("is empty")); - } - - #[test] - fn forward_auth_rejects_conflicting_credentials() { - let competing_auth = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - "base_url = \"https://example.test/v1\"\n\ - forward_auth = true\n\ - api_key_env = \"UNUSED_TEST_KEY\"", - 1, - ); - assert!( - error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env") - ); - - let static_auth = VALID_CONFIG.replacen( - "base_url = \"https://example.test\"", - "base_url = \"https://example.test\"\n\ - forward_auth = true\n\ - extra_headers = { Authorization = \"static-value\" }", - 1, - ); - assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\"")); - - let static_beta = static_auth.replace("Authorization", "anthropic-beta"); - assert!( - error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"") - ); - - for header in ["chatgpt-account-id", "x-openai-fedramp"] { - let static_context = VALID_CONFIG.replacen( - "base_url = \"https://example.test/v1\"", - &format!( - "base_url = \"https://example.test/v1\"\n\ - forward_auth = true\n\ - extra_headers = {{ \"{header}\" = \"static-value\" }}" - ), - 1, - ); - assert!( - error_message(&static_context) - .contains(&format!("extra_headers cannot set \"{header}\"")) - ); - } - } - - const ADVISOR_CONFIG: &str = r#" -schema_version = 1 - -[llm_clients.anthropic] -format = "anthropic_messages" -base_url = "https://example.test" - -[targets.executor] -id = "executor/model" -llm_client = "anthropic" - -[targets.advisor] -id = "advisor/model" -llm_client = "anthropic" - -[routes.gated] -id = "switchyard/advisor" -type = "advisor" -executor_target = "executor" -advisor_target = "advisor" -"#; - - #[test] - fn advisor_route_parses_with_defaults_and_builds() -> ServerResult<()> { - let state = server_state_from_toml(ADVISOR_CONFIG)?; - assert_eq!(state.models().collect::>(), ["switchyard/advisor"]); - Ok(()) - } - - #[test] - fn advisor_route_accepts_every_gate_knob() -> ServerResult<()> { - let tuned = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - concat!( - "advisor_target = \"advisor\"\n", - "reviewer_system_prompt = \"review it\"\n", - "redo_feedback_prefix = \"REVIEWER SAYS: \"\n", - "gate_trigger = \"pattern\"\n", - "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n", - "max_reviews = 2\n", - "gate_stall_turns = 40\n", - "gate_min_tool_results = 1\n", - "advisor_max_tokens = 1024\n", - "advisor_temperature = 0.0\n", - "transcript_max_chars = 100000\n", - "fail_open = false\n", - "context_window = 200000\n", - "tool_calling = true\n", - "reasoning = true", - ), - ); - server_state_from_toml(&tuned)?; - Ok(()) - } - - #[test] - fn advisor_route_rejects_unknown_keys() { - let invalid = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - "advisor_target = \"advisor\"\nbogus_field = 1", - ); - assert!(error_message(&invalid).contains("bogus_field")); - } - - #[test] - fn advisor_route_requires_both_targets() { - let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", ""); - assert!(error_message(&missing).contains("advisor_target")); - } - - #[test] - fn advisor_route_rejects_unknown_target() { - let invalid = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - "advisor_target = \"missing\"", - ); - assert!(error_message(&invalid).contains("missing")); - } - - #[test] - fn advisor_route_rejects_invalid_pattern() { - let invalid = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"", - ); - assert!(error_message(&invalid).contains("not a valid regex")); - } - - #[test] - fn advisor_route_rejects_pattern_without_pattern_trigger() { - let invalid = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"", - ); - assert!( - error_message(&invalid) - .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"") - ); - } - - #[test] - fn advisor_route_pattern_trigger_requires_pattern() { - let invalid = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"", - ); - assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern")); - } - - #[test] - fn advisor_route_rejects_zero_max_reviews() { - let invalid = ADVISOR_CONFIG.replace( - "advisor_target = \"advisor\"", - "advisor_target = \"advisor\"\nmax_reviews = 0", - ); - assert!(error_message(&invalid).contains("max_reviews must be at least 1")); - } + ServerState::from_runner(Runner::load(path).map_err(ServerError::from)?) } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 80f75b7fe..914fb6c8a 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -32,19 +32,21 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; -use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive}; +use libsy::{Algorithm, LibsyError, RoutingOutcome}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient}; +use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver}; use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage}; +use switchyard_runner::{ + CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError, +}; use tokio::net::{TcpListener, TcpSocket}; use tokio::task; use tracing::{Instrument, Level}; use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response}; -use crate::config::ServerConfig; use crate::response::into_http_response; use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env}; @@ -86,106 +88,51 @@ impl Display for ServerError { impl Error for ServerError {} +impl From for ServerError { + fn from(error: RunnerError) -> Self { + let mut message = error.to_string(); + let mut source = error.source(); + while let Some(error) = source { + message.push_str(": "); + message.push_str(&error.to_string()); + source = error.source(); + } + Self::new(message) + } +} + /// Result returned by server setup and lifecycle operations. pub type ServerResult = std::result::Result; -/// Capabilities that one route advertises on `GET /v1/models`. -/// -/// An unset capability is undeclared: it serializes as `null` in the OpenAI -/// `data` entry, and the Codex entry falls back to a safe default for it. -#[derive(Clone, Copy, Default)] -struct ModelCapabilities { - context_window: Option, - tool_calling: Option, - // Whether the routed model takes reasoning controls. The server cannot probe - // this, so a route opts in via config; undeclared routes advertise as - // non-reasoning to Codex (fail closed). - reasoning: Option, -} - -/// A registered algorithm route and its server-owned endpoint metadata. -struct RouteEntry { - algorithm: Arc, - /// Resolves each offloaded call to the client configured for the target the algorithm - /// selected. A route is a synthetic model with no upstream of its own, so this is a - /// per-target lookup, never one client serving the whole route. - target_clients: ClientRouter, - caller_auth: Option, - capabilities: ModelCapabilities, - count_tokens_target: Option, - config_name: Option, -} - /// Decision result using the same selected/fallback order as libsy. #[derive(Serialize)] -struct DecisionResponse<'a> { - selected: DecisionTargetResponse<'a>, - fallbacks: Vec>, +struct DecisionResponse { + selected: DecisionTargetResponse, + fallbacks: Vec, #[serde(skip_serializing_if = "Option::is_none")] response: Option, } -/// Borrowed response view over one target's existing configuration. +/// Response view over one target's existing configuration. #[derive(Serialize)] -struct DecisionTargetResponse<'a> { - target: &'a str, - model: &'a ModelId, - llm_client: DecisionLlmClientResponse<'a>, - extra_body: &'a BTreeMap, +struct DecisionTargetResponse { + target: String, + model: ModelId, + llm_client: DecisionLlmClientResponse, + extra_body: BTreeMap, } /// Non-secret client settings needed to call a selected model. #[derive(Serialize)] -struct DecisionLlmClientResponse<'a> { +struct DecisionLlmClientResponse { format: WireFormat, - base_url: &'a str, -} - -/// Caller credential family required by forwarded-auth backends. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum CallerAuthKind { - Anthropic, - OpenAi, -} - -impl CallerAuthKind { - const fn as_str(self) -> &'static str { - match self { - Self::Anthropic => "anthropic", - Self::OpenAi => "openai", - } - } - - const fn accepts(self, wire_format: WireFormat) -> bool { - matches!( - (self, wire_format), - (Self::Anthropic, WireFormat::AnthropicMessages) - | ( - Self::OpenAi, - WireFormat::OpenAiChat | WireFormat::OpenAiResponses - ) - ) - } -} - -/// Exact upstream model used by the server's Anthropic token-count endpoint. -#[derive(Clone)] -struct CountTokensTarget { - model: ModelId, - client: Arc, -} - -impl CountTokensTarget { - async fn count_tokens(&self, request: Request) -> Result { - self.client.count_tokens(&self.model, request).await - } + base_url: String, } /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { - routes: Arc>, - config: Option>, + runner: Arc, metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, @@ -227,75 +174,37 @@ impl SharedRoutingLog { } impl ServerState { - /// Creates server state from route model IDs, their libsy algorithms, and the - /// per-target client routing each route's calls are resolved through. - pub fn new( - routes: impl IntoIterator, ClientRouter)>, - ) -> ServerResult { - Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| { - ( - model, - algorithm, - clients, - None, - ModelCapabilities::default(), - None, - None, - ) - })) + /// Creates server state from route model IDs, algorithms, and per-target clients. + pub fn new(routes: Vec<(ModelId, Arc, ClientRouter)>) -> ServerResult { + let routes = routes + .into_iter() + .map(|(model, algorithm, clients)| { + ( + model, + Route::new( + algorithm, + clients, + None, + ModelCapabilities::default(), + None, + Vec::new(), + ), + ) + }) + .collect(); + let runner = Runner::new(routes); + Self::from_runner(runner) } - fn new_with_capabilities( - routes: impl IntoIterator< - Item = ( - ModelId, - Arc, - ClientRouter, - Option, - ModelCapabilities, - Option, - Option, - ), - >, - ) -> ServerResult { - let mut entries = BTreeMap::new(); - for ( - model, - algorithm, - target_clients, - caller_auth, - capabilities, - count_tokens_target, - config_name, - ) in routes - { - let model = ModelId::from(model.trim()); - if model.is_empty() { - return Err(ServerError::new("route model must not be empty")); - } - let entry = RouteEntry { - algorithm, - target_clients, - caller_auth, - capabilities, - count_tokens_target, - config_name, - }; - if entries.insert(model.clone(), entry).is_some() { - return Err(ServerError::new(format!("duplicate route model {model}"))); - } - } - if entries.is_empty() { - return Err(ServerError::new("at least one algorithm route is required")); - } + /// Creates HTTP-server state around an already configured runner. + pub fn from_runner(runner: Runner) -> ServerResult { let metrics = metrics::registry().map_err(ServerError::new)?; let stats = StatsAccumulator::new( metrics.clone(), - entries.values().map(|entry| entry.algorithm.name()), + runner.models().map(|model| model.algorithm), ); Ok(Self { - routes: Arc::new(entries), - config: None, + runner: Arc::new(runner), metrics, stats, routing_log: None, @@ -303,12 +212,6 @@ impl ServerState { }) } - /// Retains the validated configuration used to describe decision results. - fn with_config(mut self, config: Arc) -> Self { - self.config = Some(config); - self - } - /// Enables durable per-request routing records at `path`. pub fn with_routing_log(mut self, path: impl Into) -> ServerResult { self.routing_log = Some(SharedRoutingLog::new(path.into())?); @@ -317,57 +220,44 @@ impl ServerState { /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { - self.routes.keys().map(ModelId::as_str) + self.runner.models().map(|model| model.id.as_str()) } /// Returns the caller credential family used by `model`, if any. - pub fn caller_auth_kind(&self, model: &str) -> ServerResult> { - self.route_for_model(model) - .map(|entry| entry.caller_auth.map(CallerAuthKind::as_str)) - .ok_or_else(|| ServerError::new(format!("unknown route model {model:?}"))) + pub fn caller_auth_kind(&self, model: &str) -> Option<&'static str> { + self.runner + .route(model) + .and_then(|route| route.caller_auth()) + .map(|kind| kind.as_str()) } - fn route_for_model(&self, model: &str) -> Option<&RouteEntry> { - self.routes.get(model) + fn route_for_model(&self, model: &str) -> Option<&Route> { + self.runner.route(model) } - /// Resolves the routing outcome through the target names configured for its route. - fn decision_response<'a>( - &'a self, - route: &'a RouteEntry, + fn decision_response( + &self, + route_model: &ModelId, outcome: &RoutingOutcome, response: Option, - ) -> Option> { - let config = self.config.as_ref()?; - let target_names = config.routing_target_names(route.config_name.as_ref()?)?; - let resolve = |model: &ModelId| { - let (target_name, target) = target_names - .iter() - .filter_map(|name| config.targets.get_key_value(*name)) - .find(|(_, target)| target.id == *model)?; - let client = config.llm_clients.get(&target.llm_client)?; - Some(DecisionTargetResponse { - target: target_name, - model: &target.id, - llm_client: DecisionLlmClientResponse { - format: client.format.wire_format(), - base_url: client.base_url.as_str(), - }, - extra_body: &target.extra_body, - }) + ) -> Option { + let description = self.runner.describe_decision(route_model, outcome)?; + let convert = |target: DecisionTarget| DecisionTargetResponse { + target: target.target, + model: target.model, + llm_client: DecisionLlmClientResponse { + format: target.format, + base_url: target.base_url, + }, + extra_body: target.extra_body, }; Some(DecisionResponse { - selected: resolve(&outcome.selected_model_id)?, - fallbacks: outcome - .fallback_models - .iter() - .map(resolve) - .collect::>>()?, + selected: convert(description.selected), + fallbacks: description.fallbacks.into_iter().map(convert).collect(), response, }) } } - /// Runtime options shared by server entry points. #[derive(Clone, Debug)] pub struct ServerRunOptions { @@ -682,9 +572,15 @@ async fn decision( Ok(resolved) => resolved, Err(response) => return response, }; - let mut outcome = match run_decision_only(route, request).await { + let route_model = request + .llm_request + .model + .as_deref() + .map(ModelId::from) + .unwrap_or_default(); + let mut outcome = match route.decide(request).await { Ok(outcome) => outcome, - Err(error) => return algorithm_error(error), + Err(error) => return runner_error(error), }; let response = match outcome.response.take() { Some(response) => { @@ -706,7 +602,7 @@ async fn decision( } None => None, }; - match state.decision_response(route, &outcome, response) { + match state.decision_response(&route_model, &outcome, response) { Some(response) => Json(response).into_response(), None => { server_error("routing outcome contains a model with no callable target configuration") @@ -714,27 +610,6 @@ async fn decision( } } -/// Completes routing-time calls and returns the outcome without serving its answer target. -async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result { - drive(Arc::clone(&route.algorithm), request, |call| { - serve_decision_dependency(route, call) - }) - .await -} - -/// Serves a classifier or judge call that the decision depends on. -async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> { - let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?; - let result = match route.target_clients.route(&model) { - Ok(client) => client - .call(call.request.clone()) - .await - .map_err(|source| LibsyError::client_call(model, source)), - Err(source) => Err(LibsyError::client_call(model, source)), - }; - call.respond(result) -} - /// Anthropic token counting against the route's explicitly configured target. async fn anthropic_count_tokens( State(state): State, @@ -756,17 +631,16 @@ async fn anthropic_count_tokens( Ok(resolved) => resolved, Err(response) => return anthropic_error_response(response), }; - let Some(target) = route.count_tokens_target.as_ref() else { - return anthropic_error_response(error_response( + anthropic_error_response(match route.count_tokens(request).await { + Ok(payload) => (StatusCode::OK, Json(payload)).into_response(), + Err(RunnerError::CountTokensUnsupported) => error_response( StatusCode::BAD_REQUEST, "route has no Anthropic target for token counting", "invalid_request_error", "count_tokens_unsupported", - )); - }; - anthropic_error_response(match target.count_tokens(request).await { - Ok(payload) => (StatusCode::OK, Json(payload)).into_response(), - Err(error) => count_tokens_error(error), + ), + Err(RunnerError::Client(error)) => count_tokens_error(error), + Err(error) => server_error(error.to_string()), }) } @@ -867,7 +741,7 @@ fn resolve_route( metadata: Metadata, body: Value, wire_format: WireFormat, -) -> std::result::Result<(&RouteEntry, Request), Response> { +) -> std::result::Result<(&Route, Request), Response> { let llm_request = decode_request(wire_format, &body) .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?; let requested_model = llm_request @@ -890,8 +764,8 @@ fn resolve_route( "model_not_found", ) })?; - if let Some(caller_auth) = route.caller_auth - && !caller_auth.accepts(wire_format) + if let Err(RunnerError::IncompatibleCallerFormat(caller_auth)) = + route.check_caller_format(wire_format) { let (provider, expected_endpoint) = match caller_auth { CallerAuthKind::Anthropic => ("Anthropic", "/v1/messages"), @@ -929,17 +803,18 @@ async fn handle_llm_request( }; // Only the Codex namespace mapping is needed downstream, not the whole request. let request_extensions = request.llm_request.extensions.clone(); - let algorithm = Arc::clone(&route.algorithm); - let client_router = route.target_clients.clone(); let observer = stats_observer( state.stats.clone(), state.routing_log.clone().zip(routing_log_context.clone()), ); - let (selected_model, response) = - match switchyard_llm_client::run(algorithm, client_router, request, Some(observer)).await { - Ok(result) => result, - Err(error) => return algorithm_error(error), - }; + let output = match route.execute(request, Some(observer)).await { + Ok(output) => output, + Err(error) => return runner_error(error), + }; + let RunOutput { + selected_model, + response, + } = output; // The response carries the candidate that actually served it. Fall back to the routing // selection for algorithms that return a response without an offloaded model call. let served_model = response.served_model().cloned().or(Some(selected_model)); @@ -1071,6 +946,14 @@ fn algorithm_error(error: LibsyError) -> Response { client_error(source) } +fn runner_error(error: RunnerError) -> Response { + match error { + RunnerError::Algorithm(error) => algorithm_error(error), + RunnerError::Client(error) => client_error(&error), + error => server_error(error.to_string()), + } +} + fn client_error(error: &LlmClientError) -> Response { match error { LlmClientError::InvalidRequest { message } @@ -1235,9 +1118,9 @@ fn error_response( async fn models(State(state): State) -> Json { Json(model_list_payload( state - .routes - .iter() - .map(|(model, entry)| (model.as_str(), entry.capabilities)), + .runner + .models() + .map(|model| (model.id.as_str(), model.capabilities)), )) } @@ -1321,7 +1204,8 @@ async fn not_found() -> Response { fn model_list_payload<'a>( entries: impl IntoIterator, ) -> Value { - let entries = entries.into_iter().collect::>(); + let mut entries = entries.into_iter().collect::>(); + entries.sort_unstable_by_key(|(model_id, _)| *model_id); let model_ids = entries.iter().map(|(model, _)| *model).collect::>(); let first_id = model_ids.first().copied(); let last_id = model_ids.last().copied();