feat: add RPC failover support with per-request error recovery

crates/cli/src/args.rs

@@ -4,11 +4,105 @@ use clap::Parser;
 #[derive(Debug, Parser)]
 #[command(name = "kora")]
 pub struct GlobalArgs {
-    /// Solana RPC endpoint URL
+    /// Solana RPC endpoint URLs (comma-separated, or use RPC_URLS environment variable)
+    /// Falls back to RPC_URL environment variable if RPC_URLS is not set.
+    /// At least one endpoint is required.
+    #[arg(long, env = "RPC_URLS")]
+    pub rpc_urls: Option<String>,
+
+    /// Solana RPC endpoint URL (deprecated: use --rpc-urls instead)
     #[arg(long, env = "RPC_URL", default_value = "http://127.0.0.1:8899")]
     pub rpc_url: String,
 
     /// Path to Kora configuration file (TOML format)
     #[arg(long, default_value = "kora.toml")]
     pub config: String,
 }
+
+impl GlobalArgs {
+    /// Get the list of RPC endpoints, prioritizing RPC_URLS over RPC_URL.
+    /// Returns an error if the list is empty.
+    pub fn get_rpc_endpoints(&self) -> Result<Vec<String>, String> {
+        let endpoints_str = self
+            .rpc_urls
+            .as_ref()
+            .map(|s| s.as_str())
+            .unwrap_or(self.rpc_url.as_str());
+
+        let endpoints: Vec<String> =
+            endpoints_str.split(',').map(|s| s.trim().to_string()).collect();
+
+        if endpoints.is_empty() || endpoints.iter().all(|s| s.is_empty()) {
+            return Err("At least one RPC endpoint is required".to_string());
+        }
+
+        Ok(endpoints.into_iter().filter(|s| !s.is_empty()).collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_rpc_endpoints_from_urls() {
+        let args = GlobalArgs {
+            rpc_urls: Some("http://localhost:8899,http://localhost:8900".to_string()),
+            rpc_url: "http://127.0.0.1:8899".to_string(),
+            config: "kora.toml".to_string(),
+        };
+        let endpoints = args.get_rpc_endpoints().unwrap();
+        assert_eq!(endpoints.len(), 2);
+        assert_eq!(endpoints[0], "http://localhost:8899");
+        assert_eq!(endpoints[1], "http://localhost:8900");
+    }
+
+    #[test]
+    fn test_get_rpc_endpoints_fallback_to_url() {
+        let args = GlobalArgs {
+            rpc_urls: None,
+            rpc_url: "http://localhost:8899".to_string(),
+            config: "kora.toml".to_string(),
+        };
+        let endpoints = args.get_rpc_endpoints().unwrap();
+        assert_eq!(endpoints.len(), 1);
+        assert_eq!(endpoints[0], "http://localhost:8899");
+    }
+
+    #[test]
+    fn test_get_rpc_endpoints_urls_takes_priority() {
+        let args = GlobalArgs {
+            rpc_urls: Some("http://localhost:8899,http://localhost:8900".to_string()),
+            rpc_url: "http://localhost:9999".to_string(),
+            config: "kora.toml".to_string(),
+        };
+        let endpoints = args.get_rpc_endpoints().unwrap();
+        assert_eq!(endpoints.len(), 2);
+        assert_eq!(endpoints[0], "http://localhost:8899");
+    }
+
+    #[test]
+    fn test_get_rpc_endpoints_trims_whitespace() {
+        let args = GlobalArgs {
+            rpc_urls: Some("  http://localhost:8899  ,  http://localhost:8900  ".to_string()),
+            rpc_url: "http://127.0.0.1:8899".to_string(),
+            config: "kora.toml".to_string(),
+        };
+        let endpoints = args.get_rpc_endpoints().unwrap();
+        assert_eq!(endpoints.len(), 2);
+        assert_eq!(endpoints[0], "http://localhost:8899");
+        assert_eq!(endpoints[1], "http://localhost:8900");
+    }
+
+    #[test]
+    fn test_get_rpc_endpoints_uses_default_when_urls_none() {
+        let args = GlobalArgs {
+            rpc_urls: None,
+            rpc_url: "http://127.0.0.1:8899".to_string(),
+            config: "kora.toml".to_string(),
+        };
+        let endpoints = args.get_rpc_endpoints().unwrap();
+        assert_eq!(endpoints.len(), 1);
+        assert_eq!(endpoints[0], "http://127.0.0.1:8899");
+    }
+}

crates/cli/src/main.rs

@@ -7,7 +7,7 @@ use kora_lib::{
     admin::token_util::initialize_atas,
     error::KoraError,
     log::LoggingFormat,
-    rpc::get_rpc_client,
+    rpc::{get_failover_rpc_client, RpcClientEnum},
     rpc_server::{run_rpc_server, KoraRpc, RpcArgs},
     signer::init::init_signers,
     state::init_config,
@@ -120,22 +120,32 @@ async fn main() -> Result<(), KoraError> {
         std::process::exit(1);
     });
 
-    let rpc_client = get_rpc_client(&cli.global_args.rpc_url);
+    let endpoints = cli.global_args.get_rpc_endpoints().unwrap_or_else(|e| {
+        print_error(&e);
+        std::process::exit(1);
+    });
+
+    let rpc_client = get_failover_rpc_client(endpoints).unwrap_or_else(|e| {
+        print_error(&e);
+        std::process::exit(1);
+    });
+
+    let rpc_client_arc = rpc_client.get_client();
 
     match cli.command {
         Some(Commands::Config { config_command }) => {
             let validation_result = match config_command {
                 ConfigCommands::Validate { signers_config } => {
                     ConfigValidator::validate_with_result_and_signers(
-                        rpc_client.as_ref(),
+                        rpc_client_arc.as_ref(),
                         true,
                         signers_config.as_ref(),
                     )
                     .await
                 }
                 ConfigCommands::ValidateWithRpc { signers_config } => {
                     ConfigValidator::validate_with_result_and_signers(
-                        rpc_client.as_ref(),
+                        rpc_client_arc.as_ref(),
                         false,
                         signers_config.as_ref(),
                     )
@@ -152,7 +162,7 @@ async fn main() -> Result<(), KoraError> {
                 rpc_args.auth_args.apply_to_env();
 
                 match ConfigValidator::validate_with_result_and_signers(
-                    rpc_client.as_ref(),
+                    rpc_client_arc.as_ref(),
                     true,
                     rpc_args.signers_config.as_ref(),
                 )
@@ -185,9 +195,7 @@ async fn main() -> Result<(), KoraError> {
                     std::process::exit(1);
                 }
 
-                let rpc_client = get_rpc_client(&cli.global_args.rpc_url);
-
-                let kora_rpc = KoraRpc::new(rpc_client);
+                let kora_rpc = KoraRpc::new(rpc_client_arc);
 
                 let handles = run_rpc_server(kora_rpc, rpc_args.port).await?;
 
@@ -219,7 +227,7 @@ async fn main() -> Result<(), KoraError> {
                 }
 
                 if let Err(e) = initialize_atas(
-                    rpc_client.as_ref(),
+                    rpc_client_arc.as_ref(),
                     compute_unit_price,
                     compute_unit_limit,
                     chunk_size,

crates/lib/src/mod.rs

@@ -16,6 +16,8 @@ pub mod metrics;
 pub mod oracle;
 pub mod plugin;
 pub mod rpc;
+pub mod rpc_failover;
+pub mod rpc_failover_wrapper;
 pub mod rpc_server;
 pub mod sanitize;
 pub mod signer;

crates/lib/src/rpc.rs

@@ -3,10 +3,97 @@ use std::{sync::Arc, time::Duration};
 use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_commitment_config::CommitmentConfig;
 
+use crate::rpc_failover::FailoverRpcClient;
+
+/// Create an RPC client from a single endpoint (backward compatibility).
 pub fn get_rpc_client(rpc_url: &str) -> Arc<RpcClient> {
     Arc::new(RpcClient::new_with_timeout_and_commitment(
         rpc_url.to_string(),
         Duration::from_secs(90),
         CommitmentConfig::confirmed(),
     ))
 }
+
+/// Wrapper that holds either a simple RpcClient or a FailoverRpcClient.
+pub enum RpcClientEnum {
+    Simple(Arc<RpcClient>),
+    Failover(Arc<FailoverRpcClient>),
+}
+
+impl RpcClientEnum {
+    /// Get the current primary RPC client.
+    pub fn get_client(&self) -> Arc<RpcClient> {
+        match self {
+            RpcClientEnum::Simple(client) => client.clone(),
+            RpcClientEnum::Failover(failover) => failover.get_client(),
+        }
+    }
+
+    /// Get the failover client if it exists (for retry logic).
+    pub fn as_failover(&self) -> Option<Arc<FailoverRpcClient>> {
+        match self {
+            RpcClientEnum::Simple(_) => None,
+            RpcClientEnum::Failover(failover) => Some(failover.clone()),
+        }
+    }
+}
+
+/// Create a failover RPC client from multiple endpoints.
+///
+/// # Arguments
+/// * `endpoints` - List of RPC endpoint URLs. If empty, returns an error.
+///
+/// # Returns
+/// An RPC client enum that either wraps a single RpcClient or a FailoverRpcClient.
+pub fn get_failover_rpc_client(endpoints: Vec<String>) -> Result<RpcClientEnum, String> {
+    if endpoints.is_empty() {
+        return Err("At least one RPC endpoint is required".to_string());
+    }
+
+    if endpoints.len() == 1 {
+        let client = Arc::new(RpcClient::new_with_timeout_and_commitment(
+            endpoints[0].clone(),
+            Duration::from_secs(90),
+            CommitmentConfig::confirmed(),
+        ));
+        return Ok(RpcClientEnum::Simple(client));
+    }
+
+    // Multiple endpoints: return failover client wrapper
+    let failover = Arc::new(FailoverRpcClient::new(endpoints));
+    Ok(RpcClientEnum::Failover(failover))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_rpc_client() {
+        let client = get_rpc_client("http://localhost:8899");
+        assert!(Arc::strong_count(&client) >= 1);
+    }
+
+    #[test]
+    fn test_get_failover_rpc_client_single() {
+        let result = get_failover_rpc_client(vec!["http://localhost:8899".to_string()]);
+        assert!(result.is_ok());
+        assert!(result.unwrap().as_failover().is_none());
+    }
+
+    #[test]
+    fn test_get_failover_rpc_client_multiple() {
+        let result = get_failover_rpc_client(vec![
+            "http://localhost:8899".to_string(),
+            "http://localhost:8900".to_string(),
+        ]);
+        assert!(result.is_ok());
+        assert!(result.unwrap().as_failover().is_some());
+    }
+
+    #[test]
+    fn test_get_failover_rpc_client_empty() {
+        let result = get_failover_rpc_client(vec![]);
+        assert!(result.is_err());
+    }
+}