feat: Wire VFS cloud storage integration for S3, GCS, Azure support

- Add StorageConfig and StorageCacheConfig structs for storage cache settings
- Implement parseStorageConfig() to parse storage.cache section from YAML
- Wire CachingFileProvider into getFileProvider() for remote template paths
- Initialize cloud credentials from environment at startup
- Configure credentials in DuckDB after database initialization
- Add VFSHealthChecker startup verification for storage backends

Users can now configure remote file caching in flapi.yaml:
  storage:
    cache:
      enabled: true
      ttl: 300          # seconds
      max_size_mb: 50   # megabytes

Author: jrosskopf

src/config_manager.cpp

@@ -4,6 +4,8 @@
 #include "endpoint_repository.hpp"
 #include "config_validator.hpp"
 #include "config_serializer.hpp"
+#include "caching_file_provider.hpp"
+#include "vfs_adapter.hpp"
 #include <stdexcept>
 #include <filesystem>
 #include <yaml-cpp/yaml.h>
@@ -107,6 +109,7 @@ void ConfigManager::parseMainConfig() {
         parseDuckDBConfig();
         parseDuckLakeConfig();
         parseMCPConfig();
+        parseStorageConfig();
         parseTemplateConfig();
         parseGlobalHeartbeatConfig();
 
@@ -231,6 +234,38 @@ void ConfigManager::parseDuckLakeConfig() {
     }
 }
 
+// Storage configuration methods
+void ConfigManager::parseStorageConfig() {
+    CROW_LOG_INFO << "Parsing storage configuration";
+    storage_config = StorageConfig{};  // Reset to defaults
+
+    if (!config["storage"]) {
+        CROW_LOG_DEBUG << "Storage configuration not found, using defaults (cache.enabled=true, cache.ttl=300s, cache.max_size_mb=50)";
+        return;
+    }
+
+    auto storage_node = config["storage"];
+
+    if (storage_node["cache"]) {
+        auto cache_node = storage_node["cache"];
+        storage_config.cache.enabled = safeGet<bool>(cache_node, "enabled", "storage.cache.enabled", true);
+
+        if (cache_node["ttl"]) {
+            int ttl_seconds = safeGet<int>(cache_node, "ttl", "storage.cache.ttl", 300);
+            storage_config.cache.ttl = std::chrono::seconds(ttl_seconds);
+            CROW_LOG_DEBUG << "Storage cache TTL: " << ttl_seconds << " seconds";
+        }
+
+        if (cache_node["max_size_mb"]) {
+            int max_mb = safeGet<int>(cache_node, "max_size_mb", "storage.cache.max_size_mb", 50);
+            storage_config.cache.max_size_bytes = static_cast<size_t>(max_mb) * 1024UL * 1024UL;
+            CROW_LOG_DEBUG << "Storage cache max size: " << max_mb << " MB";
+        }
+
+        CROW_LOG_DEBUG << "Storage cache enabled: " << (storage_config.cache.enabled ? "true" : "false");
+    }
+}
+
 // MCP configuration methods
 void ConfigManager::parseMCPConfig() {
     CROW_LOG_INFO << "Parsing MCP configuration";
@@ -1035,7 +1070,20 @@ int ConfigManager::getHttpPort() const { return http_port; }
 void ConfigManager::setHttpPort(int port) { http_port = port; }
 std::string ConfigManager::getTemplatePath() const { return template_config.path; }
 std::filesystem::path ConfigManager::getFullTemplatePath() const { return std::filesystem::path(base_path) / template_config.path; }
-std::shared_ptr<IFileProvider> ConfigManager::getFileProvider() const { return config_loader->getFileProvider(); }
+std::shared_ptr<IFileProvider> ConfigManager::getFileProvider() const {
+    // For remote template paths with caching enabled, wrap with CachingFileProvider
+    if (storage_config.cache.enabled && PathSchemeUtils::IsRemotePath(template_config.path)) {
+        FileCacheConfig cache_config;
+        cache_config.enabled = true;
+        cache_config.ttl = storage_config.cache.ttl;
+        cache_config.max_size_bytes = storage_config.cache.max_size_bytes;
+
+        auto base_provider = FileProviderFactory::CreateDuckDBProvider();
+        return std::make_shared<CachingFileProvider>(base_provider, cache_config);
+    }
+
+    return config_loader->getFileProvider();
+}
 std::string ConfigManager::getCacheSchema() const { return cache_schema; }
 const std::unordered_map<std::string, ConnectionConfig>& ConfigManager::getConnections() const { return connections; }
 const RateLimitConfig& ConfigManager::getRateLimitConfig() const { return rate_limit_config; }

src/include/config_manager.hpp

@@ -442,6 +442,16 @@ struct DuckLakeConfig {
     std::optional<std::size_t> data_inlining_row_limit;
 };
 
+struct StorageCacheConfig {
+    bool enabled = true;
+    std::chrono::seconds ttl{300};          // 5 minutes default
+    size_t max_size_bytes = 50UL * 1024UL * 1024UL;  // 50 MB default
+};
+
+struct StorageConfig {
+    StorageCacheConfig cache;
+};
+
 // Forward declarations
 class EndpointConfigParser;
 class ConfigLoader;
@@ -501,6 +511,7 @@ class ConfigManager {
     const GlobalHeartbeatConfig& getGlobalHeartbeatConfig() const { return global_heartbeat_config; }
     const DuckLakeConfig& getDuckLakeConfig() const { return ducklake_config; }
     const MCPConfig& getMCPConfig() const { return mcp_config; }
+    const StorageConfig& getStorageConfig() const { return storage_config; }
 
     // Load MCP server instructions (inline or from file)
     std::string loadMCPInstructions() const;
@@ -559,6 +570,7 @@ class ConfigManager {
     GlobalHeartbeatConfig global_heartbeat_config;
     DuckLakeConfig ducklake_config;
     MCPConfig mcp_config;
+    StorageConfig storage_config;
     ExtendedYamlParser yaml_parser;
 
     // Extracted classes for delegation (Facade pattern)
@@ -580,6 +592,7 @@ class ConfigManager {
     void parseTemplateConfig();
     void parseDuckLakeConfig();
     void parseMCPConfig();
+    void parseStorageConfig();
     void parseEndpointConfig(const std::filesystem::path& config_file);
     void parseEndpointRequestFields(const YAML::Node& endpoint_config, EndpointConfig& endpoint);
     void parseEndpointValidators(const YAML::Node& req, RequestFieldConfig& field);

src/main.cpp

@@ -22,6 +22,8 @@
 #include "database_manager.hpp"
 #include "rate_limit_middleware.hpp"
 #include "config_token_utils.hpp"
+#include "credential_manager.hpp"
+#include "vfs_health_checker.hpp"
 
 using namespace flapi;
 
@@ -137,6 +139,32 @@ void initializeDatabase(std::shared_ptr<ConfigManager> config_manager) {
     }
 }
 
+void initializeCloudCredentials() {
+    CROW_LOG_INFO << "Initializing cloud storage credentials...";
+    auto& cred_manager = flapi::getGlobalCredentialManager();
+    cred_manager.loadFromEnvironment();
+    cred_manager.logCredentialStatus();
+}
+
+void configureCloudCredentialsInDuckDB() {
+    auto& cred_manager = flapi::getGlobalCredentialManager();
+    if (cred_manager.hasS3Credentials() || cred_manager.hasGCSCredentials() || cred_manager.hasAzureCredentials()) {
+        CROW_LOG_INFO << "Configuring cloud credentials in DuckDB...";
+        if (cred_manager.configureDuckDB()) {
+            CROW_LOG_INFO << "Cloud credentials configured successfully";
+        } else {
+            CROW_LOG_WARNING << "Failed to configure some cloud credentials in DuckDB";
+        }
+    }
+}
+
+void verifyStorageHealth(std::shared_ptr<ConfigManager> config_manager) {
+    flapi::VFSHealthChecker health_checker;
+    std::string config_path = config_manager->getBasePath();
+    std::string templates_path = config_manager->getTemplatePath();
+    health_checker.verifyStartupHealth(config_path, templates_path);
+}
+
 void terminateHandler() {
     CROW_LOG_ERROR << "Unhandled exception caught! flapi is giving up :-(";
 
@@ -282,12 +310,21 @@ int main(int argc, char* argv[])
         return validateConfiguration(config_manager, config_file);
     }
 
+    // Initialize cloud storage credentials (reads environment variables)
+    initializeCloudCredentials();
+
     if (cmd_port != -1) {
         config_manager->setHttpPort(cmd_port);
     }
 
     initializeDatabase(config_manager);
 
+    // Configure cloud credentials in DuckDB after database is initialized
+    configureCloudCredentialsInDuckDB();
+
+    // Verify storage health at startup
+    verifyStorageHealth(config_manager);
+
     // Create unified API server with MCP support (always enabled in unified configuration)
     api_server = std::make_shared<APIServer>(
         config_manager,