Merge pull request #6247 from makr-code/copilot/deep-dive-raid-sharding

[EPIC] Sharding runtime closes latency, rebalance, and GSI roadmap gaps

Author: makr-code

.github/workflows/reusable-cmake-build.yml

@@ -141,6 +141,11 @@ jobs:
           THEMIS_BUILD_SIG: ${{ secrets.THEMIS_BUILD_SIG }}
         run: |
           cd "$WORKING_DIRECTORY"
+          # Clean stale CMake cache entries before reconfiguring so a previous
+          # toolchain/bootstrap state cannot poison a later runner build.
+          find . -maxdepth 2 \( -name CMakeCache.txt -o -name CMakeFiles \) -print
+          find . -maxdepth 2 -name CMakeCache.txt -delete
+          find . -maxdepth 2 -type d -name CMakeFiles -prune -exec rm -rf {} +
           script_path="$RUNNER_TEMP/cmake-build-configure.sh"
           printf '%s\n' "$CONFIGURE_COMMAND" > "$script_path"
           bash "$script_path"

cmake/CMakeLists.txt

@@ -2854,6 +2854,7 @@ set(THEMIS_CORE_SOURCES
     ../src/sharding/distributed_coordinator.cpp
     ../src/sharding/shard_resource_manager.cpp
     ../src/sharding/locality_aware_router.cpp
+    ../src/sharding/global_secondary_index.cpp
     ../src/sharding/raft_configuration.cpp
     ../src/sharding/raft_log.cpp
     ../src/sharding/raft_state.cpp

cmake/Dependencies.cmake

@@ -9,7 +9,12 @@ else()
     set(_the_vcpkg_root "${CMAKE_SOURCE_DIR}/vcpkg")
 endif()
 
-if(EXISTS "${_the_vcpkg_root}")
+# Respect the system-package build mode: the community/system presets intentionally
+# leave CMAKE_TOOLCHAIN_FILE empty to avoid network-vcpkg bootstrapping. A local
+# checkout must not silently override that explicit opt-out.
+if(DEFINED CMAKE_TOOLCHAIN_FILE AND "${CMAKE_TOOLCHAIN_FILE}" STREQUAL "")
+    message(STATUS "CMAKE_TOOLCHAIN_FILE is empty; skipping automatic vcpkg activation (system package mode)")
+elseif(EXISTS "${_the_vcpkg_root}")
     set(CMAKE_TOOLCHAIN_FILE "${_the_vcpkg_root}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "Vcpkg toolchain file")
     set(_vcpkg_prefix_roots)
     if(WIN32)

include/cache/grpc_remote_cache_peer.h

@@ -80,10 +80,12 @@ class GrpcRemoteCachePeer final : public IRemoteCachePeer {
         /// RPC deadline in milliseconds (default: 1 000 ms).
         int rpc_timeout_ms = 1000;
 
-        /// TLS: set to true and provide CA cert path to use SSL credentials.
-        /// When false, insecure channel credentials are used (not recommended
-        /// for production).
+        /// TLS is required for production.  Insecure transport is only allowed
+        /// when a human explicitly opts into a local/test override.
         bool        tls_enabled   = false;
+        /// Explicit local/dev-only insecure override.  Must remain false in
+        /// production and is rejected by default.
+        bool        allow_insecure = false;
         std::string tls_ca_cert;  ///< PEM-encoded CA certificate (in-memory)
 
         Config() = default;
@@ -101,9 +103,11 @@ class GrpcRemoteCachePeer final : public IRemoteCachePeer {
     explicit GrpcRemoteCachePeer(Config config);
 
     /**
-     * @brief Convenience constructor: insecure peer at the given address.
+     * @brief Convenience constructor for a configured peer address.
      *
-     * Equivalent to GrpcRemoteCachePeer(Config(addr)).
+     * This does not opt into insecure transport; production callers must set
+     * `tls_enabled=true` or provide an explicit local/test override via
+     * `Config::allow_insecure`.
      */
     explicit GrpcRemoteCachePeer(const std::string& addr);
 
@@ -178,6 +182,7 @@ class GrpcRemoteCachePeer final : public IRemoteCachePeer {
         std::string address;
         int rpc_timeout_ms = 1000;
         bool        tls_enabled   = false;
+        bool        allow_insecure = false;
         std::string tls_ca_cert;
 
         Config() = default;

include/security/access_control_manager.h

@@ -62,12 +62,22 @@ struct AccessDecision {
     }
 };
 
+/// Authorization exception policy. The secure default is fail-closed; explicit
+/// fail-open requires a clear justification and is intended only for documented
+/// maintenance / local override scenarios.
+enum class AuthorizationFailureMode {
+    DenyOnError,             // Default secure behavior: deny access on auth failures.
+    AllowOnErrorExplicit     // Explicit fail-open override: requires an override reason.
+};
+
 /// Access control policy configuration
 struct AccessControlConfig {
     std::string rbac_config_path;                 // Path to RBAC configuration
     std::string user_role_store_path;             // Path to user-role mappings
     bool enable_audit_logging = true;             // Enable access control audit logs
-    bool fail_closed = true;                      // Deny access on errors (fail-safe)
+    bool fail_closed = true;                      // Legacy compatibility flag; prefer failure_mode.
+    AuthorizationFailureMode failure_mode = AuthorizationFailureMode::DenyOnError;
+    std::string fail_open_reason;                 // Required when failure_mode is AllowOnErrorExplicit.
     bool enable_resource_wildcards = true;        // Allow wildcards in resources
 
     // ABAC configuration

include/sharding/global_secondary_index.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <chrono>
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace themis::sharding {
+
+class GlobalSecondaryIndexManager {
+public:
+    struct IndexEntry {
+        std::string index_name;
+        std::string field_name;
+        std::string value;
+        std::string shard_id;
+        std::string primary_key;
+        std::chrono::system_clock::time_point updated_at;
+    };
+
+    struct Config {
+        bool asynchronous_updates = true;
+        bool eventual_consistency = true;
+        std::chrono::milliseconds staleness_budget{5000};
+    };
+
+    GlobalSecondaryIndexManager();
+    explicit GlobalSecondaryIndexManager(const Config& config);
+
+    void createIndex(const std::string& index_name, const std::string& field_name);
+    bool hasIndex(const std::string& index_name) const;
+
+    void upsert(const std::string& index_name, const std::string& field_name,
+                const std::string& shard_id, const std::string& primary_key,
+                const std::string& value);
+    void erase(const std::string& index_name, const std::string& shard_id, const std::string& primary_key);
+    void eraseShard(const std::string& index_name, const std::string& shard_id);
+
+    std::vector<IndexEntry> queryEquals(const std::string& index_name, const std::string& value) const;
+    std::vector<IndexEntry> queryRange(const std::string& index_name,
+                                      const std::string& lower_bound,
+                                      const std::string& upper_bound) const;
+
+    size_t size() const;
+
+private:
+    struct IndexDefinition {
+        std::string name;
+        std::string field_name;
+    };
+
+    Config config_;
+    std::map<std::string, IndexDefinition> indexes_;
+    std::map<std::string, std::map<std::string, std::vector<IndexEntry>>> entries_;
+};
+
+}  // namespace themis::sharding

include/sharding/shard_rpc_client.h

@@ -93,6 +93,7 @@ class ShardRPCClient {
 
         // mTLS Configuration (optional, required for production)
         bool enable_mtls = true;        // Enable mutual TLS authentication (default: on)
+        bool allow_insecure = false;    // Explicit local/test-only override; rejected by default
         std::string tls_cert_path;      // Path to client certificate (PEM format)
         std::string tls_key_path;       // Path to client private key (PEM format)
         std::string tls_ca_cert_path;   // Path to CA certificate for server verification (PEM format)

include/transaction/grpc_rpc_adapter.h

@@ -56,10 +56,9 @@ namespace themis::transaction {
 /**
  * @brief Mutual-TLS credential bundle for gRPC channel creation.
  *
- * When all three PEM fields are non-empty the adapters will create a channel
- * backed by `grpc::SslCredentials`; if any field is empty (or the struct is
- * absent) the adapters fall back to `InsecureChannelCredentials()` and emit a
- * `spdlog::warn` so the fallback is always visible in logs.
+ * This path is fail-closed: the adapter will only create a secure gRPC channel
+ * when all three PEM fields are present and valid. Missing material is rejected
+ * unless `allow_insecure` is explicitly set for a local test-only override.
  *
  * @note For production deployments populate from files or a secret manager —
  *       never hard-code PEM material in source code.
@@ -79,6 +78,13 @@ struct MtlsConfig {
      * match the dial address (e.g. `"localhost"` vs `"127.0.0.1"`).
      */
     std::string target_name_override;
+    /**
+     * @brief Allow the explicit development/test-only insecure fallback.
+     *
+     * This must remain false in production. The distributed transaction path
+     * treats any other usage as a hard error to avoid silent trust degradation.
+     */
+    bool allow_insecure = false;
 };
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -109,9 +115,9 @@ class GrpcRpcPhase1Adapter {
      * @param timeout         gRPC deadline applied to every PREPARE call.
      * @param mtls            Optional mTLS credential bundle.  When present and
      *                        all three PEM fields are non-empty, the channel is
-     *                        created with `grpc::SslCredentials`.  Otherwise
-     *                        `InsecureChannelCredentials()` is used and a
-     *                        warning is logged.
+     *                        created with `grpc::SslCredentials`.  Missing PEM
+     *                        material is rejected unless `allow_insecure` is
+     *                        explicitly set for a local test override.
      * @return                Callable compatible with
      *                        `DistributedTransactionManager::RpcPhase1Fn`.
      */

include/utils/grpc_channel_pool.h

@@ -61,6 +61,7 @@ class GrpcChannelPool {
         std::chrono::seconds keepalive_time{30};      ///< Keepalive time
         std::chrono::seconds keepalive_timeout{10};   ///< Keepalive timeout
         int max_concurrent_streams = 100;             ///< Max concurrent streams per channel
+        bool allow_insecure = false;                  ///< Explicit local/test-only fallback; false by default
     };
 
     GrpcChannelPool();
@@ -86,7 +87,8 @@ class GrpcChannelPool {
      * - Caller should implement retry logic or fallback strategy
      * 
      * @param target Target address (e.g., "localhost:50051")
-     * @param credentials Channel credentials (optional, uses insecure if nullptr)
+     * @param credentials Channel credentials. A null pointer is rejected by default;
+     *                   set Config::allow_insecure=true for an explicit local/test override.
      * @return Shared pointer to gRPC channel, or nullptr if acquisition failed
      * 
      * @error_contract

src/auth/distributed_token_blacklist.cpp

@@ -336,16 +336,16 @@ DistributedTokenBlacklist::DistributedTokenBlacklist(
         config_.column_family, rocksdb::ColumnFamilyOptions{}));
 
     std::vector<rocksdb::ColumnFamilyHandle*> cf_handles;
-    std::unique_ptr<rocksdb::DB> db_instance;
+    rocksdb::DB* db_raw = nullptr;
     rocksdb::Status status = rocksdb::DB::Open(
-        rocksdb::DBOptions{opts}, config_.db_path, cf_descriptors, &cf_handles, &db_instance);
+        rocksdb::DBOptions{opts}, config_.db_path, cf_descriptors, &cf_handles, &db_raw);
 
     if (!status.ok()) {
         throw std::runtime_error(
             std::string("Cannot open RocksDB: ") + status.ToString());
     }
 
-    db_ = std::move(db_instance);
+    db_ = std::unique_ptr<rocksdb::DB>(db_raw);
     cf_ = cf_handles[1];  // Our column family (not default)
 
     // Keep other CF handles alive for proper cleanup