dynamic_modules: fix cluster specifier override validation, extend ABI

api/envoy/extensions/router/cluster_specifiers/dynamic_modules/v3/dynamic_modules.proto

@@ -29,8 +29,8 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // freely by the module on each request.
 //
 // Each entry is built and validated when the cluster specifier is configured, so an invalid entry
-// is rejected at configuration load rather than on the request path. At least one property must be
-// specified.
+// is rejected at configuration load rather than on the request path. Every entry must replace at
+// least one property, so an entry that builds to no replacement is rejected as well.
 message RouteActionOverride {
   // Retry policy replacing the retry policy of the matched route. If not specified, the retry
   // policy of the matched route is used, so an entry cannot be used to remove a retry policy that
@@ -43,19 +43,22 @@ message RouteActionOverride {
   // :ref:`RouteAction.metadata_match
   // <envoy_v3_api_field_config.route.v3.RouteAction.metadata_match>`. If not specified, or if there
   // is no ``envoy.lb`` entry, the criteria of the matched route are used, so an entry cannot be used
-  // to remove the criteria that the matched route configures.
+  // to remove the criteria that the matched route configures. An entry whose only property is a
+  // ``metadata_match`` without an ``envoy.lb`` entry therefore replaces nothing and is rejected.
   config.core.v3.Metadata metadata_match = 2;
 
   // Request mirroring policies replacing the request mirroring policies of the matched route. If
   // not specified, the request mirroring policies of the matched route are used, so an entry cannot
-  // be used to turn off mirroring that the matched route configures.
+  // be used to turn off mirroring that the matched route configures. Statically named mirror
+  // clusters are checked against the cluster manager when :ref:`validate_clusters
+  // <envoy_v3_api_field_config.route.v3.RouteConfiguration.validate_clusters>` is enabled.
   repeated config.route.v3.RouteAction.RequestMirrorPolicy request_mirror_policies = 3;
 }
 
 // Configuration for the Dynamic Modules Cluster Specifier. This cluster specifier allows loading
 // shared object files via ``dlopen`` to select the upstream cluster for a request, and to replace
-// the timeout, idle timeout, priority, request body buffer limit, retry policy, metadata match
-// criteria and request mirroring policies of the matched route.
+// the timeout, idle timeout, priority, request body buffer limit, cluster not found response code,
+// retry policy, metadata match criteria and request mirroring policies of the matched route.
 //
 // A module can be loaded by multiple cluster specifiers. It is loaded only once and shared across
 // multiple cluster specifier instances. The module is invoked while the route is being
@@ -112,8 +115,8 @@ message DynamicModuleClusterSpecifier {
   google.protobuf.Any specifier_config = 3;
 
   // Route action overrides that the module may select by name for a request. If not specified, the
-  // module can only select the cluster, the timeout, the idle timeout, the priority and the request
-  // body buffer limit.
+  // module can only select the cluster, the timeout, the idle timeout, the priority, the request
+  // body buffer limit and the cluster not found response code.
   map<string, RouteActionOverride> route_action_overrides = 4
       [(validate.rules).map = {keys {string {min_len: 1}}}];
 }

changelogs/current/new_features/dynamic_modules__added-a-cluster-specifier-extension.rst

@@ -1,7 +1,8 @@
 Added a dynamic modules cluster specifier extension
 (``envoy.router.cluster_specifier_plugin.dynamic_modules``) that lets a dynamic module select the
 upstream cluster for a request and replace the timeout, idle timeout, priority, request body buffer
-limit, retry policy, metadata match criteria and request mirroring policies of the matched route.
+limit, cluster not found response code, retry policy, metadata match criteria and request mirroring
+policies of the matched route.
 The selection context exposes the request headers, stream info attributes, dynamic metadata, the
 route name, and the random value Envoy generated for cluster selection, and the module is invoked
 again whenever a filter refreshes the route cluster. The Rust SDK exposes this through the

docs/root/configuration/http/cluster_specifier/dynamic_modules.rst

@@ -9,8 +9,8 @@ Overview
 The :ref:`DynamicModuleClusterSpecifier <envoy_v3_api_msg_extensions.router.cluster_specifiers.dynamic_modules.v3.DynamicModuleClusterSpecifier>`
 configuration specifies a cluster specifier backed by a :ref:`dynamic module <arch_overview_dynamic_modules>`.
 The module selects the upstream cluster for a request and may replace the timeout, idle timeout,
-priority, request body buffer limit, retry policy, metadata match criteria and request mirroring
-policies of the matched route.
+priority, request body buffer limit, cluster not found response code, retry policy, metadata match
+criteria and request mirroring policies of the matched route.
 
 The module is invoked while the route is being resolved, so its selection is visible to the router
 without clearing the route cache. It is invoked again whenever a filter calls
@@ -22,8 +22,12 @@ The properties that are built from other extensions are declared as named
 :ref:`route action overrides <envoy_v3_api_field_extensions.router.cluster_specifiers.dynamic_modules.v3.DynamicModuleClusterSpecifier.route_action_overrides>`.
 Each override is built and validated once when the cluster specifier is configured, so an invalid
 override is rejected at configuration load rather than on the request path, and the module selects
-one by name. Selecting a name that is not declared leaves the route action properties of the matched
-route in effect and logs a warning.
+one by name. An override that replaces no property is rejected too, so a module can rely on a
+declared name changing something. Selecting a name that is not declared leaves the route action
+properties of the matched route in effect and logs a warning. When
+:ref:`validate_clusters <envoy_v3_api_field_config.route.v3.RouteConfiguration.validate_clusters>`
+is enabled, the statically named mirror clusters of every override are checked against the cluster
+manager.
 
 Not every property can be changed once the request is under way, because Envoy reads each of them at
 a different point:
@@ -36,6 +40,8 @@ a different point:
 * The idle timeout and the request body buffer limit are read while the route is resolved, and the
   timeout, the retry policy and the request mirroring policies are read before the first upstream
   attempt, so only the first selection applies to them.
+* The cluster not found response code is read only when the selected cluster does not exist, which
+  ends the request, so the selection that named the missing cluster is the one that applies.
 
 Configuration
 -------------

source/extensions/dynamic_modules/abi/abi.h

@@ -14890,6 +14890,36 @@ bool envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata(
     envoy_dynamic_module_type_module_buffer filter_name,
     envoy_dynamic_module_type_module_buffer path, envoy_dynamic_module_type_envoy_buffer* result);
 
+/**
+ * envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_number is called by the
+ * module to get a number value from dynamic metadata by filter name and key path.
+ *
+ * @param context_envoy_ptr is the pointer to the cluster selection context.
+ * @param filter_name is the filter namespace in dynamic metadata.
+ * @param path is the key path within the filter namespace, which may be nested with dots.
+ * @param result receives the number value. It is left untouched when this returns false.
+ * @return true if a number value exists at the path, false otherwise.
+ */
+bool envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_number(
+    envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr context_envoy_ptr,
+    envoy_dynamic_module_type_module_buffer filter_name,
+    envoy_dynamic_module_type_module_buffer path, double* result);
+
+/**
+ * envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_bool is called by the module
+ * to get a boolean value from dynamic metadata by filter name and key path.
+ *
+ * @param context_envoy_ptr is the pointer to the cluster selection context.
+ * @param filter_name is the filter namespace in dynamic metadata.
+ * @param path is the key path within the filter namespace, which may be nested with dots.
+ * @param result receives the boolean value. It is left untouched when this returns false.
+ * @return true if a boolean value exists at the path, false otherwise.
+ */
+bool envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_bool(
+    envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr context_envoy_ptr,
+    envoy_dynamic_module_type_module_buffer filter_name,
+    envoy_dynamic_module_type_module_buffer path, bool* result);
+
 /**
  * envoy_dynamic_module_callback_cluster_specifier_get_route_name is called by the module to get the
  * name of the matched route.
@@ -14986,6 +15016,22 @@ void envoy_dynamic_module_callback_cluster_specifier_set_priority(
     envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr context_envoy_ptr,
     envoy_dynamic_module_type_resource_priority priority);
 
+/**
+ * envoy_dynamic_module_callback_cluster_specifier_set_cluster_not_found_response_code sets the
+ * status code Envoy replies with when the selected cluster does not exist, replacing the one of the
+ * matched route. A module that derives cluster names from the request can use this to distinguish a
+ * name that resolves to nothing from an upstream that is unavailable.
+ *
+ * @param context_envoy_ptr is the pointer to the cluster selection context.
+ * @param status_code is the HTTP status code to reply with. Only codes in the range [200, 600) are
+ * accepted, because the code is used to terminate the request.
+ * @return true if the status code was accepted, false when it is out of range, in which case the
+ * call changes nothing.
+ */
+bool envoy_dynamic_module_callback_cluster_specifier_set_cluster_not_found_response_code(
+    envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr context_envoy_ptr,
+    uint32_t status_code);
+
 /**
  * envoy_dynamic_module_callback_cluster_specifier_set_route_action_override selects one of the
  * route action overrides declared in the cluster specifier configuration. The selected override

source/extensions/dynamic_modules/abi_impl.cc

@@ -2836,6 +2836,24 @@ __attribute__((weak)) bool envoy_dynamic_module_callback_cluster_specifier_get_d
   return false;
 }
 
+__attribute__((weak)) bool
+envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_number(
+    envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr,
+    envoy_dynamic_module_type_module_buffer, envoy_dynamic_module_type_module_buffer, double*) {
+  IS_ENVOY_BUG("envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_number: not "
+               "implemented in this context");
+  return false;
+}
+
+__attribute__((weak)) bool
+envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_bool(
+    envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr,
+    envoy_dynamic_module_type_module_buffer, envoy_dynamic_module_type_module_buffer, bool*) {
+  IS_ENVOY_BUG("envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_bool: not "
+               "implemented in this context");
+  return false;
+}
+
 __attribute__((weak)) uint64_t envoy_dynamic_module_callback_cluster_specifier_get_random_value(
     envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr) {
   IS_ENVOY_BUG("envoy_dynamic_module_callback_cluster_specifier_get_random_value: not implemented "
@@ -2897,6 +2915,14 @@ __attribute__((weak)) void envoy_dynamic_module_callback_cluster_specifier_set_p
       "context");
 }
 
+__attribute__((weak)) bool
+envoy_dynamic_module_callback_cluster_specifier_set_cluster_not_found_response_code(
+    envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr, uint32_t) {
+  IS_ENVOY_BUG("envoy_dynamic_module_callback_cluster_specifier_set_cluster_not_found_response_"
+               "code: not implemented in this context");
+  return false;
+}
+
 __attribute__((weak)) void
 envoy_dynamic_module_callback_cluster_specifier_set_request_body_buffer_limit(
     envoy_dynamic_module_type_cluster_specifier_context_envoy_ptr, uint64_t) {

source/extensions/dynamic_modules/sdk/rust/src/cluster_specifier.rs

@@ -225,6 +225,49 @@ impl ClusterSpecifierContext {
     }
   }
 
+  /// Get a number value from dynamic metadata.
+  ///
+  /// The arguments are the same as [`Self::get_dynamic_metadata`]. Only number values are returned.
+  pub fn get_dynamic_metadata_number(&self, filter_name: &str, path: &str) -> Option<f64> {
+    let filter_buf = crate::str_to_module_buffer(filter_name);
+    let path_buf = crate::str_to_module_buffer(path);
+    let mut result: f64 = 0.0;
+    if unsafe {
+      abi::envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_number(
+        self.envoy_ptr,
+        filter_buf,
+        path_buf,
+        &mut result,
+      )
+    } {
+      Some(result)
+    } else {
+      None
+    }
+  }
+
+  /// Get a boolean value from dynamic metadata.
+  ///
+  /// The arguments are the same as [`Self::get_dynamic_metadata`]. Only boolean values are
+  /// returned.
+  pub fn get_dynamic_metadata_bool(&self, filter_name: &str, path: &str) -> Option<bool> {
+    let filter_buf = crate::str_to_module_buffer(filter_name);
+    let path_buf = crate::str_to_module_buffer(path);
+    let mut result: bool = false;
+    if unsafe {
+      abi::envoy_dynamic_module_callback_cluster_specifier_get_dynamic_metadata_bool(
+        self.envoy_ptr,
+        filter_buf,
+        path_buf,
+        &mut result,
+      )
+    } {
+      Some(result)
+    } else {
+      None
+    }
+  }
+
   /// Get the name of the matched route.
   ///
   /// This returns `None` when the route has no name.
@@ -318,6 +361,22 @@ impl ClusterSpecifierContext {
     }
   }
 
+  /// Set the status code Envoy replies with when the selected cluster does not exist.
+  ///
+  /// This replaces the cluster-not-found response code of the matched route, letting a module that
+  /// derives cluster names from the request distinguish a name that resolves to nothing from an
+  /// upstream that is unavailable. Returns `false` when the status code is outside the range
+  /// [200, 600), in which case the call changes nothing.
+  #[must_use]
+  pub fn set_cluster_not_found_response_code(&mut self, status_code: u32) -> bool {
+    unsafe {
+      abi::envoy_dynamic_module_callback_cluster_specifier_set_cluster_not_found_response_code(
+        self.envoy_ptr,
+        status_code,
+      )
+    }
+  }
+
   /// Select one of the route action overrides declared in the cluster specifier configuration.
   ///
   /// The selected override replaces the retry policy, the subset load balancing metadata match