overload: add shutdown overload action

api/envoy/config/overload/v3/overload.proto

@@ -153,6 +153,25 @@ message ShrinkHeapConfig {
   google.protobuf.UInt64Value max_unfreed_memory_bytes = 2;
 }
 
+// Typed configuration for the "envoy.overload_actions.shutdown" action. See
+// :ref:`the docs <config_overload_manager_shutdown>` for an example of how to configure this
+// action.
+message ShutdownConfig {
+  // How long the action must stay continuously saturated before Envoy shuts down. The countdown
+  // restarts from zero every time the action leaves the saturated state, so a restarted Envoy
+  // cannot shut down again until it has been saturated for this long. This is the lower bound on
+  // the interval between two shutdowns of a supervised Envoy.
+  google.protobuf.Duration saturation_duration = 1 [(validate.rules).duration = {
+    required: true
+    gte {seconds: 1}
+  }];
+
+  // Upper bound of a random delay added to ``saturation_duration`` each time the countdown starts.
+  // Set this to keep a fleet of Envoys that share an overload condition from shutting down at the
+  // same instant. Defaults to no jitter.
+  google.protobuf.Duration max_jitter = 2 [(validate.rules).duration = {gte {}}];
+}
+
 message OverloadAction {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.overload.v2alpha.OverloadAction";
@@ -167,6 +186,7 @@ message OverloadAction {
   // - envoy.overload_actions.shrink_heap
   // - envoy.overload_actions.reduce_timeouts
   // - envoy.overload_actions.reset_high_memory_stream
+  // - envoy.overload_actions.shutdown
   string name = 1 [(validate.rules).string = {min_len: 1}];
 
   // A set of triggers for this action. The state of the action is the maximum

changelogs/current/new_features/overload_manager__shutdown-action.rst

@@ -0,0 +1,6 @@
+Added the ``envoy.overload_actions.shutdown`` overload action. When the action stays saturated for
+:ref:`saturation_duration <envoy_v3_api_field_config.overload.v3.ShutdownConfig.saturation_duration>`
+without interruption, Envoy drains and exits so that a supervising process can restart it. This
+gives a way out of overload conditions that never clear on their own, such as a memory leak or heap
+fragmentation that keeps memory pressure above the threshold at which Envoy stops accepting
+requests. See :ref:`the docs <config_overload_manager_shutdown>` for details.

docs/root/configuration/operations/overload_manager/_include/shutdown_overload.yaml

@@ -0,0 +1,58 @@
+static_resources:
+  listeners:
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 8000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          http_filters:
+          - name: envoy.filters.http.router
+            typed_config:
+              '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - domains:
+              - '*'
+              name: local_service
+              routes:
+              - match: {prefix: "/"}
+                route: {cluster: default_service}
+  clusters:
+  - name: default_service
+    load_assignment:
+      cluster_name: default_service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 10001
+overload_manager:
+  refresh_interval: 0.25s
+  resource_monitors:
+  - name: "envoy.resource_monitors.fixed_heap"
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig
+      max_heap_size_bytes: 2147483648
+  actions:
+  - name: "envoy.overload_actions.stop_accepting_requests"
+    triggers:
+    - name: "envoy.resource_monitors.fixed_heap"
+      threshold:
+        value: 0.95
+  - name: "envoy.overload_actions.shutdown"
+    typed_config:
+      "@type": type.googleapis.com/envoy.config.overload.v3.ShutdownConfig
+      saturation_duration: 300s
+      max_jitter: 60s
+    triggers:
+    - name: "envoy.resource_monitors.fixed_heap"
+      threshold:
+        value: 0.95

docs/root/configuration/operations/overload_manager/overload_manager.rst

@@ -155,6 +155,11 @@ The following overload actions are supported:
       When the action is in a *scaled active* state, the idle timer threshold is still respected.
       Note that this action is currently only supported for HTTP/3 QUIC connections.
 
+  * - envoy.overload_actions.shutdown
+    - Envoy will shut itself down once the action has stayed *saturated* for a configured duration,
+      leaving it to the supervising process to restart it. See
+      :ref:`below <config_overload_manager_shutdown>` for details on configuration.
+
 .. _config_overload_manager_shrink_heap:
 
 Shrink Heap
@@ -193,6 +198,60 @@ Example configuration:
 
 If no ``typed_config`` is provided, the action will use default values.
 
+.. _config_overload_manager_shutdown:
+
+Shutdown
+^^^^^^^^
+
+Overload conditions are expected to be transient. One that persists can leave Envoy in a state
+that only a restart clears: a memory leak or heap fragmentation, for example, can hold the memory
+pressure above the threshold that made Envoy stop accepting requests, so the pressure never drops
+and Envoy never recovers. The ``envoy.overload_actions.shutdown`` overload action detects that
+state and exits the process. Restarting Envoy is left to whatever supervises it, such as
+Kubernetes, systemd, or the :ref:`hot restart wrapper <operations_hot_restarter>`. Envoy does not
+restart itself.
+
+The action requires a :ref:`ShutdownConfig <envoy_v3_api_msg_config.overload.v3.ShutdownConfig>`
+``typed_config``:
+
+.. list-table::
+  :header-rows: 1
+  :widths: 1, 1, 2
+
+  * - Parameter
+    - Default
+    - Description
+  * - saturation_duration
+    - required
+    - How long the action must stay continuously saturated before Envoy shuts down
+  * - max_jitter
+    - 0s
+    - Upper bound of a random delay added to ``saturation_duration`` each time the countdown starts
+
+Envoy shuts down only once the action has been saturated without interruption for
+``saturation_duration``. Any recovery cancels the countdown, which starts again from zero the next
+time the action saturates. A restarted Envoy therefore has to stay saturated for that long before
+it can shut down again, which is what bounds how often the deployment restarts. Choose a value
+comfortably longer than the time a healthy Envoy needs to start up and work off a traffic spike.
+Set ``max_jitter`` when a fleet of Envoys shares the same overload condition, so that they do not
+all exit at the same instant.
+
+Envoy shuts down the same way it does for a graceful restart: it fails its health check, stops
+accepting new connections, drain closes established ones over :option:`--drain-time-s` at the rate
+set by :option:`--drain-strategy`, and exits when that window closes. Lower
+:option:`--drain-time-s` if the default delays the restart more than the deployment can afford. The
+action increments ``overload.envoy.overload_actions.shutdown.shutdown_count`` when it decides to
+shut Envoy down.
+
+Example configuration:
+
+.. literalinclude:: _include/shutdown_overload.yaml
+    :language: yaml
+    :lines: 44-58
+    :emphasize-lines: 7-15
+    :linenos:
+    :caption: :download:`shutdown_overload.yaml <_include/shutdown_overload.yaml>`
+
 Load Shed Points
 ----------------
 

envoy/server/overload/overload_manager.h

@@ -44,17 +44,21 @@ class OverloadActionNameValues {
   // Overload action to terminate idle downstream HTTP connections.
   const std::string CloseIdleHttpConnections = "envoy.overload_actions.close_idle_http_connections";
 
+  // Overload action to shut the server down so that a supervisor can restart it.
+  const std::string Shutdown = "envoy.overload_actions.shutdown";
+
   // This should be kept current with the Overload actions available.
   // This is the last member of this class to duplicating the strings with
   // proper lifetime guarantees.
-  const std::array<absl::string_view, 8> WellKnownActions = {StopAcceptingRequests,
+  const std::array<absl::string_view, 9> WellKnownActions = {StopAcceptingRequests,
                                                              DisableHttpKeepAlive,
                                                              StopAcceptingConnections,
                                                              RejectIncomingConnections,
                                                              ShrinkHeap,
                                                              ReduceTimeouts,
                                                              ResetStreams,
-                                                             CloseIdleHttpConnections};
+                                                             CloseIdleHttpConnections,
+                                                             Shutdown};
 };
 
 using OverloadActionNames = ConstSingleton<OverloadActionNameValues>;
@@ -119,6 +123,12 @@ class OverloadManager : public LoadShedPointProvider {
    */
   virtual std::optional<envoy::config::overload::v3::ShrinkHeapConfig>
   getShrinkHeapConfig() const PURE;
+
+  /**
+   * Get the configuration for the Shutdown overload action.
+   * @return optional config, empty if no Shutdown action is configured.
+   */
+  virtual std::optional<envoy::config::overload::v3::ShutdownConfig> getShutdownConfig() const PURE;
 };
 
 } // namespace Server

source/server/BUILD

@@ -338,6 +338,23 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "overload_shutdown_lib",
+    srcs = ["overload_shutdown.cc"],
+    hdrs = ["overload_shutdown.h"],
+    deps = [
+        "//envoy/event:timer_interface",
+        "//envoy/server:drain_manager_interface",
+        "//envoy/server:instance_interface",
+        "//envoy/server/overload:overload_manager_interface",
+        "//envoy/stats:stats_interface",
+        "//source/common/common:logger_lib",
+        "//source/common/protobuf:utility_lib",
+        "//source/common/stats:symbol_table_lib",
+        "@envoy_api//envoy/config/overload/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_cc_library(
     name = "factory_context_lib",
     srcs = ["factory_context_impl.cc"],
@@ -397,6 +414,7 @@ envoy_cc_library(
         ":configuration_lib",
         ":listener_hooks_lib",
         ":listener_manager_factory_lib",
+        ":overload_shutdown_lib",
         ":regex_engine_lib",
         ":utils_lib",
         ":worker_lib",

source/server/null_overload_manager.h

@@ -67,6 +67,9 @@ class NullOverloadManager : public OverloadManager {
   getShrinkHeapConfig() const override {
     return std::nullopt;
   }
+  std::optional<envoy::config::overload::v3::ShutdownConfig> getShutdownConfig() const override {
+    return std::nullopt;
+  }
 
   ThreadLocal::SlotPtr tls_;
   // The admin code runs in non-permissive mode, rejecting connections and

source/server/overload_manager_impl.cc

@@ -522,6 +522,15 @@ OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::S
             MessageUtil::anyConvertAndValidate<envoy::config::overload::v3::ShrinkHeapConfig>(
                 action.typed_config(), validation_visitor);
       }
+    } else if (name == OverloadActionNames::get().Shutdown) {
+      if (!action.has_typed_config()) {
+        creation_status = absl::InvalidArgumentError(
+            fmt::format("Overload action \"{}\" requires a ShutdownConfig typed_config.", name));
+        return;
+      }
+      shutdown_config_ =
+          MessageUtil::anyConvertAndValidate<envoy::config::overload::v3::ShutdownConfig>(
+              action.typed_config(), validation_visitor);
     } else if (action.has_typed_config()) {
       creation_status = absl::InvalidArgumentError(fmt::format(
           "Overload action \"{}\" has an unexpected value for the typed_config field", name));

source/server/overload_manager_impl.h

@@ -171,6 +171,9 @@ class OverloadManagerImpl : Logger::Loggable<Logger::Id::main>, public OverloadM
   getShrinkHeapConfig() const override {
     return shrink_heap_config_;
   }
+  std::optional<envoy::config::overload::v3::ShutdownConfig> getShutdownConfig() const override {
+    return shutdown_config_;
+  }
 
 protected:
   OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::Scope& stats_scope,
@@ -260,6 +263,7 @@ class OverloadManagerImpl : Logger::Loggable<Logger::Id::main>, public OverloadM
   ActionToCallbackMap action_to_callbacks_;
 
   std::optional<envoy::config::overload::v3::ShrinkHeapConfig> shrink_heap_config_;
+  std::optional<envoy::config::overload::v3::ShutdownConfig> shutdown_config_;
 };
 
 } // namespace Server

source/server/overload_shutdown.cc

@@ -0,0 +1,83 @@
+#include "source/server/overload_shutdown.h"
+
+#include "envoy/server/drain_manager.h"
+
+#include "source/common/protobuf/utility.h"
+#include "source/common/stats/symbol_table.h"
+
+#include "absl/strings/str_cat.h"
+
+namespace Envoy {
+namespace Server {
+
+OverloadShutdown::OverloadShutdown(Instance& server, OverloadManager& overload_manager,
+                                   Stats::Scope& stats)
+    : server_(server) {
+  const auto config = overload_manager.getShutdownConfig();
+  if (!config.has_value()) {
+    return;
+  }
+
+  saturation_duration_ = std::chrono::milliseconds(
+      DurationUtil::durationToMilliseconds(config->saturation_duration()));
+  max_jitter_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(*config, max_jitter, 0));
+
+  const std::string& action_name = OverloadActionNames::get().Shutdown;
+  if (!overload_manager.registerForAction(
+          action_name, server.dispatcher(),
+          [this](OverloadActionState state) { onActionStateChanged(state); })) {
+    return;
+  }
+
+  Stats::StatNameManagedStorage stat_name(absl::StrCat("overload.", action_name, ".shutdown_count"),
+                                          stats.symbolTable());
+  shutdown_counter_ = &stats.counterFromStatName(stat_name.statName());
+  saturation_timer_ = server.dispatcher().createTimer([this] { shutdownServer(); });
+}
+
+void OverloadShutdown::onActionStateChanged(OverloadActionState state) {
+  if (shutting_down_) {
+    return;
+  }
+
+  if (!state.isSaturated()) {
+    if (saturation_timer_->enabled()) {
+      ENVOY_LOG(info, "overload action {} is no longer saturated, canceling the pending shutdown",
+                OverloadActionNames::get().Shutdown);
+      saturation_timer_->disableTimer();
+    }
+    return;
+  }
+
+  if (saturation_timer_->enabled()) {
+    return;
+  }
+  const std::chrono::milliseconds delay = shutdownDelay();
+  ENVOY_LOG(warn,
+            "overload action {} is saturated, shutting down the server in {} ms unless it recovers",
+            OverloadActionNames::get().Shutdown, delay.count());
+  saturation_timer_->enableTimer(delay);
+}
+
+std::chrono::milliseconds OverloadShutdown::shutdownDelay() const {
+  if (max_jitter_.count() == 0) {
+    return saturation_duration_;
+  }
+  return saturation_duration_ + std::chrono::milliseconds(server_.api().randomGenerator().random() %
+                                                          (max_jitter_.count() + 1));
+}
+
+void OverloadShutdown::shutdownServer() {
+  shutting_down_ = true;
+  shutdown_counter_->inc();
+  ENVOY_LOG(critical, "shutting down the server because overload action {} stayed saturated",
+            OverloadActionNames::get().Shutdown);
+
+  server_.failHealthcheck(true);
+  server_.drainListeners();
+  server_.drainManager().startDrainSequence(Network::DrainDirection::All,
+                                            [this]() { server_.shutdown(); });
+}
+
+} // namespace Server
+} // namespace Envoy