From 84993ae05fc25f41b17a6bb3caa031b8dec80863 Mon Sep 17 00:00:00 2001 From: salman-frs Date: Mon, 24 Aug 2026 18:48:30 +0700 Subject: [PATCH] overload: add shutdown overload action An overload condition is meant to be transient, but some never clear. A memory leak or heap fragmentation can hold the fixed heap monitor above the stop_accepting_requests threshold indefinitely, so Envoy stays up rejecting everything, and no OOM killer fires because the process is not actually at a hard limit. Adds envoy.overload_actions.shutdown: when the action stays saturated without interruption for saturation_duration, Envoy optionally drains for drain_time and then exits, leaving the restart to the supervisor. Requiring continuous saturation is what bounds the restart rate, since a freshly started Envoy has to be saturated for saturation_duration all over again before it can shut down. Signed-off-by: salman-frs --- api/envoy/config/overload/v3/overload.proto | 20 +++ .../overload_manager__shutdown-action.rst | 6 + .../_include/shutdown_overload.yaml | 58 ++++++ .../overload_manager/overload_manager.rst | 59 ++++++ envoy/server/overload/overload_manager.h | 14 +- source/server/BUILD | 18 ++ source/server/null_overload_manager.h | 3 + source/server/overload_manager_impl.cc | 9 + source/server/overload_manager_impl.h | 4 + source/server/overload_shutdown.cc | 83 +++++++++ source/server/overload_shutdown.h | 39 ++++ source/server/server.cc | 2 + source/server/server.h | 2 + test/integration/overload_integration_test.cc | 21 +++ test/mocks/server/overload_manager.h | 2 + test/server/BUILD | 13 ++ test/server/overload_manager_impl_test.cc | 63 +++++++ test/server/overload_shutdown_test.cc | 169 ++++++++++++++++++ 18 files changed, 583 insertions(+), 2 deletions(-) create mode 100644 changelogs/current/new_features/overload_manager__shutdown-action.rst create mode 100644 docs/root/configuration/operations/overload_manager/_include/shutdown_overload.yaml create mode 100644 source/server/overload_shutdown.cc create mode 100644 source/server/overload_shutdown.h create mode 100644 test/server/overload_shutdown_test.cc diff --git a/api/envoy/config/overload/v3/overload.proto b/api/envoy/config/overload/v3/overload.proto index 05e6b2a129331..77aef6ce59967 100644 --- a/api/envoy/config/overload/v3/overload.proto +++ b/api/envoy/config/overload/v3/overload.proto @@ -153,6 +153,25 @@ message ShrinkHeapConfig { google.protobuf.UInt64Value max_unfreed_memory_bytes = 2; } +// Typed configuration for the "envoy.overload_actions.shutdown" action. See +// :ref:`the docs ` for an example of how to configure this +// action. +message ShutdownConfig { + // How long the action must stay continuously saturated before Envoy shuts down. The countdown + // restarts from zero every time the action leaves the saturated state, so a restarted Envoy + // cannot shut down again until it has been saturated for this long. This is the lower bound on + // the interval between two shutdowns of a supervised Envoy. + google.protobuf.Duration saturation_duration = 1 [(validate.rules).duration = { + required: true + gte {seconds: 1} + }]; + + // Upper bound of a random delay added to ``saturation_duration`` each time the countdown starts. + // Set this to keep a fleet of Envoys that share an overload condition from shutting down at the + // same instant. Defaults to no jitter. + google.protobuf.Duration max_jitter = 2 [(validate.rules).duration = {gte {}}]; +} + message OverloadAction { option (udpa.annotations.versioning).previous_message_type = "envoy.config.overload.v2alpha.OverloadAction"; @@ -167,6 +186,7 @@ message OverloadAction { // - envoy.overload_actions.shrink_heap // - envoy.overload_actions.reduce_timeouts // - envoy.overload_actions.reset_high_memory_stream + // - envoy.overload_actions.shutdown string name = 1 [(validate.rules).string = {min_len: 1}]; // A set of triggers for this action. The state of the action is the maximum diff --git a/changelogs/current/new_features/overload_manager__shutdown-action.rst b/changelogs/current/new_features/overload_manager__shutdown-action.rst new file mode 100644 index 0000000000000..6f2998179ccdb --- /dev/null +++ b/changelogs/current/new_features/overload_manager__shutdown-action.rst @@ -0,0 +1,6 @@ +Added the ``envoy.overload_actions.shutdown`` overload action. When the action stays saturated for +:ref:`saturation_duration ` +without interruption, Envoy drains and exits so that a supervising process can restart it. This +gives a way out of overload conditions that never clear on their own, such as a memory leak or heap +fragmentation that keeps memory pressure above the threshold at which Envoy stops accepting +requests. See :ref:`the docs ` for details. diff --git a/docs/root/configuration/operations/overload_manager/_include/shutdown_overload.yaml b/docs/root/configuration/operations/overload_manager/_include/shutdown_overload.yaml new file mode 100644 index 0000000000000..d0499544618e2 --- /dev/null +++ b/docs/root/configuration/operations/overload_manager/_include/shutdown_overload.yaml @@ -0,0 +1,58 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + http_filters: + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_route + virtual_hosts: + - domains: + - '*' + name: local_service + routes: + - match: {prefix: "/"} + route: {cluster: default_service} + clusters: + - name: default_service + load_assignment: + cluster_name: default_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 10001 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig + max_heap_size_bytes: 2147483648 + actions: + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + - name: "envoy.overload_actions.shutdown" + typed_config: + "@type": type.googleapis.com/envoy.config.overload.v3.ShutdownConfig + saturation_duration: 300s + max_jitter: 60s + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index ec28f16ce150c..406c2cb80bc26 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -155,6 +155,11 @@ The following overload actions are supported: When the action is in a *scaled active* state, the idle timer threshold is still respected. Note that this action is currently only supported for HTTP/3 QUIC connections. + * - envoy.overload_actions.shutdown + - Envoy will shut itself down once the action has stayed *saturated* for a configured duration, + leaving it to the supervising process to restart it. See + :ref:`below ` for details on configuration. + .. _config_overload_manager_shrink_heap: Shrink Heap @@ -193,6 +198,60 @@ Example configuration: If no ``typed_config`` is provided, the action will use default values. +.. _config_overload_manager_shutdown: + +Shutdown +^^^^^^^^ + +Overload conditions are expected to be transient. One that persists can leave Envoy in a state +that only a restart clears: a memory leak or heap fragmentation, for example, can hold the memory +pressure above the threshold that made Envoy stop accepting requests, so the pressure never drops +and Envoy never recovers. The ``envoy.overload_actions.shutdown`` overload action detects that +state and exits the process. Restarting Envoy is left to whatever supervises it, such as +Kubernetes, systemd, or the :ref:`hot restart wrapper `. Envoy does not +restart itself. + +The action requires a :ref:`ShutdownConfig ` +``typed_config``: + +.. list-table:: + :header-rows: 1 + :widths: 1, 1, 2 + + * - Parameter + - Default + - Description + * - saturation_duration + - required + - How long the action must stay continuously saturated before Envoy shuts down + * - max_jitter + - 0s + - Upper bound of a random delay added to ``saturation_duration`` each time the countdown starts + +Envoy shuts down only once the action has been saturated without interruption for +``saturation_duration``. Any recovery cancels the countdown, which starts again from zero the next +time the action saturates. A restarted Envoy therefore has to stay saturated for that long before +it can shut down again, which is what bounds how often the deployment restarts. Choose a value +comfortably longer than the time a healthy Envoy needs to start up and work off a traffic spike. +Set ``max_jitter`` when a fleet of Envoys shares the same overload condition, so that they do not +all exit at the same instant. + +Envoy shuts down the same way it does for a graceful restart: it fails its health check, stops +accepting new connections, drain closes established ones over :option:`--drain-time-s` at the rate +set by :option:`--drain-strategy`, and exits when that window closes. Lower +:option:`--drain-time-s` if the default delays the restart more than the deployment can afford. The +action increments ``overload.envoy.overload_actions.shutdown.shutdown_count`` when it decides to +shut Envoy down. + +Example configuration: + +.. literalinclude:: _include/shutdown_overload.yaml + :language: yaml + :lines: 44-58 + :emphasize-lines: 7-15 + :linenos: + :caption: :download:`shutdown_overload.yaml <_include/shutdown_overload.yaml>` + Load Shed Points ---------------- diff --git a/envoy/server/overload/overload_manager.h b/envoy/server/overload/overload_manager.h index f6e5bc8e809c6..2a3b890f43fde 100644 --- a/envoy/server/overload/overload_manager.h +++ b/envoy/server/overload/overload_manager.h @@ -44,17 +44,21 @@ class OverloadActionNameValues { // Overload action to terminate idle downstream HTTP connections. const std::string CloseIdleHttpConnections = "envoy.overload_actions.close_idle_http_connections"; + // Overload action to shut the server down so that a supervisor can restart it. + const std::string Shutdown = "envoy.overload_actions.shutdown"; + // This should be kept current with the Overload actions available. // This is the last member of this class to duplicating the strings with // proper lifetime guarantees. - const std::array WellKnownActions = {StopAcceptingRequests, + const std::array WellKnownActions = {StopAcceptingRequests, DisableHttpKeepAlive, StopAcceptingConnections, RejectIncomingConnections, ShrinkHeap, ReduceTimeouts, ResetStreams, - CloseIdleHttpConnections}; + CloseIdleHttpConnections, + Shutdown}; }; using OverloadActionNames = ConstSingleton; @@ -119,6 +123,12 @@ class OverloadManager : public LoadShedPointProvider { */ virtual std::optional getShrinkHeapConfig() const PURE; + + /** + * Get the configuration for the Shutdown overload action. + * @return optional config, empty if no Shutdown action is configured. + */ + virtual std::optional getShutdownConfig() const PURE; }; } // namespace Server diff --git a/source/server/BUILD b/source/server/BUILD index b8ccdf9b47b8c..70ca7474501da 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -338,6 +338,23 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "overload_shutdown_lib", + srcs = ["overload_shutdown.cc"], + hdrs = ["overload_shutdown.h"], + deps = [ + "//envoy/event:timer_interface", + "//envoy/server:drain_manager_interface", + "//envoy/server:instance_interface", + "//envoy/server/overload:overload_manager_interface", + "//envoy/stats:stats_interface", + "//source/common/common:logger_lib", + "//source/common/protobuf:utility_lib", + "//source/common/stats:symbol_table_lib", + "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "factory_context_lib", srcs = ["factory_context_impl.cc"], @@ -397,6 +414,7 @@ envoy_cc_library( ":configuration_lib", ":listener_hooks_lib", ":listener_manager_factory_lib", + ":overload_shutdown_lib", ":regex_engine_lib", ":utils_lib", ":worker_lib", diff --git a/source/server/null_overload_manager.h b/source/server/null_overload_manager.h index 875ec8cf42c1e..ad76ccd2c71b0 100644 --- a/source/server/null_overload_manager.h +++ b/source/server/null_overload_manager.h @@ -67,6 +67,9 @@ class NullOverloadManager : public OverloadManager { getShrinkHeapConfig() const override { return std::nullopt; } + std::optional getShutdownConfig() const override { + return std::nullopt; + } ThreadLocal::SlotPtr tls_; // The admin code runs in non-permissive mode, rejecting connections and diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index e48137de2195f..d55ce49fb3f6f 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -522,6 +522,15 @@ OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::S MessageUtil::anyConvertAndValidate( action.typed_config(), validation_visitor); } + } else if (name == OverloadActionNames::get().Shutdown) { + if (!action.has_typed_config()) { + creation_status = absl::InvalidArgumentError( + fmt::format("Overload action \"{}\" requires a ShutdownConfig typed_config.", name)); + return; + } + shutdown_config_ = + MessageUtil::anyConvertAndValidate( + action.typed_config(), validation_visitor); } else if (action.has_typed_config()) { creation_status = absl::InvalidArgumentError(fmt::format( "Overload action \"{}\" has an unexpected value for the typed_config field", name)); diff --git a/source/server/overload_manager_impl.h b/source/server/overload_manager_impl.h index 38dc744e92b37..ab3b5361458b4 100644 --- a/source/server/overload_manager_impl.h +++ b/source/server/overload_manager_impl.h @@ -171,6 +171,9 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM getShrinkHeapConfig() const override { return shrink_heap_config_; } + std::optional getShutdownConfig() const override { + return shutdown_config_; + } protected: OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::Scope& stats_scope, @@ -260,6 +263,7 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM ActionToCallbackMap action_to_callbacks_; std::optional shrink_heap_config_; + std::optional shutdown_config_; }; } // namespace Server diff --git a/source/server/overload_shutdown.cc b/source/server/overload_shutdown.cc new file mode 100644 index 0000000000000..a55c371eb04a5 --- /dev/null +++ b/source/server/overload_shutdown.cc @@ -0,0 +1,83 @@ +#include "source/server/overload_shutdown.h" + +#include "envoy/server/drain_manager.h" + +#include "source/common/protobuf/utility.h" +#include "source/common/stats/symbol_table.h" + +#include "absl/strings/str_cat.h" + +namespace Envoy { +namespace Server { + +OverloadShutdown::OverloadShutdown(Instance& server, OverloadManager& overload_manager, + Stats::Scope& stats) + : server_(server) { + const auto config = overload_manager.getShutdownConfig(); + if (!config.has_value()) { + return; + } + + saturation_duration_ = std::chrono::milliseconds( + DurationUtil::durationToMilliseconds(config->saturation_duration())); + max_jitter_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(*config, max_jitter, 0)); + + const std::string& action_name = OverloadActionNames::get().Shutdown; + if (!overload_manager.registerForAction( + action_name, server.dispatcher(), + [this](OverloadActionState state) { onActionStateChanged(state); })) { + return; + } + + Stats::StatNameManagedStorage stat_name(absl::StrCat("overload.", action_name, ".shutdown_count"), + stats.symbolTable()); + shutdown_counter_ = &stats.counterFromStatName(stat_name.statName()); + saturation_timer_ = server.dispatcher().createTimer([this] { shutdownServer(); }); +} + +void OverloadShutdown::onActionStateChanged(OverloadActionState state) { + if (shutting_down_) { + return; + } + + if (!state.isSaturated()) { + if (saturation_timer_->enabled()) { + ENVOY_LOG(info, "overload action {} is no longer saturated, canceling the pending shutdown", + OverloadActionNames::get().Shutdown); + saturation_timer_->disableTimer(); + } + return; + } + + if (saturation_timer_->enabled()) { + return; + } + const std::chrono::milliseconds delay = shutdownDelay(); + ENVOY_LOG(warn, + "overload action {} is saturated, shutting down the server in {} ms unless it recovers", + OverloadActionNames::get().Shutdown, delay.count()); + saturation_timer_->enableTimer(delay); +} + +std::chrono::milliseconds OverloadShutdown::shutdownDelay() const { + if (max_jitter_.count() == 0) { + return saturation_duration_; + } + return saturation_duration_ + std::chrono::milliseconds(server_.api().randomGenerator().random() % + (max_jitter_.count() + 1)); +} + +void OverloadShutdown::shutdownServer() { + shutting_down_ = true; + shutdown_counter_->inc(); + ENVOY_LOG(critical, "shutting down the server because overload action {} stayed saturated", + OverloadActionNames::get().Shutdown); + + server_.failHealthcheck(true); + server_.drainListeners(); + server_.drainManager().startDrainSequence(Network::DrainDirection::All, + [this]() { server_.shutdown(); }); +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/overload_shutdown.h b/source/server/overload_shutdown.h new file mode 100644 index 0000000000000..7a2102ea9f098 --- /dev/null +++ b/source/server/overload_shutdown.h @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include "envoy/config/overload/v3/overload.pb.h" +#include "envoy/event/timer.h" +#include "envoy/server/instance.h" +#include "envoy/server/overload/overload_manager.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats.h" + +#include "source/common/common/logger.h" + +namespace Envoy { +namespace Server { + +/** + * Shuts the server down once the "envoy.overload_actions.shutdown" overload action has stayed + * saturated for the configured duration, leaving it to a supervising process to restart Envoy. + */ +class OverloadShutdown : Logger::Loggable { +public: + OverloadShutdown(Instance& server, OverloadManager& overload_manager, Stats::Scope& stats); + +private: + void onActionStateChanged(OverloadActionState state); + void shutdownServer(); + std::chrono::milliseconds shutdownDelay() const; + + Instance& server_; + std::chrono::milliseconds saturation_duration_{}; + std::chrono::milliseconds max_jitter_{}; + Stats::Counter* shutdown_counter_{}; + Event::TimerPtr saturation_timer_; + bool shutting_down_{false}; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/server.cc b/source/server/server.cc index 0ab8331c4788f..3ce7909b7cb93 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -698,6 +698,8 @@ absl::Status InstanceBase::initializeOrThrow(Network::Address::InstanceConstShar null_overload_manager_ = createNullOverloadManager(); maybeCreateHeapShrinker(); + overload_shutdown_ = + std::make_unique(*this, *overload_manager_, *stats_store_.rootScope()); for (const auto& bootstrap_extension : bootstrap_.bootstrap_extensions()) { auto& factory = Config::Utility::getAndCheckFactory( diff --git a/source/server/server.h b/source/server/server.h index c07cdd1471369..f80b1ae9aa341 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -48,6 +48,7 @@ #include "source/server/configuration_impl.h" #include "source/server/listener_hooks.h" +#include "source/server/overload_shutdown.h" #include "source/server/worker_impl.h" #include "absl/container/node_hash_map.h" @@ -428,6 +429,7 @@ class InstanceBase : Logger::Loggable, std::unique_ptr hds_delegate_; std::unique_ptr overload_manager_; std::unique_ptr null_overload_manager_; + std::unique_ptr overload_shutdown_; std::vector bootstrap_extensions_; std::unique_ptr http_server_properties_cache_manager_; Envoy::MutexTracer* mutex_tracer_; diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index 87f962a12f129..2090a6b7c435b 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -269,6 +269,27 @@ TEST_P(OverloadIntegrationTest, StopAcceptingConnectionsWhenOverloaded) { codec_client_->close(); } +TEST_P(OverloadIntegrationTest, ShutDownWhenSaturationPersists) { + // The long drain time keeps the server up after it decides to shut down, so that the test can + // observe the decision rather than race the process exiting. + drain_time_ = std::chrono::seconds(999); + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.overload_actions.shutdown" + typed_config: + "@type": type.googleapis.com/envoy.config.overload.v3.ShutdownConfig + saturation_duration: 1s + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.95 + )EOF")); + + updateResource(0.95); + test_server_->waitForCounter("overload.envoy.overload_actions.shutdown.shutdown_count", Eq(1)); + test_server_->waitForGauge("server.live", Eq(0)); +} + TEST_P(OverloadIntegrationTest, BypassOverloadManagerTest) { initializeWithBypassOverloadManager( TestUtility::parseYaml(R"EOF( diff --git a/test/mocks/server/overload_manager.h b/test/mocks/server/overload_manager.h index 8b8b677e82d55..11f55ea4f5cba 100644 --- a/test/mocks/server/overload_manager.h +++ b/test/mocks/server/overload_manager.h @@ -40,6 +40,8 @@ class MockOverloadManager : public OverloadManager { MOCK_METHOD(void, stop, ()); MOCK_METHOD(std::optional, getShrinkHeapConfig, (), (const, override)); + MOCK_METHOD(std::optional, getShutdownConfig, (), + (const, override)); testing::NiceMock overload_state_; }; diff --git a/test/server/BUILD b/test/server/BUILD index 6b8e82decbb93..36f466035d893 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -150,6 +150,19 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "overload_shutdown_test", + srcs = ["overload_shutdown_test.cc"], + rbe_pool = "6gig", + deps = [ + "//source/server:overload_shutdown_lib", + "//test/common/stats:stat_test_utility_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/server:instance_mocks", + "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "filter_config_test", srcs = ["filter_config_test.cc"], diff --git a/test/server/overload_manager_impl_test.cc b/test/server/overload_manager_impl_test.cc index 221ea29285fe4..c4d614ff8b7d9 100644 --- a/test/server/overload_manager_impl_test.cc +++ b/test/server/overload_manager_impl_test.cc @@ -826,6 +826,69 @@ TEST_F(OverloadManagerImplTest, ShrinkHeapWithoutTypedConfig) { EXPECT_FALSE(config_opt.has_value()); } +TEST_F(OverloadManagerImplTest, ShutdownWithTypedConfig) { + const std::string config = R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.fake_resource1" + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + actions: + - name: "envoy.overload_actions.shutdown" + typed_config: + "@type": type.googleapis.com/envoy.config.overload.v3.ShutdownConfig + saturation_duration: 300s + max_jitter: 60s + triggers: + - name: "envoy.resource_monitors.fake_resource1" + threshold: + value: 0.9 + )EOF"; + + auto manager(createOverloadManager(config)); + auto config_opt = manager->getShutdownConfig(); + ASSERT_TRUE(config_opt.has_value()); + EXPECT_EQ(config_opt->saturation_duration().seconds(), 300); + EXPECT_EQ(config_opt->max_jitter().seconds(), 60); +} + +TEST_F(OverloadManagerImplTest, ShutdownWithoutTypedConfig) { + const std::string config = R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.fake_resource1" + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + actions: + - name: "envoy.overload_actions.shutdown" + triggers: + - name: "envoy.resource_monitors.fake_resource1" + threshold: + value: 0.9 + )EOF"; + + EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, + ".* requires a ShutdownConfig typed_config."); +} + +TEST_F(OverloadManagerImplTest, ShutdownWithoutSaturationDuration) { + const std::string config = R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.fake_resource1" + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + actions: + - name: "envoy.overload_actions.shutdown" + typed_config: + "@type": type.googleapis.com/envoy.config.overload.v3.ShutdownConfig + triggers: + - name: "envoy.resource_monitors.fake_resource1" + threshold: + value: 0.9 + )EOF"; + + EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, + "SaturationDuration: value is required"); +} + TEST_F(OverloadManagerImplTest, ReduceTimeoutsWithoutAction) { const std::string config = R"EOF( actions: diff --git a/test/server/overload_shutdown_test.cc b/test/server/overload_shutdown_test.cc new file mode 100644 index 0000000000000..72dd025c8d212 --- /dev/null +++ b/test/server/overload_shutdown_test.cc @@ -0,0 +1,169 @@ +#include + +#include "envoy/config/overload/v3/overload.pb.h" + +#include "source/server/overload_shutdown.h" + +#include "test/common/stats/stat_test_utility.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/server/instance.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::Return; + +namespace Envoy { +namespace Server { +namespace { + +constexpr char ShutdownCountStat[] = "overload.envoy.overload_actions.shutdown.shutdown_count"; + +class OverloadShutdownTest : public testing::Test { +protected: + envoy::config::overload::v3::ShutdownConfig config_; + + void expectConfigured() { + EXPECT_CALL(server_.overload_manager_, getShutdownConfig()) + .WillRepeatedly(Return(std::make_optional(config_))); + } + + void expectRegistration(bool registered) { + EXPECT_CALL(server_.overload_manager_, registerForAction(_, _, _)) + .WillOnce(Invoke([this, registered](const std::string& action, Event::Dispatcher&, + OverloadActionCb callback) { + EXPECT_EQ(OverloadActionNames::get().Shutdown, action); + action_cb_ = callback; + return registered; + })); + } + + // Creates the mock timer that the next createTimer call on the server dispatcher returns. + Event::MockTimer* expectTimer() { + return new testing::NiceMock(&server_.dispatcher_); + } + + uint64_t shutdownCount() { return stats_.counter(ShutdownCountStat).value(); } + + Envoy::Stats::TestUtil::TestStore stats_; + testing::NiceMock server_; + OverloadActionCb action_cb_; +}; + +TEST_F(OverloadShutdownTest, DoesNothingWhenActionIsNotConfigured) { + EXPECT_CALL(server_.overload_manager_, getShutdownConfig()).WillRepeatedly(Return(std::nullopt)); + EXPECT_CALL(server_.overload_manager_, registerForAction(_, _, _)).Times(0); + EXPECT_CALL(server_.dispatcher_, createTimer_(_)).Times(0); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); +} + +TEST_F(OverloadShutdownTest, DoesNotCreateTimerWhenRegistrationFails) { + config_.mutable_saturation_duration()->set_seconds(60); + expectConfigured(); + expectRegistration(false); + EXPECT_CALL(server_.dispatcher_, createTimer_(_)).Times(0); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); +} + +TEST_F(OverloadShutdownTest, ShutsDownAfterSustainedSaturation) { + config_.mutable_saturation_duration()->set_seconds(60); + expectConfigured(); + expectRegistration(true); + Event::MockTimer* saturation_timer = expectTimer(); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); + + EXPECT_CALL(*saturation_timer, enableTimer(std::chrono::milliseconds(60000), _)); + action_cb_(OverloadActionState::saturated()); + + EXPECT_CALL(server_, failHealthcheck(true)); + EXPECT_CALL(server_, drainListeners(_)); + EXPECT_CALL(server_, shutdown()).Times(0); + saturation_timer->invokeCallback(); + EXPECT_EQ(1, shutdownCount()); + EXPECT_EQ(Network::DrainDirection::All, server_.drain_manager_.drain_direction_); + + EXPECT_CALL(server_, shutdown()); + server_.drain_manager_.drain_sequence_completion_(); +} + +TEST_F(OverloadShutdownTest, RecoveryCancelsPendingShutdown) { + config_.mutable_saturation_duration()->set_seconds(60); + expectConfigured(); + expectRegistration(true); + Event::MockTimer* saturation_timer = expectTimer(); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); + + EXPECT_CALL(*saturation_timer, enableTimer(std::chrono::milliseconds(60000), _)); + action_cb_(OverloadActionState::saturated()); + + EXPECT_CALL(*saturation_timer, disableTimer()); + EXPECT_CALL(server_, shutdown()).Times(0); + action_cb_(OverloadActionState::inactive()); + EXPECT_EQ(0, shutdownCount()); +} + +TEST_F(OverloadShutdownTest, ScalingBelowSaturationDoesNotStartCountdown) { + config_.mutable_saturation_duration()->set_seconds(60); + expectConfigured(); + expectRegistration(true); + Event::MockTimer* saturation_timer = expectTimer(); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); + + EXPECT_CALL(*saturation_timer, enableTimer(_, _)).Times(0); + action_cb_(OverloadActionState(UnitFloat(0.9))); +} + +TEST_F(OverloadShutdownTest, RepeatedSaturationDoesNotRestartCountdown) { + config_.mutable_saturation_duration()->set_seconds(60); + expectConfigured(); + expectRegistration(true); + Event::MockTimer* saturation_timer = expectTimer(); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); + + EXPECT_CALL(*saturation_timer, enableTimer(std::chrono::milliseconds(60000), _)); + action_cb_(OverloadActionState::saturated()); + action_cb_(OverloadActionState::saturated()); +} + +TEST_F(OverloadShutdownTest, JitterExtendsCountdown) { + config_.mutable_saturation_duration()->set_seconds(60); + config_.mutable_max_jitter()->set_seconds(30); + expectConfigured(); + expectRegistration(true); + Event::MockTimer* saturation_timer = expectTimer(); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); + + EXPECT_CALL(server_.api_.random_, random()).WillOnce(Return(12345)); + EXPECT_CALL(*saturation_timer, enableTimer(std::chrono::milliseconds(60000 + 12345 % 30001), _)); + action_cb_(OverloadActionState::saturated()); +} + +TEST_F(OverloadShutdownTest, RecoveryDuringDrainDoesNotCancelShutdown) { + config_.mutable_saturation_duration()->set_seconds(60); + expectConfigured(); + expectRegistration(true); + Event::MockTimer* saturation_timer = expectTimer(); + + OverloadShutdown shutdown(server_, server_.overload_manager_, *stats_.rootScope()); + action_cb_(OverloadActionState::saturated()); + saturation_timer->invokeCallback(); + + EXPECT_CALL(*saturation_timer, disableTimer()).Times(0); + action_cb_(OverloadActionState::inactive()); + + EXPECT_CALL(server_, shutdown()); + server_.drain_manager_.drain_sequence_completion_(); +} + +} // namespace +} // namespace Server +} // namespace Envoy