Skip to content

Add experimental prepared startup for Agent rollouts #54032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
AliDatadog wants to merge 9 commits into
base: main
Choose a base branch
from
Draft

Add experimental prepared startup for Agent rollouts #54032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9208f0b
Add experimental prepared Agent startup
AliDatadog Jul 22, 2026
8cf90e3
Refresh Agent component index
AliDatadog Jul 23, 2026
a565198
Harden experimental Agent rollout startup
AliDatadog Jul 23, 2026
292d1c1
Refresh Agent Bazel dependencies
AliDatadog Jul 23, 2026
00d47ac
Fix Windows prepared rollout test wiring
AliDatadog Jul 23, 2026
839ba53
Use kubelet siblings for prepared Agent rollout
AliDatadog Jul 24, 2026
1cc57a9
Gate surged Agent containers before initialization
AliDatadog Jul 28, 2026
6256379
Fix Agent lifecycle race tests
AliDatadog Jul 28, 2026
2c9af7f
Remove obsolete rollout lifecycle hooks
AliDatadog Jul 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion cmd/agent/subcommands/run/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ go_library(
"//comp/collector/collector/impl",
"//comp/connectivitychecker/fx",
"//comp/core",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/agenttelemetry/def",
"//comp/core/agenttelemetry/fx",
"//comp/core/autodiscovery/def",
Expand Down Expand Up @@ -286,6 +288,7 @@ go_library(
"//comp/trace/etwtracer/def",
"//comp/trace/etwtracer/fx",
"//pkg/config/model",
"//pkg/config/setup",
"//pkg/util/winutil",
],
"//conditions:default": [],
Expand All @@ -305,5 +308,12 @@ dd_agent_go_test(
"//comp/core/pid/impl",
"//pkg/util/fxutil",
"@com_github_stretchr_testify//require",
],
] + select({
"@rules_go//go/platform:windows": [
"//comp/core/config",
"//pkg/config/setup",
"@org_uber_go_fx//:fx",
],
"//conditions:default": [],
}),
)
7 changes: 5 additions & 2 deletions cmd/agent/subcommands/run/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ import (
collectorimpl "github.com/DataDog/datadog-agent/comp/collector/collector/impl"
connectivitycheckerfx "github.com/DataDog/datadog-agent/comp/connectivitychecker/fx"
"github.com/DataDog/datadog-agent/comp/core"
agentlifecycle "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/def"
agentlifecyclefx "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/fx"
autodiscovery "github.com/DataDog/datadog-agent/comp/core/autodiscovery/def"
adfx "github.com/DataDog/datadog-agent/comp/core/autodiscovery/fx"
"github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers"
Expand Down Expand Up @@ -235,7 +237,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
config.WithExtraConfFiles(cliParams.ExtraConfFilePath),
config.WithFleetPoliciesDirPath(cliParams.FleetPoliciesDirPath),
}
return fxutil.OneShot(run,
return fxutil.OneShotWithStartupGate[agentlifecycle.Component](run,

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 23, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Gate the Windows service startup path too

This adds the rollout gate only to the Cobra agent run path, but the packaged Windows service starts through StartAgentWithDefaults in command_windows.go, which still builds the app with plain fxutil.OneShot and no agentlifecyclefx.Module. With rollout enabled on Windows, the core Agent service bypasses the lock and never publishes prepared/active state, so service rollouts can run ungated or report misleading lifecycle state; wire the service path as well or explicitly disable the setting there.

AGENTS.md reference: AGENTS.md:L177-L178

Useful? React with 👍 / 👎.

fx.Invoke(func(_ log.Component) {
ddruntime.SetMaxProcs()
}),
Expand All @@ -245,6 +247,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
LogParams: log.ForDaemon(command.LoggerName, "log_file", defaultpaths.GetDefaultLogFile()),
}),
fx.Supply(pidimpl.NewParams(cliParams.pidfilePath)),
fx.Supply(agentlifecycle.Params{ComponentName: "agent"}),
agentlifecyclefx.Module(),
logging.EnableFxLoggingOnDebug[log.Component](),
fxinstrumentation.Module(),
getSharedFxOption(),
Expand Down Expand Up @@ -392,7 +396,6 @@ func run(log log.Component,
); err != nil {
return err
}

agentStarted := tlm.NewCounter(
"runtime",
"started",
Expand Down
10 changes: 10 additions & 0 deletions cmd/agent/subcommands/run/command_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ package run

import (
"context"
"errors"
_ "expvar" // Blank import used because this isn't directly used in this file
_ "net/http/pprof" // Blank import used because this isn't directly used in this file

Expand Down Expand Up @@ -84,6 +85,7 @@ import (
rcclient "github.com/DataDog/datadog-agent/comp/remote-config/rcclient/def"
snmpscanmanager "github.com/DataDog/datadog-agent/comp/snmpscanmanager/def"
softwareinventoryfx "github.com/DataDog/datadog-agent/comp/softwareinventory/fx"
configsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/serializer"
"github.com/DataDog/datadog-agent/pkg/util/defaultpaths"
"github.com/DataDog/datadog-agent/pkg/util/fxutil"
Expand Down Expand Up @@ -210,6 +212,7 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error
SysprobeConfigParams: sysprobeconfigimpl.NewParams(),
LogParams: log.ForDaemon(command.LoggerName, "log_file", defaultpaths.GetDefaultLogFile()),
}),
fx.Invoke(validateWindowsPreparedRollout),
getSharedFxOption(),
getPlatformModules(),
)
Expand All @@ -228,6 +231,13 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error
return errChan, nil
}

func validateWindowsPreparedRollout(config config.Component) error {
if config.GetBool(configsetup.ExperimentalNodeAgentRolloutEnabled) {
return errors.New("experimental node Agent rollout is not supported by the Windows service")
}
return nil
}

// Re-register the ctrl handler because Go uses SetConsoleCtrlHandler and Python uses posix signal calls.
// This is only needed when using the embedded Python module.
// When Python imports the signal module, it overrides the ctrl handler set by Go and installs the Windows posix signal handler.
Expand Down
22 changes: 22 additions & 0 deletions cmd/agent/subcommands/run/command_windows_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@ import (
"context"
"testing"

coreconfig "github.com/DataDog/datadog-agent/comp/core/config"
configsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/util/fxutil"
"github.com/stretchr/testify/require"
"go.uber.org/fx"
)

func TestStartAgentWithDefaults(t *testing.T) {
Expand All @@ -23,3 +26,22 @@ func TestStartAgentWithDefaults(t *testing.T) {
require.NoError(t, err)
})
}

func TestWindowsServiceRejectsPreparedRolloutBeforeStart(t *testing.T) {
cfg := coreconfig.NewMockWithOverrides(t, map[string]interface{}{
configsetup.ExperimentalNodeAgentRolloutEnabled: true,
})
started := false
app := fx.New(
fx.Provide(func() coreconfig.Component { return cfg }),
fx.Invoke(validateWindowsPreparedRollout),
fx.Invoke(func(lifecycle fx.Lifecycle) {
lifecycle.Append(fx.Hook{OnStart: func(context.Context) error {
started = true
return nil
}})
}),
)
require.ErrorContains(t, app.Err(), "not supported")
require.False(t, started)
}
4 changes: 4 additions & 0 deletions cmd/host-profiler/subcommands/run/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ go_library(
"//cmd/agent/command",
"//cmd/host-profiler/globalparams",
"//comp/core",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/config",
"//comp/core/configsync/def",
"//comp/core/configsync/fx",
Expand Down Expand Up @@ -46,6 +48,8 @@ go_library(
"//cmd/agent/command",
"//cmd/host-profiler/globalparams",
"//comp/core",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/config",
"//comp/core/configsync/def",
"//comp/core/configsync/fx",
Expand Down
6 changes: 5 additions & 1 deletion cmd/host-profiler/subcommands/run/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"github.com/DataDog/datadog-agent/cmd/agent/command"
"github.com/DataDog/datadog-agent/cmd/host-profiler/globalparams"
"github.com/DataDog/datadog-agent/comp/core"
agentlifecycle "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/def"
agentlifecyclefx "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/fx"
"github.com/DataDog/datadog-agent/comp/core/config"
configsync "github.com/DataDog/datadog-agent/comp/core/configsync/def"
configsyncfx "github.com/DataDog/datadog-agent/comp/core/configsync/fx"
Expand Down Expand Up @@ -90,6 +92,8 @@ func runHostProfilerCommand(ctx context.Context, cliParams *cliParams) error {
}

var opts = []fx.Option{
fx.Supply(agentlifecycle.Params{ComponentName: "host-profiler"}),
agentlifecyclefx.Module(),
hostprofiler.Bundle(collectorimpl.NewParams(cliParams.GlobalParams.ConfigURI(), cliParams.GoRuntimeMetrics)),
logging.DefaultFxLoggingOption(),
}
Expand Down Expand Up @@ -120,7 +124,7 @@ func runHostProfilerCommand(ctx context.Context, cliParams *cliParams) error {
)
}

return fxutil.OneShot(run, opts...)
return fxutil.OneShotWithStartupGate[agentlifecycle.Component](run, opts...)
}

func run(collector collector.Component) error {
Expand Down
2 changes: 2 additions & 0 deletions cmd/otel-agent/subcommands/run/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ go_library(
deps = [
"//cmd/otel-agent/config",
"//cmd/otel-agent/subcommands",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/agenttelemetry/fx",
"//comp/core/config",
"//comp/core/configsync/def",
Expand Down
15 changes: 12 additions & 3 deletions cmd/otel-agent/subcommands/run/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (

agentConfig "github.com/DataDog/datadog-agent/cmd/otel-agent/config"
"github.com/DataDog/datadog-agent/cmd/otel-agent/subcommands"
agentlifecycle "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/def"
agentlifecyclefx "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/fx"
agenttelemetryfx "github.com/DataDog/datadog-agent/comp/core/agenttelemetry/fx"
coreconfig "github.com/DataDog/datadog-agent/comp/core/config"
configsync "github.com/DataDog/datadog-agent/comp/core/configsync/def"
Expand Down Expand Up @@ -132,11 +134,16 @@ func runOTelAgentCommand(ctx context.Context, params *cliParams, opts ...fx.Opti
fmt.Println("*** OpenTelemetry Collector is not enabled, exiting application ***. Set the config option `otelcollector.enabled` or the environment variable `DD_OTELCOLLECTOR_ENABLED` at true to enable it.")
return nil
}
gateOptions := []fx.Option{
fx.Supply(agentlifecycle.Params{ComponentName: "otel-agent"}),
agentlifecyclefx.Module(),
}

uris := buildConfigURIs(params)

if err == agentConfig.ErrNoDDExporter {
return fxutil.Run(
return fxutil.RunWithStartupGate[agentlifecycle.Component](
fx.Options(gateOptions...),
fx.Supply(uris),
fx.Provide(func() coreconfig.Component {
return acfg
Expand Down Expand Up @@ -170,12 +177,14 @@ func runOTelAgentCommand(ctx context.Context, params *cliParams, opts ...fx.Opti
}

if acfg.GetBool("otel_standalone") {
return fxutil.Run(
return fxutil.RunWithStartupGate[agentlifecycle.Component](
fx.Options(gateOptions...),
commonAgentFxOptions(ctx, params, acfg, uris, opts...),
standaloneAgentFxOptions(params),
)
}
return fxutil.Run(
return fxutil.RunWithStartupGate[agentlifecycle.Component](
fx.Options(gateOptions...),
commonAgentFxOptions(ctx, params, acfg, uris, opts...),
connectedAgentFxOptions(params),
)
Expand Down
2 changes: 2 additions & 0 deletions cmd/privateactionrunner/subcommands/run/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ go_library(
deps = [
"//cmd/privateactionrunner/command",
"//comp/core",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/config",
"//comp/core/hostname/remotehostnameimpl",
"//comp/core/ipc/fx",
Expand Down
6 changes: 5 additions & 1 deletion cmd/privateactionrunner/subcommands/run/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (

"github.com/DataDog/datadog-agent/cmd/privateactionrunner/command"
"github.com/DataDog/datadog-agent/comp/core"
agentlifecycle "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/def"
agentlifecyclefx "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/fx"
"github.com/DataDog/datadog-agent/comp/core/config"
ipcfx "github.com/DataDog/datadog-agent/comp/core/ipc/fx"
log "github.com/DataDog/datadog-agent/comp/core/log/def"
Expand Down Expand Up @@ -103,6 +105,8 @@ func runPrivateActionRunner(ctx context.Context, confPath string, extraConfFiles
ConfigParams: config.NewAgentParams(confPath, config.WithExtraConfFiles(extraConfFiles)),
LogParams: log.ForDaemon(command.LoggerName, pkgconfigsetup.PARLogFile, defaultpaths.GetDefaultPrivateActionRunnerLogFile())}),
core.Bundle(core.WithSecrets()),
fx.Supply(agentlifecycle.Params{ComponentName: "private-action-runner"}),
agentlifecyclefx.Module(),
fx.Provide(func(c config.Component) settings.Params {
return settings.Params{
Settings: map[string]settings.RuntimeSetting{
Expand All @@ -127,7 +131,7 @@ func runPrivateActionRunner(ctx context.Context, confPath string, extraConfFiles
privateactionrunnerfx.Module(),
}

err := fxutil.Run(fxOptions...)
err := fxutil.RunWithStartupGate[agentlifecycle.Component](fxOptions...)
if errors.Is(err, privateactionrunner.ErrNotEnabled) {
return nil
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/process-agent/command/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ go_library(
"//comp/agent/autoexit/def",
"//comp/agent/autoexit/fx",
"//comp/core",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/config",
"//comp/core/configstreamconsumer/def",
"//comp/core/configstreamconsumer/fx",
Expand Down
16 changes: 15 additions & 1 deletion cmd/process-agent/command/main_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
autoexit "github.com/DataDog/datadog-agent/comp/agent/autoexit/def"
autoexitfx "github.com/DataDog/datadog-agent/comp/agent/autoexit/fx"
"github.com/DataDog/datadog-agent/comp/core"
agentlifecycle "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/def"
agentlifecyclefx "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/fx"
"github.com/DataDog/datadog-agent/comp/core/config"
configstreamconsumer "github.com/DataDog/datadog-agent/comp/core/configstreamconsumer/def"
configstreamconsumerfx "github.com/DataDog/datadog-agent/comp/core/configstreamconsumer/fx"
Expand Down Expand Up @@ -108,7 +110,9 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error {
RCClient rcclient.Component
WorkloadMeta workloadmeta.Component
}
var startupGate fxutil.StartupGate
app := fx.New(
fxutil.StartupGateOption[agentlifecycle.Component](ctx, &startupGate),
fx.Supply(
core.BundleParams{
SysprobeConfigParams: sysprobeconfigimpl.NewParams(
Expand All @@ -119,6 +123,8 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error {
LogParams: DaemonLogParams,
},
),
fx.Supply(agentlifecycle.Params{ComponentName: "process-agent"}),
agentlifecyclefx.Module(),
fx.Supply(
status.Params{
PythonVersionGetFunc: python.GetPythonVersion,
Expand Down Expand Up @@ -233,6 +239,15 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error {
return nil
}),
)
if err := app.Err(); err != nil && !errors.Is(err, errAgentDisabled) {
if startupGate != nil {
return errors.Join(fxutil.UnwrapIfErrArgumentsFailed(err), startupGate.Close())
}
return fxutil.UnwrapIfErrArgumentsFailed(err)
}
if startupGate != nil {
defer startupGate.Close()
}

err := app.Start(ctx)
if err != nil {
Expand All @@ -252,7 +267,6 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error {
return err
}
}

// Wait for exit signal
select {
case <-exitSignal:
Expand Down
2 changes: 2 additions & 0 deletions cmd/system-probe/subcommands/run/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ go_library(
"//cmd/system-probe/common",
"//comp/agent/autoexit/def",
"//comp/agent/autoexit/fx",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/config",
"//comp/core/configstreamconsumer/def",
"//comp/core/configstreamconsumer/fx",
Expand Down
6 changes: 5 additions & 1 deletion cmd/system-probe/subcommands/run/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
"github.com/DataDog/datadog-agent/cmd/system-probe/common"
autoexit "github.com/DataDog/datadog-agent/comp/agent/autoexit/def"
autoexitfx "github.com/DataDog/datadog-agent/comp/agent/autoexit/fx"
agentlifecycle "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/def"
agentlifecyclefx "github.com/DataDog/datadog-agent/comp/core/agentlifecycle/fx"
"github.com/DataDog/datadog-agent/comp/core/config"
configstreamconsumer "github.com/DataDog/datadog-agent/comp/core/configstreamconsumer/def"
configstreamconsumerfx "github.com/DataDog/datadog-agent/comp/core/configstreamconsumer/fx"
Expand Down Expand Up @@ -122,6 +124,8 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
Long: `Runs the system-probe in the foreground`,
RunE: func(_ *cobra.Command, _ []string) error {
opts := []fx.Option{
fx.Supply(agentlifecycle.Params{ComponentName: "system-probe"}),
agentlifecyclefx.Module(),
fx.Invoke(func(_ log.Component) {
ddruntime.SetMaxProcs()
}),
Expand All @@ -135,7 +139,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
configstreamconsumerfx.Module(),
getSharedFxOption(),
}
return fxutil.OneShot(run, opts...)
return fxutil.OneShotWithStartupGate[agentlifecycle.Component](run, opts...)
},
}
runCmd.Flags().StringVarP(&cliParams.pidfilePath, "pid", "p", "", "path to the pidfile")
Expand Down
2 changes: 2 additions & 0 deletions cmd/trace-agent/subcommands/run/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ go_library(
"//cmd/trace-agent/subcommands",
"//comp/agent/autoexit/def",
"//comp/agent/autoexit/fx",
"//comp/core/agentlifecycle/def",
"//comp/core/agentlifecycle/fx",
"//comp/core/agenttelemetry/def",
"//comp/core/agenttelemetry/fx",
"//comp/core/config",
Expand Down
Loading
Loading