From 75b3e68cb2e7b4e583c9e5d143139bc05dd2a502 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Thu, 16 Jul 2026 17:05:57 +0400 Subject: [PATCH] fix(stack): gate backend-switch destroy in init --force on live-services confirm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: Init() destroyed the old backend's cluster (destroyOldBackendIfSwitching) unconditionally on --force, then only afterward validated the new backend via NewBackend/Prerequisites — either of which can fail, leaving the old cluster destroyed with nothing running. Unlike Down()/Purge(), this path had no ConfirmRunningServicesLoss gate, so a live-traffic-serving stack could be silently wiped by 'stack init --force --backend '. Fix: (1) reorder Init() so NewBackend + backend.Prerequisites run before destroyOldBackendIfSwitching, so an invalid/unavailable target backend aborts before anything is destroyed. (2) thread a skipConfirm (--yes) bool through Init and gate the destroy on ConfirmRunningServicesLoss, mirroring Down/Purge; non-interactive callers without --yes now fail closed instead of destroying. CLI wiring in cmd/obol/main.go adds --yes/-y to 'stack init', matching purge's flag. bootstrap.go's internal (force=false) call passes skipConfirm=false, a no-op since force=false never reaches the destroy path. Confirmed Canary402 full-surface audit finding. Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- cmd/obol/bootstrap.go | 2 +- cmd/obol/main.go | 7 ++- internal/stack/stack.go | 53 ++++++++++++++++------ internal/stack/stack_test.go | 85 ++++++++++++++++++++++++++++++++++-- 4 files changed, 128 insertions(+), 19 deletions(-) diff --git a/cmd/obol/bootstrap.go b/cmd/obol/bootstrap.go index 502df83b..1db87933 100644 --- a/cmd/obol/bootstrap.go +++ b/cmd/obol/bootstrap.go @@ -31,7 +31,7 @@ func bootstrapCommand(cfg *config.Config) *cli.Command { // Step 1: Initialize stack backendName := stack.DetectExistingBackend(cfg) - if err := stack.Init(cfg, u, false, backendName); err != nil { + if err := stack.Init(cfg, u, false, backendName, false); err != nil { if !strings.Contains(err.Error(), "already exists") { return fmt.Errorf("bootstrap init failed: %w", err) } diff --git a/cmd/obol/main.go b/cmd/obol/main.go index d82d2f69..3a3cd647 100644 --- a/cmd/obol/main.go +++ b/cmd/obol/main.go @@ -190,9 +190,14 @@ GLOBAL OPTIONS:{{template "visibleFlagTemplate" .}}{{end}} Usage: "Cluster backend: k3d (Docker-based) or k3s (bare-metal)", Sources: cli.EnvVars("OBOL_BACKEND"), }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Skip the live-services confirmation prompt when --force switches backends (required in non-interactive shells when offers are running)", + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { - return stack.Init(cfg, getUI(cmd), cmd.Bool("force"), cmd.String("backend")) + return stack.Init(cfg, getUI(cmd), cmd.Bool("force"), cmd.String("backend"), cmd.Bool("yes")) }, }, { diff --git a/internal/stack/stack.go b/internal/stack/stack.go index 9135aceb..0743ebac 100644 --- a/internal/stack/stack.go +++ b/internal/stack/stack.go @@ -42,8 +42,10 @@ const ( stackIDFile = ".stack-id" ) -// Init initializes the stack configuration -func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error { +// Init initializes the stack configuration. skipConfirm (--yes) bypasses the +// live-services confirmation prompt when --force switches backends and would +// otherwise destroy a cluster serving live traffic (mirrors Down/Purge). +func Init(cfg *config.Config, u *ui.UI, force bool, backendName string, skipConfirm bool) error { // Check if any stack config already exists (legacy k3d.yaml included). stackIDPath := filepath.Join(cfg.ConfigDir, stackIDFile) hasExistingConfig := false @@ -76,13 +78,13 @@ func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error { backendName = BackendK3d } - // If switching backends, destroy the old one first to prevent - // orphaned clusters (e.g., k3d containers still running after - // switching to k3s, or k3s process still alive after switching to k3d). - if hasExistingConfig && force { - destroyOldBackendIfSwitching(cfg, u, backendName, stackID) - } - + // Validate the new backend BEFORE destroying anything. Previously the old + // backend was torn down first and NewBackend/Prerequisites only checked + // afterward — an invalid or unavailable target backend left the old + // cluster destroyed with nothing running in its place (confirmed + // Canary402 full-surface audit finding). Resolve and validate the new + // backend first so a failure here aborts Init without having touched the + // old cluster. backend, err := NewBackend(backendName) if err != nil { return err @@ -97,6 +99,18 @@ func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error { return fmt.Errorf("prerequisites check failed: %w", err) } + // If switching backends, destroy the old one first to prevent + // orphaned clusters (e.g., k3d containers still running after + // switching to k3s, or k3s process still alive after switching to k3d). + // Gated on ConfirmRunningServicesLoss (same safety bar as Down/Purge) so + // this destructive step can't run unconfirmed against a cluster serving + // live traffic. + if hasExistingConfig && force { + if err := destroyOldBackendIfSwitching(cfg, u, backendName, stackID, skipConfirm); err != nil { + return err + } + } + // Generate backend-specific config if err := backend.Init(cfg, u, stackID); err != nil { return err @@ -123,15 +137,27 @@ func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error { } // destroyOldBackendIfSwitching checks if the backend is changing and tears down -// the old one to prevent orphaned clusters running side by side. -func destroyOldBackendIfSwitching(cfg *config.Config, u *ui.UI, newBackend, stackID string) { +// the old one to prevent orphaned clusters running side by side. The destroy +// is gated on ConfirmRunningServicesLoss — the same safety bar `obol stack +// down`/`purge` use — since it's just as capable of destroying a cluster that +// is currently serving paid traffic. +func destroyOldBackendIfSwitching(cfg *config.Config, u *ui.UI, newBackend, stackID string, skipConfirm bool) error { oldBackend, err := LoadBackend(cfg) if err != nil { - return + return nil } if oldBackend.Name() == newBackend { - return // same backend, nothing to clean up + return nil // same backend, nothing to clean up + } + + proceed, err := ConfirmRunningServicesLoss(cfg, u, "stack init --force (backend switch)", skipConfirm) + if err != nil { + return err + } + if !proceed { + u.Info("Aborted.") + return errSafetyAborted } u.Warnf("Switching backend from %s to %s — destroying old cluster", oldBackend.Name(), newBackend) @@ -145,6 +171,7 @@ func destroyOldBackendIfSwitching(cfg *config.Config, u *ui.UI, newBackend, stac // Clean up stale config files from the old backend cleanupStaleBackendConfigs(cfg, oldBackend.Name()) + return nil } // cleanupStaleBackendConfigs removes config files belonging to the old backend diff --git a/internal/stack/stack_test.go b/internal/stack/stack_test.go index 4244e16c..2d4866e8 100644 --- a/internal/stack/stack_test.go +++ b/internal/stack/stack_test.go @@ -304,7 +304,9 @@ func TestDestroyOldBackendIfSwitching_CleansStaleConfigs(t *testing.T) { // Switch to k3s — k3d config should be cleaned up // (Destroy will fail because no real cluster, but cleanup should still work) - destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3s, "test-id") + if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3s, "test-id", false); err != nil { + t.Fatalf("destroyOldBackendIfSwitching: %v", err) + } if _, err := os.Stat(k3dPath); !os.IsNotExist(err) { t.Error("k3d.yaml should be removed when switching to k3s") @@ -325,7 +327,9 @@ func TestDestroyOldBackendIfSwitching_NoopSameBackend(t *testing.T) { os.WriteFile(k3dPath, []byte("k3d config"), 0o644) // Same backend — nothing should be cleaned up - destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id") + if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id", false); err != nil { + t.Fatalf("destroyOldBackendIfSwitching: %v", err) + } if _, err := os.Stat(k3dPath); os.IsNotExist(err) { t.Error("k3d.yaml should NOT be removed when re-initing same backend") @@ -347,7 +351,9 @@ func TestDestroyOldBackendIfSwitching_K3sToK3d(t *testing.T) { } // Switch to k3d — k3s files should be cleaned up - destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id") + if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id", false); err != nil { + t.Fatalf("destroyOldBackendIfSwitching: %v", err) + } for _, f := range []string{k3sConfigFile, k3sPidFile, k3sLogFile} { if _, err := os.Stat(filepath.Join(tmpDir, f)); !os.IsNotExist(err) { @@ -367,7 +373,78 @@ func TestDestroyOldBackendIfSwitching_NoBackendFile(t *testing.T) { } // Should not panic or error - destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id") + if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id", false); err != nil { + t.Fatalf("destroyOldBackendIfSwitching: %v", err) + } +} + +// TestDestroyOldBackendIfSwitching_LiveServicesRefusesNonInteractiveWithoutYes +// is the regression test for the Canary402 `stack init --force --backend ` +// finding: destroying the old backend's cluster is exactly as dangerous as +// `obol stack down`/`purge`, so it must be gated on the same +// ConfirmRunningServicesLoss safety bar instead of running unconditionally. +func TestDestroyOldBackendIfSwitching_LiveServicesRefusesNonInteractiveWithoutYes(t *testing.T) { + tmpDir := t.TempDir() + cfg := &config.Config{ + ConfigDir: tmpDir, + DataDir: filepath.Join(tmpDir, "data"), + BinDir: filepath.Join(tmpDir, "bin"), + StateDir: filepath.Join(tmpDir, "state"), + } + + SaveBackend(cfg, BackendK3d) + k3dPath := filepath.Join(tmpDir, k3dConfigFile) + os.WriteFile(k3dPath, []byte("k3d config"), 0o644) + + // A live sell-inference gateway makes this stack "serving traffic". + writeGatewayPID(t, cfg, "aeon", os.Getpid()) + + var buf bytes.Buffer + u := ui.NewForTest(&buf, &buf) // isTTY defaults false, no --yes + + err := destroyOldBackendIfSwitching(cfg, u, BackendK3s, "test-id", false) + if err == nil { + t.Fatal("expected error when switching backends non-interactively with live services and no --yes") + } + if !strings.Contains(err.Error(), "--yes") { + t.Errorf("error should mention --yes (operator escape hatch): %v", err) + } + + // Destroy (and the cleanup that follows it) must not have run: the old + // backend's stale config file must survive the refused call. + if _, statErr := os.Stat(k3dPath); statErr != nil { + t.Errorf("k3d.yaml should NOT be removed when the safety gate refuses: %v", statErr) + } +} + +// TestDestroyOldBackendIfSwitching_SkipConfirmStillDestroys ensures --yes +// keeps working as the non-interactive escape hatch after the safety gate +// was added, mirroring Down/Purge's --yes behavior. +func TestDestroyOldBackendIfSwitching_SkipConfirmStillDestroys(t *testing.T) { + tmpDir := t.TempDir() + cfg := &config.Config{ + ConfigDir: tmpDir, + DataDir: filepath.Join(tmpDir, "data"), + BinDir: filepath.Join(tmpDir, "bin"), + StateDir: filepath.Join(tmpDir, "state"), + } + + SaveBackend(cfg, BackendK3d) + k3dPath := filepath.Join(tmpDir, k3dConfigFile) + os.WriteFile(k3dPath, []byte("k3d config"), 0o644) + + writeGatewayPID(t, cfg, "aeon", os.Getpid()) + + var buf bytes.Buffer + u := ui.NewForTest(&buf, &buf) + + if err := destroyOldBackendIfSwitching(cfg, u, BackendK3s, "test-id", true); err != nil { + t.Fatalf("destroyOldBackendIfSwitching with skipConfirm: %v", err) + } + + if _, statErr := os.Stat(k3dPath); !os.IsNotExist(statErr) { + t.Error("k3d.yaml should be removed when --yes overrides the confirmation") + } } func TestOllamaHostIPForBackend_K3s(t *testing.T) {