Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/obol/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func bootstrapCommand(cfg *config.Config) *cli.Command {

// Step 1: Initialize stack
backendName := stack.DetectExistingBackend(cfg)
if err := stack.Init(cfg, u, false, backendName); err != nil {
if err := stack.Init(cfg, u, false, backendName, false); err != nil {
if !strings.Contains(err.Error(), "already exists") {
return fmt.Errorf("bootstrap init failed: %w", err)
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/obol/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,14 @@ GLOBAL OPTIONS:{{template "visibleFlagTemplate" .}}{{end}}
Usage: "Cluster backend: k3d (Docker-based) or k3s (bare-metal)",
Sources: cli.EnvVars("OBOL_BACKEND"),
},
&cli.BoolFlag{
Name: "yes",
Aliases: []string{"y"},
Usage: "Skip the live-services confirmation prompt when --force switches backends (required in non-interactive shells when offers are running)",
},
},
Action: func(ctx context.Context, cmd *cli.Command) error {
return stack.Init(cfg, getUI(cmd), cmd.Bool("force"), cmd.String("backend"))
return stack.Init(cfg, getUI(cmd), cmd.Bool("force"), cmd.String("backend"), cmd.Bool("yes"))
},
},
{
Expand Down
53 changes: 40 additions & 13 deletions internal/stack/stack.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@ const (
stackIDFile = ".stack-id"
)

// Init initializes the stack configuration
func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error {
// Init initializes the stack configuration. skipConfirm (--yes) bypasses the
// live-services confirmation prompt when --force switches backends and would
// otherwise destroy a cluster serving live traffic (mirrors Down/Purge).
func Init(cfg *config.Config, u *ui.UI, force bool, backendName string, skipConfirm bool) error {
// Check if any stack config already exists (legacy k3d.yaml included).
stackIDPath := filepath.Join(cfg.ConfigDir, stackIDFile)
hasExistingConfig := false
Expand Down Expand Up @@ -76,13 +78,13 @@ func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error {
backendName = BackendK3d
}

// If switching backends, destroy the old one first to prevent
// orphaned clusters (e.g., k3d containers still running after
// switching to k3s, or k3s process still alive after switching to k3d).
if hasExistingConfig && force {
destroyOldBackendIfSwitching(cfg, u, backendName, stackID)
}

// Validate the new backend BEFORE destroying anything. Previously the old
// backend was torn down first and NewBackend/Prerequisites only checked
// afterward — an invalid or unavailable target backend left the old
// cluster destroyed with nothing running in its place (confirmed
// Canary402 full-surface audit finding). Resolve and validate the new
// backend first so a failure here aborts Init without having touched the
// old cluster.
backend, err := NewBackend(backendName)
if err != nil {
return err
Expand All @@ -97,6 +99,18 @@ func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error {
return fmt.Errorf("prerequisites check failed: %w", err)
}

// If switching backends, destroy the old one first to prevent
// orphaned clusters (e.g., k3d containers still running after
// switching to k3s, or k3s process still alive after switching to k3d).
// Gated on ConfirmRunningServicesLoss (same safety bar as Down/Purge) so
// this destructive step can't run unconfirmed against a cluster serving
// live traffic.
if hasExistingConfig && force {
if err := destroyOldBackendIfSwitching(cfg, u, backendName, stackID, skipConfirm); err != nil {
return err
}
}

// Generate backend-specific config
if err := backend.Init(cfg, u, stackID); err != nil {
return err
Expand All @@ -123,15 +137,27 @@ func Init(cfg *config.Config, u *ui.UI, force bool, backendName string) error {
}

// destroyOldBackendIfSwitching checks if the backend is changing and tears down
// the old one to prevent orphaned clusters running side by side.
func destroyOldBackendIfSwitching(cfg *config.Config, u *ui.UI, newBackend, stackID string) {
// the old one to prevent orphaned clusters running side by side. The destroy
// is gated on ConfirmRunningServicesLoss — the same safety bar `obol stack
// down`/`purge` use — since it's just as capable of destroying a cluster that
// is currently serving paid traffic.
func destroyOldBackendIfSwitching(cfg *config.Config, u *ui.UI, newBackend, stackID string, skipConfirm bool) error {
oldBackend, err := LoadBackend(cfg)
if err != nil {
return
return nil
}

if oldBackend.Name() == newBackend {
return // same backend, nothing to clean up
return nil // same backend, nothing to clean up
}

proceed, err := ConfirmRunningServicesLoss(cfg, u, "stack init --force (backend switch)", skipConfirm)
if err != nil {
return err
}
if !proceed {
u.Info("Aborted.")
return errSafetyAborted
}

u.Warnf("Switching backend from %s to %s — destroying old cluster", oldBackend.Name(), newBackend)
Expand All @@ -145,6 +171,7 @@ func destroyOldBackendIfSwitching(cfg *config.Config, u *ui.UI, newBackend, stac

// Clean up stale config files from the old backend
cleanupStaleBackendConfigs(cfg, oldBackend.Name())
return nil
}

// cleanupStaleBackendConfigs removes config files belonging to the old backend
Expand Down
85 changes: 81 additions & 4 deletions internal/stack/stack_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,9 @@ func TestDestroyOldBackendIfSwitching_CleansStaleConfigs(t *testing.T) {

// Switch to k3s — k3d config should be cleaned up
// (Destroy will fail because no real cluster, but cleanup should still work)
destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3s, "test-id")
if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3s, "test-id", false); err != nil {
t.Fatalf("destroyOldBackendIfSwitching: %v", err)
}

if _, err := os.Stat(k3dPath); !os.IsNotExist(err) {
t.Error("k3d.yaml should be removed when switching to k3s")
Expand All @@ -325,7 +327,9 @@ func TestDestroyOldBackendIfSwitching_NoopSameBackend(t *testing.T) {
os.WriteFile(k3dPath, []byte("k3d config"), 0o644)

// Same backend — nothing should be cleaned up
destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id")
if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id", false); err != nil {
t.Fatalf("destroyOldBackendIfSwitching: %v", err)
}

if _, err := os.Stat(k3dPath); os.IsNotExist(err) {
t.Error("k3d.yaml should NOT be removed when re-initing same backend")
Expand All @@ -347,7 +351,9 @@ func TestDestroyOldBackendIfSwitching_K3sToK3d(t *testing.T) {
}

// Switch to k3d — k3s files should be cleaned up
destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id")
if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id", false); err != nil {
t.Fatalf("destroyOldBackendIfSwitching: %v", err)
}

for _, f := range []string{k3sConfigFile, k3sPidFile, k3sLogFile} {
if _, err := os.Stat(filepath.Join(tmpDir, f)); !os.IsNotExist(err) {
Expand All @@ -367,7 +373,78 @@ func TestDestroyOldBackendIfSwitching_NoBackendFile(t *testing.T) {
}

// Should not panic or error
destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id")
if err := destroyOldBackendIfSwitching(cfg, ui.New(false), BackendK3d, "test-id", false); err != nil {
t.Fatalf("destroyOldBackendIfSwitching: %v", err)
}
}

// TestDestroyOldBackendIfSwitching_LiveServicesRefusesNonInteractiveWithoutYes
// is the regression test for the Canary402 `stack init --force --backend <X>`
// finding: destroying the old backend's cluster is exactly as dangerous as
// `obol stack down`/`purge`, so it must be gated on the same
// ConfirmRunningServicesLoss safety bar instead of running unconditionally.
func TestDestroyOldBackendIfSwitching_LiveServicesRefusesNonInteractiveWithoutYes(t *testing.T) {
tmpDir := t.TempDir()
cfg := &config.Config{
ConfigDir: tmpDir,
DataDir: filepath.Join(tmpDir, "data"),
BinDir: filepath.Join(tmpDir, "bin"),
StateDir: filepath.Join(tmpDir, "state"),
}

SaveBackend(cfg, BackendK3d)
k3dPath := filepath.Join(tmpDir, k3dConfigFile)
os.WriteFile(k3dPath, []byte("k3d config"), 0o644)

// A live sell-inference gateway makes this stack "serving traffic".
writeGatewayPID(t, cfg, "aeon", os.Getpid())

var buf bytes.Buffer
u := ui.NewForTest(&buf, &buf) // isTTY defaults false, no --yes

err := destroyOldBackendIfSwitching(cfg, u, BackendK3s, "test-id", false)
if err == nil {
t.Fatal("expected error when switching backends non-interactively with live services and no --yes")
}
if !strings.Contains(err.Error(), "--yes") {
t.Errorf("error should mention --yes (operator escape hatch): %v", err)
}

// Destroy (and the cleanup that follows it) must not have run: the old
// backend's stale config file must survive the refused call.
if _, statErr := os.Stat(k3dPath); statErr != nil {
t.Errorf("k3d.yaml should NOT be removed when the safety gate refuses: %v", statErr)
}
}

// TestDestroyOldBackendIfSwitching_SkipConfirmStillDestroys ensures --yes
// keeps working as the non-interactive escape hatch after the safety gate
// was added, mirroring Down/Purge's --yes behavior.
func TestDestroyOldBackendIfSwitching_SkipConfirmStillDestroys(t *testing.T) {
tmpDir := t.TempDir()
cfg := &config.Config{
ConfigDir: tmpDir,
DataDir: filepath.Join(tmpDir, "data"),
BinDir: filepath.Join(tmpDir, "bin"),
StateDir: filepath.Join(tmpDir, "state"),
}

SaveBackend(cfg, BackendK3d)
k3dPath := filepath.Join(tmpDir, k3dConfigFile)
os.WriteFile(k3dPath, []byte("k3d config"), 0o644)

writeGatewayPID(t, cfg, "aeon", os.Getpid())

var buf bytes.Buffer
u := ui.NewForTest(&buf, &buf)

if err := destroyOldBackendIfSwitching(cfg, u, BackendK3s, "test-id", true); err != nil {
t.Fatalf("destroyOldBackendIfSwitching with skipConfirm: %v", err)
}

if _, statErr := os.Stat(k3dPath); !os.IsNotExist(statErr) {
t.Error("k3d.yaml should be removed when --yes overrides the confirmation")
}
}

func TestOllamaHostIPForBackend_K3s(t *testing.T) {
Expand Down