Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ func runSync(_ *cobra.Command, _ []string) error {
telemetry.TrackError("sync", err)
return err
}
// Surface plan-level errors before apply (#134). A plan that already
// knows e.g. "unsupported VCS type" should fail fast instead of
// re-discovering the same error per resource inside RunOnce.
if plan.HasErrors {
err := fmt.Errorf("plan has errors; refusing to apply (run `gaal sync --dry-run` for details)")
telemetry.TrackError("sync", err)
return &ExitCodeError{Code: 2, Cause: err}
}

start := time.Now()
if err := eng.RunOnce(ctx); err != nil {
Expand Down
46 changes: 40 additions & 6 deletions internal/engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,45 @@ func (e *Engine) RunOnce(ctx context.Context) error {
return nil
}

// RunService runs synchronisation in a loop until the context is cancelled.
// maxConsecutiveServiceFailures bounds the number of consecutive failed
// RunOnce iterations RunService tolerates before exiting. A misconfigured
// service (bad config, dead remote) used to retry forever, burning CPU
// and network indefinitely. After the cap, RunService returns the last
// error so a process supervisor (systemd, kubelet, …) can intervene. #134.
const maxConsecutiveServiceFailures = 5

// RunService runs synchronisation in a loop until the context is cancelled
// or maxConsecutiveServiceFailures is reached. A successful sync resets
// the failure counter; transient errors are absorbed.
func (e *Engine) RunService(ctx context.Context, interval time.Duration) error {
slog.Info("service mode started", "interval", interval)
slog.Info("service mode started", "interval", interval,
"failure_cap", maxConsecutiveServiceFailures)

consecutiveFailures := 0
var lastErr error

runIteration := func(label string) {
err := e.RunOnce(ctx)
if err != nil {
consecutiveFailures++
lastErr = err
slog.Error(label+" sync failed", "err", err,
"consecutive_failures", consecutiveFailures,
"failure_cap", maxConsecutiveServiceFailures)
return
}
if consecutiveFailures > 0 {
slog.Info("service: sync recovered", "after_failures", consecutiveFailures)
}
consecutiveFailures = 0
lastErr = nil
}

// Run immediately on startup.
if err := e.RunOnce(ctx); err != nil {
slog.Error("initial sync failed", "err", err)
runIteration("initial")
if consecutiveFailures >= maxConsecutiveServiceFailures {
return fmt.Errorf("service exiting after %d consecutive failures: %w",
consecutiveFailures, lastErr)
}

ticker := time.NewTicker(interval)
Expand All @@ -178,8 +210,10 @@ func (e *Engine) RunService(ctx context.Context, interval time.Duration) error {
return nil
case t := <-ticker.C:
slog.Info("periodic sync triggered", "time", t)
if err := e.RunOnce(ctx); err != nil {
slog.Error("periodic sync failed", "err", err)
runIteration("periodic")
if consecutiveFailures >= maxConsecutiveServiceFailures {
return fmt.Errorf("service exiting after %d consecutive failures: %w",
consecutiveFailures, lastErr)
}
}
}
Expand Down
Loading