Fix tier tactic health filter bypass for fast circuit breaker recovery

.design/priority-routing.md

@@ -58,8 +58,8 @@ Three properties fall out:
 The breaker is a three-state machine (`Closed → Open → HalfOpen`):
 
 - **Closed** — normal. `Allow()` returns true, failures are counted.
-- **Open** — too many consecutive failures (`FailureThreshold`, default 3). `Allow()` returns false. After `OpenDuration` (default 30 s) the next `Allow()` call lazily flips to HalfOpen.
-- **HalfOpen** — exactly one probe is permitted. Success → Closed, failure → Open with a fresh timer.
+- **Open** — too many consecutive failures (`FailureThreshold`, default 3). `Allow()` returns false. After `OpenDuration` (default 60 s) the next `Allow()` call lazily flips to HalfOpen.
+- **HalfOpen** — exactly one probe is permitted. Success → Closed, failure → Open with exponential backoff (60 s → 120 s → 240 s → 5 min cap). Success resets backoff to the base duration.
 
 Recovery requires **no separate scheduler**. Selection re-evaluates the tier list every request, and the breaker's lazy state transition admits one probe naturally. Active probing was considered and rejected for v1 — for hot rules it's redundant, and for cold rules there is no one to serve anyway.
 

internal/loadbalance/breaker.go

@@ -17,7 +17,8 @@ const (
 // Default circuit breaker tunables.
 const (
 	DefaultBreakerFailureThreshold = 3
-	DefaultBreakerOpenDuration     = 30 * time.Second
+	DefaultBreakerOpenDuration     = 60 * time.Second
+	DefaultBreakerMaxOpenDuration  = 5 * time.Minute
 )
 
 // Breaker is a simple three-state circuit breaker for a single service.
@@ -37,9 +38,11 @@ type Breaker struct {
 	consecFails      int
 	openedAt         time.Time
 	halfOpenInFlight bool
+	halfOpenFails    int // consecutive half-open probe failures for backoff
 
 	FailureThreshold int
 	OpenDuration     time.Duration
+	MaxOpenDuration  time.Duration
 }
 
 // NewBreaker creates a breaker with the supplied thresholds. Zero values
@@ -55,6 +58,7 @@ func NewBreaker(failureThreshold int, openDuration time.Duration) *Breaker {
 		state:            BreakerClosed,
 		FailureThreshold: failureThreshold,
 		OpenDuration:     openDuration,
+		MaxOpenDuration:  DefaultBreakerMaxOpenDuration,
 	}
 }
 
@@ -69,7 +73,7 @@ func (b *Breaker) Allow() bool {
 	case BreakerClosed:
 		return true
 	case BreakerOpen:
-		if time.Since(b.openedAt) >= b.OpenDuration {
+		if time.Since(b.openedAt) >= b.currentOpenDuration() {
 			b.state = BreakerHalfOpen
 			b.halfOpenInFlight = true
 			return true
@@ -91,18 +95,21 @@ func (b *Breaker) RecordSuccess() {
 	defer b.mu.Unlock()
 	b.state = BreakerClosed
 	b.consecFails = 0
+	b.halfOpenFails = 0
 	b.halfOpenInFlight = false
 }
 
 // RecordFailure increments failure tracking and trips the breaker when
-// the threshold is reached. A failure during HalfOpen immediately re-opens.
+// the threshold is reached. A failure during HalfOpen immediately re-opens
+// and increases the backoff for the next open window.
 func (b *Breaker) RecordFailure() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
 	if b.state == BreakerHalfOpen {
 		b.state = BreakerOpen
 		b.openedAt = time.Now()
+		b.halfOpenFails++
 		b.halfOpenInFlight = false
 		return
 	}
@@ -113,12 +120,26 @@ func (b *Breaker) RecordFailure() {
 	}
 }
 
+// currentOpenDuration returns the backoff-adjusted open duration.
+// Each consecutive half-open failure doubles the wait (e.g. 60s → 120s → 240s → cap).
+// Must be called with b.mu held.
+func (b *Breaker) currentOpenDuration() time.Duration {
+	d := b.OpenDuration
+	for i := 0; i < b.halfOpenFails; i++ {
+		d *= 2
+		if d >= b.MaxOpenDuration {
+			return b.MaxOpenDuration
+		}
+	}
+	return d
+}
+
 // State returns the current breaker state. Intended for introspection / UI.
 func (b *Breaker) State() BreakerState {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	// Apply the lazy Open→HalfOpen transition for read consistency.
-	if b.state == BreakerOpen && time.Since(b.openedAt) >= b.OpenDuration {
+	if b.state == BreakerOpen && time.Since(b.openedAt) >= b.currentOpenDuration() {
 		return BreakerHalfOpen
 	}
 	return b.state

internal/loadbalance/breaker_test.go

@@ -73,6 +73,90 @@ func TestBreakerSuccessResetsCounter(t *testing.T) {
 	}
 }
 
+// windForward moves a breaker's openedAt backwards in time so the open
+// window appears to have elapsed, without any real time.Sleep.
+func windForward(b *Breaker, d time.Duration) {
+	b.mu.Lock()
+	b.openedAt = b.openedAt.Add(-d)
+	b.mu.Unlock()
+}
+
+func TestBreakerExponentialBackoff(t *testing.T) {
+	base := 100 * time.Millisecond
+	b := NewBreaker(1, base)
+	b.MaxOpenDuration = 500 * time.Millisecond
+
+	b.RecordFailure() // → open
+
+	// 1st window: base (100 ms). Fast-forward past it.
+	windForward(b, base)
+	if !b.Allow() {
+		t.Fatal("should allow probe after 1st open window")
+	}
+	b.RecordFailure() // half-open → open, halfOpenFails=1
+
+	// 2nd window: 200 ms. Advance only 150 ms — too early.
+	windForward(b, 150*time.Millisecond)
+	if b.Allow() {
+		t.Fatal("should NOT allow probe yet (backoff doubled to 200 ms)")
+	}
+	// Advance the remaining 50 ms.
+	windForward(b, 50*time.Millisecond)
+	if !b.Allow() {
+		t.Fatal("should allow probe after 2nd backoff window")
+	}
+	b.RecordFailure() // halfOpenFails=2
+
+	// 3rd window: 400 ms. Advance only 300 ms — too early.
+	windForward(b, 300*time.Millisecond)
+	if b.Allow() {
+		t.Fatal("should NOT allow probe yet (backoff at 400 ms)")
+	}
+	windForward(b, 100*time.Millisecond)
+	if !b.Allow() {
+		t.Fatal("should allow probe after 3rd backoff window")
+	}
+	b.RecordFailure() // halfOpenFails=3, next would be 800 ms → capped at 500 ms
+
+	// 4th window: capped at 500 ms. Advance only 450 ms — too early.
+	windForward(b, 450*time.Millisecond)
+	if b.Allow() {
+		t.Fatal("should NOT allow probe yet (capped at 500 ms)")
+	}
+	windForward(b, 50*time.Millisecond)
+	if !b.Allow() {
+		t.Fatal("should allow probe after cap-limited window")
+	}
+}
+
+func TestBreakerBackoffResetsOnSuccess(t *testing.T) {
+	base := 100 * time.Millisecond
+	b := NewBreaker(1, base)
+	b.MaxOpenDuration = 5 * time.Second
+
+	// Build up backoff: trip → probe fail → trip → probe fail.
+	b.RecordFailure()
+	windForward(b, base)
+	b.Allow()
+	b.RecordFailure() // halfOpenFails=1, next window=200 ms
+
+	windForward(b, 200*time.Millisecond)
+	b.Allow()
+
+	// Probe succeeds → everything resets.
+	b.RecordSuccess()
+	if b.State() != BreakerClosed {
+		t.Fatal("should be closed after success")
+	}
+
+	// Trip again — backoff should be back to base, not 400 ms.
+	b.RecordFailure()
+	windForward(b, base)
+	if !b.Allow() {
+		t.Fatal("after success+retrip, should probe at base duration, not backed-off")
+	}
+}
+
 func TestBreakerStoreLazyCreation(t *testing.T) {
 	store := NewBreakerStore(2, time.Second)
 	b1 := store.Get("svc:a")

internal/loadbalance/health_monitor.go

@@ -317,6 +317,20 @@ func (hm *HealthMonitor) GetAllHealth() map[string]*ServiceHealth {
 	return result
 }
 
+// HasAuthError returns true if the service is currently marked with an auth
+// error (401/403). Auth errors are permanent until the key is fixed, so
+// callers that manage their own transient-failure recovery (e.g. circuit
+// breakers) can still exclude these.
+func (hm *HealthMonitor) HasAuthError(serviceID string) bool {
+	health := hm.getHealth(serviceID)
+	if health == nil {
+		return false
+	}
+	health.mutex.RLock()
+	defer health.mutex.RUnlock()
+	return health.AuthError
+}
+
 // ResetHealth manually resets a service's health to healthy
 func (hm *HealthMonitor) ResetHealth(serviceID string) {
 	hm.recoverService(serviceID)

internal/server/load_balancer.go

@@ -76,18 +76,33 @@ func (lb *LoadBalancer) SelectService(rule *typ.Rule) (*loadbalance.Service, err
 		return nil, fmt.Errorf("no active services for rule %s", rule.RequestModel)
 	}
 
+	// Tier tactic has its own circuit-breaker-based health management with
+	// fast recovery (30 s). The full HealthFilter would hide services for the
+	// monitor's recovery window (5 min), defeating the breaker's quick
+	// failover/recovery. However auth errors (401/403) are permanent — a
+	// revoked key never self-heals — so those must still be filtered out.
+	isTier := rule.LBTactic.Type == loadbalance.TacticTier
+
 	// Filter healthy services using health filter. When every active service is
 	// currently marked unhealthy (e.g. a transient 429 on a single-service rule,
 	// or all services inside the recovery window at once), fall back to the full
 	// active set instead of failing the whole rule. Trying an unhealthy upstream
 	// is strictly better than a hard "no service available": the service may have
 	// already recovered, and if it really is still failing the caller gets the
 	// real upstream error (e.g. 429) rather than a confusing routing error.
-	healthyServices := lb.healthFilter.Filter(activeServices)
-	if len(healthyServices) == 0 {
-		logrus.Warnf("[load_balancer] all %d active services for rule %s are unhealthy; "+
-			"falling back to active set", len(activeServices), rule.RequestModel)
-		healthyServices = activeServices
+	var healthyServices []*loadbalance.Service
+	if isTier {
+		healthyServices = lb.healthFilter.FilterAuthErrors(activeServices)
+		if len(healthyServices) == 0 {
+			healthyServices = activeServices
+		}
+	} else {
+		healthyServices = lb.healthFilter.Filter(activeServices)
+		if len(healthyServices) == 0 {
+			logrus.Warnf("[load_balancer] all %d active services for rule %s are unhealthy; "+
+				"falling back to active set", len(activeServices), rule.RequestModel)
+			healthyServices = activeServices
+		}
 	}
 
 	// For single healthy service rules, return it directly