fix: make account usage refresh resilient

README.md

@@ -87,7 +87,7 @@ Server without making that account active:
 # Query the active managed account now.
 codex-switch account usage
 
-# Query one saved account, or all accounts concurrently.
+# Query one saved account, or all accounts in a reliable sequence.
 codex-switch account usage work
 codex-switch account usage --all
 
@@ -100,10 +100,18 @@ codex-switch account usage work --cached
 ```
 
 Normal `account list` calls refresh only missing snapshots or snapshots older
-than 60 seconds. Each query runs in an isolated temporary `CODEX_HOME`; it does
+than five minutes. Explicit `--refresh` queries every account. Failed queries
+are retried up to three times with backoff, and multi-account refreshes run in a
+reliable sequence instead of starting several Codex App Servers at once. Each
+query runs in an isolated temporary `CODEX_HOME`; it does
 not switch `$CODEX_HOME/auth.json`, sessions, plugins, or UI state. The cache
 contains usage numbers and public account metadata only, never tokens.
 
+The compact list preserves the last successful values when a refresh fails,
+shows an error summary in the `ERROR` column, and prints the complete redacted
+error below the table. The last failure is retained in the local usage cache so
+`--cached` remains diagnostic after the failing process exits.
+
 The compact list shows the next automatic reset for each quota window. Its
 `RESETS` column is separate: it reports earned manual rate-limit resets and the
 nearest known expiration, when Codex provides them. `account usage <alias>`

docs/architecture.md

@@ -82,17 +82,20 @@ acquire shared operation lock
   -> create one temporary CODEX_HOME per profile
   -> write only that profile plus file-store config
   -> initialize the official Codex App Server
-  -> read account/rateLimits/read and account/usage/read
+  -> read account/read, account/rateLimits/read, and account/usage/read in order
   -> retain quota reset times and any earned-reset count and expiration metadata
   -> stop the server and delete the temporary home
-  -> reconcile any newer credential generation
+  -> reconcile any newer credential generation, including one produced by a failed query
   -> atomically save credential-free usage snapshots
 ```
 
 The live account selection never changes. If Codex rotates a refresh token while
 answering the query, identity and generation checks run before the new document
 is saved. For an active profile, compare-before-replace protects the live
-projection from a concurrent Codex write.
+projection from a concurrent Codex write. Transient failures are retried with
+backoff using the newest validated credential generation returned by the prior
+attempt. Multi-account refreshes are intentionally sequential so independent
+App Server processes do not compete for startup or authentication resources.
 
 ## Data locations
 

docs/security.md

@@ -45,7 +45,9 @@ focuses on:
   never contains authentication documents.
 - A token refreshed during an isolated query is accepted only after account,
   workspace, and refresh-generation checks. Active-file updates use a
-  compare-before-replace check under the shared operation lock.
+  compare-before-replace check under the shared operation lock. A validated
+  newer generation is preserved even when the accompanying usage request fails,
+  so retries do not fall back to a rotated token.
 - Active-account detection uses the live account and workspace identifiers;
   stale recorded state, aliases, and email addresses cannot redirect live
   credentials into another saved profile. Synchronization rechecks the live

docs/troubleshooting.md

@@ -123,7 +123,11 @@ codex-switch account usage <alias> --cached
 ```
 
 A stale cached snapshot is labeled `stale`; a failed refresh keeps that snapshot
-and displays a warning rather than discarding useful data.
+and displays the complete redacted error rather than discarding useful data.
+The tool retries a failed isolated query up to three times with backoff and
+queries saved accounts sequentially to avoid competing Codex App Server startup
+or authentication work. The last failure remains visible with `--cached` until
+that account refreshes successfully.
 
 ## Interrupted switch journal
 

internal/accountusage/service.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"sync"
 	"time"
 
 	"github.com/SilkageNet/codex-switch/internal/atomicfile"
@@ -22,7 +21,11 @@ import (
 	"github.com/SilkageNet/codex-switch/internal/vault"
 )
 
-const queryTimeout = 20 * time.Second
+const (
+	queryTimeout          = 20 * time.Second
+	defaultQueryAttempts  = 3
+	defaultRetryBaseDelay = 250 * time.Millisecond
+)
 
 type QueryRunner interface {
 	Query(context.Context, json.RawMessage) (codexusage.Snapshot, json.RawMessage, error)
@@ -33,19 +36,24 @@ type Service struct {
 	Paths  appconfig.Paths
 	Vault  *vault.Manager
 	Runner QueryRunner
+
+	queryAttempts  int
+	retryBaseDelay time.Duration
 }
 
 type Result struct {
 	ProfileID string              `json:"profileId"`
 	Snapshot  codexusage.Snapshot `json:"snapshot,omitempty"`
 	Error     string              `json:"error,omitempty"`
+	Attempts  int                 `json:"attempts,omitempty"`
 }
 
 type queryResult struct {
 	profileID string
 	snapshot  codexusage.Snapshot
 	auth      json.RawMessage
 	err       error
+	attempts  int
 }
 
 func (service Service) Cached() (usagecache.Cache, error) {
@@ -79,42 +87,26 @@ func (service Service) Refresh(ctx context.Context, profileIDs []string) (map[st
 		}
 	}
 
-	queried := make(chan queryResult, len(selected))
-	semaphore := make(chan struct{}, 4)
-	var group sync.WaitGroup
+	queried := make([]queryResult, 0, len(selected))
 	for _, profile := range selected {
-		profile := profile
-		group.Add(1)
-		go func() {
-			defer group.Done()
-			select {
-			case semaphore <- struct{}{}:
-				defer func() { <-semaphore }()
-			case <-ctx.Done():
-				queried <- queryResult{profileID: profile.ID, err: ctx.Err()}
-				return
-			}
-			queryContext, cancel := context.WithTimeout(ctx, queryTimeout)
-			defer cancel()
-			snapshot, updatedAuth, queryErr := service.Runner.Query(queryContext, profile.Auth)
-			queried <- queryResult{profileID: profile.ID, snapshot: snapshot, auth: updatedAuth, err: queryErr}
-		}()
+		snapshot, updatedAuth, attempts, queryErr := service.queryWithRetry(ctx, profile.Auth)
+		queried = append(queried, queryResult{profileID: profile.ID, snapshot: snapshot, auth: updatedAuth, err: queryErr, attempts: attempts})
 	}
-	group.Wait()
-	close(queried)
 
 	results := make(map[string]Result, len(selected))
 	candidates := make(map[string]json.RawMessage, len(selected))
 	cache, err := usagecache.Load(service.Paths.UsageCache)
 	if err != nil {
 		return nil, err
 	}
-	for result := range queried {
-		entry := Result{ProfileID: result.profileID}
+	for _, result := range queried {
+		entry := Result{ProfileID: result.profileID, Attempts: result.attempts}
 		if result.err != nil {
 			entry.Error = result.err.Error()
 		} else {
 			entry.Snapshot = result.snapshot
+		}
+		if len(result.auth) > 0 {
 			candidates[result.profileID] = result.auth
 		}
 		results[result.profileID] = entry
@@ -159,7 +151,9 @@ func (service Service) Refresh(ctx context.Context, profileIDs []string) (map[st
 			results[profileID] = entry
 			continue
 		}
-		cache.Profiles[profileID] = results[profileID].Snapshot
+		if results[profileID].Error == "" {
+			cache.Profiles[profileID] = results[profileID].Snapshot
+		}
 		saved, parseErr := authschema.Parse(profile.Auth)
 		if parseErr != nil {
 			entry := results[profileID]
@@ -210,12 +204,73 @@ func (service Service) Refresh(ctx context.Context, profileIDs []string) (map[st
 			delete(cache.Profiles, profileID)
 		}
 	}
+	for profileID, result := range results {
+		if result.Error == "" {
+			delete(cache.Failures, profileID)
+			continue
+		}
+		cache.Failures[profileID] = usagecache.Failure{
+			AttemptedAt: time.Now().UTC(),
+			Message:     result.Error,
+			Attempts:    result.Attempts,
+		}
+	}
+	for profileID := range cache.Failures {
+		if !validProfiles[profileID] {
+			delete(cache.Failures, profileID)
+		}
+	}
 	if err := usagecache.Save(service.Paths.UsageCache, cache); err != nil {
 		return nil, err
 	}
 	return results, nil
 }
 
+func (service Service) queryWithRetry(ctx context.Context, auth json.RawMessage) (codexusage.Snapshot, json.RawMessage, int, error) {
+	attemptLimit := service.queryAttempts
+	if attemptLimit <= 0 {
+		attemptLimit = defaultQueryAttempts
+	}
+	baseDelay := service.retryBaseDelay
+	if baseDelay <= 0 {
+		baseDelay = defaultRetryBaseDelay
+	}
+	currentAuth := append(json.RawMessage(nil), auth...)
+	var latestAuth json.RawMessage
+	var lastErr error
+	for attempt := 1; attempt <= attemptLimit; attempt++ {
+		if err := ctx.Err(); err != nil {
+			return codexusage.Snapshot{}, nil, attempt - 1, err
+		}
+		queryContext, cancel := context.WithTimeout(ctx, queryTimeout)
+		snapshot, updatedAuth, err := service.Runner.Query(queryContext, currentAuth)
+		cancel()
+		if len(updatedAuth) > 0 {
+			latestAuth = append(latestAuth[:0], updatedAuth...)
+			currentAuth = append(currentAuth[:0], updatedAuth...)
+		}
+		if err == nil {
+			if len(updatedAuth) == 0 {
+				updatedAuth = latestAuth
+			}
+			return snapshot, updatedAuth, attempt, nil
+		}
+		lastErr = err
+		if attempt == attemptLimit {
+			break
+		}
+		delay := baseDelay << (attempt - 1)
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return codexusage.Snapshot{}, latestAuth, attempt, ctx.Err()
+		case <-timer.C:
+		}
+	}
+	return codexusage.Snapshot{}, latestAuth, attemptLimit, fmt.Errorf("query failed after %d attempts: %w", attemptLimit, lastErr)
+}
+
 func (service Service) reconcileActive(profile *vault.Profile, candidate authschema.Document, state *appstate.State) (bool, error) {
 	liveRaw, err := service.Home.ReadAuth()
 	if errors.Is(err, os.ErrNotExist) {

internal/accountusage/service_test.go

@@ -3,8 +3,10 @@ package accountusage
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -23,6 +25,23 @@ type fakeRunner struct {
 	err      error
 }
 
+type retryRunner struct {
+	calls    int
+	failures int
+	snapshot codexusage.Snapshot
+	auth     json.RawMessage
+	inputs   []json.RawMessage
+}
+
+func (runner *retryRunner) Query(_ context.Context, auth json.RawMessage) (codexusage.Snapshot, json.RawMessage, error) {
+	runner.calls++
+	runner.inputs = append(runner.inputs, append(json.RawMessage(nil), auth...))
+	if runner.calls <= runner.failures {
+		return codexusage.Snapshot{}, runner.auth, errors.New("temporary connection failure")
+	}
+	return runner.snapshot, runner.auth, nil
+}
+
 func (runner fakeRunner) Query(context.Context, json.RawMessage) (codexusage.Snapshot, json.RawMessage, error) {
 	return runner.snapshot, runner.auth, runner.err
 }
@@ -57,6 +76,67 @@ func TestRefreshCachesUsageAndAdoptsRotatedCredentials(t *testing.T) {
 	}
 }
 
+func TestRefreshRetriesAndClearsPersistedFailure(t *testing.T) {
+	service, _, profile := testService(t, false)
+	runner := &retryRunner{
+		failures: 2,
+		snapshot: codexusage.Snapshot{FetchedAt: time.Now().UTC(), PlanType: "pro"},
+		auth:     authBytes("account-a", "refresh-new", "2026-08-20T01:00:00Z"),
+	}
+	service.Runner = runner
+	service.retryBaseDelay = time.Nanosecond
+	results, err := service.Refresh(context.Background(), []string{profile.ID})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runner.calls != 3 || results[profile.ID].Attempts != 3 || results[profile.ID].Error != "" {
+		t.Fatalf("unexpected retry result: calls=%d result=%#v", runner.calls, results[profile.ID])
+	}
+	retriedWith, err := authschema.Parse(runner.inputs[1])
+	if err != nil || retriedWith.Tokens.RefreshToken != "refresh-new" {
+		t.Fatalf("retry did not use rotated credentials: %#v, %v", retriedWith.Tokens, err)
+	}
+	cache, err := service.Cached()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := cache.Failures[profile.ID]; ok {
+		t.Fatalf("successful retry retained failure: %#v", cache.Failures[profile.ID])
+	}
+}
+
+func TestRefreshPersistsFailureDetails(t *testing.T) {
+	service, manager, profile := testService(t, false)
+	runner := &retryRunner{failures: 3, auth: authBytes("account-a", "refresh-new", "2026-08-20T01:00:00Z")}
+	service.Runner = runner
+	service.retryBaseDelay = time.Nanosecond
+	results, err := service.Refresh(context.Background(), []string{profile.ID})
+	if err != nil {
+		t.Fatal(err)
+	}
+	result := results[profile.ID]
+	if result.Attempts != 3 || !strings.Contains(result.Error, "temporary connection failure") {
+		t.Fatalf("unexpected failure result: %#v", result)
+	}
+	cache, err := service.Cached()
+	if err != nil {
+		t.Fatal(err)
+	}
+	failure := cache.Failures[profile.ID]
+	if failure.Attempts != 3 || failure.AttemptedAt.IsZero() || failure.Message != result.Error {
+		t.Fatalf("failure was not persisted: %#v", failure)
+	}
+	data, err := manager.Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	saved, _ := data.Find(profile.ID)
+	document, _ := authschema.Parse(saved.Auth)
+	if document.Tokens.RefreshToken != "refresh-new" {
+		t.Fatal("rotated credentials from failed queries were not preserved")
+	}
+}
+
 func TestRefreshProjectsRotatedCredentialsForActiveProfile(t *testing.T) {
 	service, manager, profile := testService(t, true)
 	service.Runner = fakeRunner{