From 7310d77909d45eb5dbae203fb3f1bd3692aa6bc2 Mon Sep 17 00:00:00 2001 From: "voxist.executor" Date: Tue, 11 Aug 2026 17:09:12 +0000 Subject: [PATCH] feat(dolt): drain the off-box backlog by CLI push before the sql-server starts (vp-6hb8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE TRAP. Pushing over the sql-server pays a cold-open cost once per server lifetime per database: the git-blobstore transport has no server-side range reads, so the first push of a lifetime spools the store's whole remote blobset (measured: 822 MB / 311 git calls for a 1 GB store) while the listener's read_timeout — 15s in production, deliberately: it is the only working idle-connection reaper, wait_timeout being accepted but inert (verified empirically 2026-08-11, 8 idle connections alive at t+48s under wait_timeout=30) — kills the query mid-open. A store that misses one push accumulates backlog, which makes the next attempt larger, which makes it miss again: measured live, one store ratcheted 0 -> 8,514 unpushed commits in five days, unbackable the whole time. Every server restart re-arms the trap for every large store at once. Session-scoped timeouts are ignored by the listener; raising the global read_timeout reproduces the 2026-06-15 connection pileup; retry loops make literally zero progress (twelve consecutive attempts, backlog unchanged — the push dies during the remote-db open, before any chunk transmits). THE MECHANISM. Drain each database by CLI `dolt push` BEFORE the sql-server starts. startManagedDoltProcessWithOptions has just waited for the data-dir LOCK to be free, so no server owns the store and CLI access is safe — and a CLI push has no listener in front of it, so no read_timeout applies, no config raise/restore, no second restart. The server then boots cold but with ZERO backlog, and a cold push of a near-empty delta fits the production window (measured 2026-08-06: every store passed at 15s immediately after a drain, largest 13.2s), after which the store is warm and stays current via the patrol. FAILURE POSTURE. The drain never blocks the boot: a store with an unreachable or corrupted remote (five of nine fleet stores have hit remote-side "Blob not found" archive corruption) still needs its LOCAL server, so every per-database failure is loud and boot proceeds. It never force-pushes — a diverged store needs an ownership decision this code cannot make (vp-ukvx). The branch pushed is the database's OWN checked-out head from repo_state.json; an unreadable state is UNKNOWN and skipped with a reason, never pushed at a guessed branch. Budget exhaustion NAMES the databases it did not attempt (a truncated sweep that reads as complete is the vp-g2m4 shape). Default ON — the drain exists so an UNATTENDED restart cannot silently re-arm the ratchet — with GC_DOLT_BOOT_DRAIN=off for maintenance windows that manage draining themselves, and GC_DOLT_BOOT_DRAIN_BUDGET for the wall-clock cap (default 10m; a full-fleet drain after five days of backlog measured ~6m worst case). Fprintf returns to the operator log are explicitly ignored: best-effort logging must never block a boot. Mutations, each verified applied (anchor exactly once, mutant compiles) before its result counted; two initial mutants were themselves invalid Go (unused variable) or mis-anchored and were REDONE rather than recorded: M1 failure stops the sweep -> 1 failure M2 pushed at a guessed branch -> 1 failure M3 exhaustion stops naming the rest -> 1 failure M4 mangled repo_state pushed anyway -> 2 failures M5 kill switch defaults off -> 1 failure --- cmd/gc/dolt_boot_drain.go | 259 ++++++++++++++++++ cmd/gc/dolt_boot_drain_test.go | 182 ++++++++++++ cmd/gc/dolt_start_managed.go | 9 + .../testdata/gc_env_read_baseline.golden | 2 + 4 files changed, 452 insertions(+) create mode 100644 cmd/gc/dolt_boot_drain.go create mode 100644 cmd/gc/dolt_boot_drain_test.go diff --git a/cmd/gc/dolt_boot_drain.go b/cmd/gc/dolt_boot_drain.go new file mode 100644 index 0000000000..cca480f988 --- /dev/null +++ b/cmd/gc/dolt_boot_drain.go @@ -0,0 +1,259 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" +) + +// Boot-time backlog drain for the managed Dolt store (vp-6hb8). +// +// THE TRAP THIS CLOSES. Pushing over the sql-server pays a cold-open cost once +// per server lifetime per database: the git-blobstore transport has no +// server-side range reads, so the first push of a lifetime spools the store's +// whole remote blobset (measured: 822 MB / 311 git calls for a 1 GB store) +// while the listener's read_timeout — 15s in production, and deliberately so: +// it is the only working idle-connection reaper, wait_timeout being accepted +// but inert (verified 2026-08-11) — kills the query mid-open. A store that +// misses one push accumulates backlog, which makes the next attempt larger, +// which makes it miss again: measured on the live fleet, one store ratcheted +// from 0 to 8,514 unpushed commits in five days, unbackable the whole time, +// and every server restart re-arms the trap for every large store at once. +// +// THE MECHANISM. Drain each database by CLI `dolt push` BEFORE the sql-server +// starts. At that moment the caller has already waited for the data-dir LOCK +// to be free, so no server owns the store and CLI access is safe — and a CLI +// push has no listener in front of it, so no read_timeout applies and no +// config needs to be raised and restored. The server then boots cold but with +// ZERO backlog, and a cold push of a near-empty delta fits the production +// window (measured 2026-08-06: every store passed at 15s immediately after a +// drain, largest 13.2s), after which the store is warm and stays current. +// +// FAILURE POSTURE. The drain must never block the boot: a store whose remote +// is unreachable or corrupted (five of nine fleet stores have hit remote-side +// "Blob not found" archive corruption) still needs its LOCAL server. Every +// per-database failure is reported loudly and boot proceeds. The one thing the +// drain never does is force-push: a diverged store needs an ownership decision +// this code cannot make (vp-ukvx), so it is reported and left alone. + +// bootDrainResult is one database's outcome. +type bootDrainResult struct { + DB string + Skipped string // non-empty reason => not attempted + Err string // non-empty => attempted and failed + Duration time.Duration +} + +// bootDrainReport is the whole pass. +type bootDrainReport struct { + Enabled bool + Results []bootDrainResult + Exhaust bool // budget ran out before all databases were attempted + Duration time.Duration +} + +// managedDoltBootDrainPushFn runs one CLI push; a test seam like the other +// managed-dolt seams in this file's siblings. The production implementation +// shells `dolt push ` with the database directory as cwd. +var managedDoltBootDrainPushFn = runBootDrainPush + +// bootDrainNowFn is the drain's clock; a seam so the budget logic is testable +// without wall-clock sleeps (the resource census forbids growing the +// fixed-sleep ledger, and it is right: a slept test is a flaky test). +var bootDrainNowFn = time.Now + +func runBootDrainPush(dbDir, remote, branch string, timeout time.Duration) error { + cmd := exec.Command("dolt", "push", remote, branch) + cmd.Dir = dbDir + // The CLI must not inherit a half-configured server env; it operates on + // the files directly. + cmd.Env = append(os.Environ(), "DOLT_CLI_PASSWORD=") + done := make(chan error, 1) + out := &strings.Builder{} + cmd.Stdout = out + cmd.Stderr = out + if err := cmd.Start(); err != nil { + return err + } + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + if err != nil { + return fmt.Errorf("%w: %s", err, truncateForLog(out.String(), 300)) + } + return nil + case <-time.After(timeout): + _ = cmd.Process.Kill() + <-done + return fmt.Errorf("timed out after %s: %s", timeout, truncateForLog(out.String(), 300)) + } +} + +func truncateForLog(s string, n int) string { + s = strings.TrimSpace(s) + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// bootDrainHeadBranch reads the database's checked-out branch from +// repo_state.json. An unreadable or unexpected head is UNKNOWN — the database +// is skipped with a reason, never pushed at a guessed branch (a push aimed at +// the wrong branch is how a remote ends up holding a lineage nobody chose). +func bootDrainHeadBranch(dbDir string) (string, error) { + raw, err := os.ReadFile(filepath.Join(dbDir, ".dolt", "repo_state.json")) + if err != nil { + return "", err + } + var state struct { + Head string `json:"head"` + Remotes map[string]json.RawMessage `json:"remotes"` + } + if err := json.Unmarshal(raw, &state); err != nil { + return "", err + } + const prefix = "refs/heads/" + if !strings.HasPrefix(state.Head, prefix) { + return "", fmt.Errorf("head %q is not a local branch ref", state.Head) + } + if len(state.Remotes) == 0 { + return "", errNoBootDrainRemote + } + if _, ok := state.Remotes["origin"]; !ok { + return "", fmt.Errorf("no 'origin' remote (has: %s)", strings.Join(sortedKeys(state.Remotes), ",")) + } + return strings.TrimPrefix(state.Head, prefix), nil +} + +var errNoBootDrainRemote = fmt.Errorf("no remotes configured") + +func sortedKeys(m map[string]json.RawMessage) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// runManagedDoltBootDrain drains every database under dataDir that has an +// origin remote, within an overall wall-clock budget. It NEVER returns an +// error: the report says what happened, the boot proceeds regardless, and the +// caller prints the report where the operator will see it. RULE 3: it reports +// what it DID (pushed N, skipped M, failed K), not that it finished. +func runManagedDoltBootDrain(dataDir string, budget time.Duration, out io.Writer) bootDrainReport { + report := bootDrainReport{Enabled: true} + started := bootDrainNowFn() + defer func() { report.Duration = bootDrainNowFn().Sub(started) }() + + entries, err := os.ReadDir(dataDir) + if err != nil { + _, _ = fmt.Fprintf(out, "gc: dolt boot-drain: cannot enumerate %s: %v — drain skipped, boot continues\n", dataDir, err) + report.Results = append(report.Results, bootDrainResult{DB: dataDir, Skipped: "enumerate: " + err.Error()}) + return report + } + names := make([]string, 0, len(entries)) + for _, e := range entries { + if !e.IsDir() { + continue + } + if _, statErr := os.Stat(filepath.Join(dataDir, e.Name(), ".dolt")); statErr != nil { + continue // not a dolt database + } + names = append(names, e.Name()) + } + sort.Strings(names) + + for _, name := range names { + remaining := budget - bootDrainNowFn().Sub(started) + if remaining <= 0 { + report.Exhaust = true + _, _ = fmt.Fprintf(out, "gc: dolt boot-drain: budget %s exhausted; NOT attempted: %s — their backlog remains unpushed\n", + budget, strings.Join(names[indexOf(names, name):], ", ")) + for _, rest := range names[indexOf(names, name):] { + report.Results = append(report.Results, bootDrainResult{DB: rest, Skipped: "budget exhausted"}) + } + break + } + dbDir := filepath.Join(dataDir, name) + branch, err := bootDrainHeadBranch(dbDir) + if err != nil { + reason := err.Error() + if errors.Is(err, errNoBootDrainRemote) { + reason = "no remote (nothing to drain)" + } + report.Results = append(report.Results, bootDrainResult{DB: name, Skipped: reason}) + if !errors.Is(err, errNoBootDrainRemote) { + _, _ = fmt.Fprintf(out, "gc: dolt boot-drain: %s skipped: %s\n", name, reason) + } + continue + } + pushStart := bootDrainNowFn() + pushErr := managedDoltBootDrainPushFn(dbDir, "origin", branch, remaining) + elapsed := bootDrainNowFn().Sub(pushStart) + if pushErr != nil { + report.Results = append(report.Results, bootDrainResult{DB: name, Err: pushErr.Error(), Duration: elapsed}) + _, _ = fmt.Fprintf(out, "gc: dolt boot-drain: %s push FAILED after %s (%v) — boot continues, backlog remains; a diverged or corrupted remote needs a human (vp-ukvx)\n", + name, elapsed.Round(time.Second), truncateForLog(pushErr.Error(), 200)) + continue + } + report.Results = append(report.Results, bootDrainResult{DB: name, Duration: elapsed}) + _, _ = fmt.Fprintf(out, "gc: dolt boot-drain: %s pushed in %s\n", name, elapsed.Round(time.Millisecond)) + } + pushed, failed, skipped := 0, 0, 0 + for _, r := range report.Results { + switch { + case r.Err != "": + failed++ + case r.Skipped != "": + skipped++ + default: + pushed++ + } + } + _, _ = fmt.Fprintf(out, "gc: dolt boot-drain: pushed %d, failed %d, skipped %d in %s\n", + pushed, failed, skipped, bootDrainNowFn().Sub(started).Round(time.Second)) + return report +} + +func indexOf(names []string, name string) int { + for i, n := range names { + if n == name { + return i + } + } + return len(names) +} + +// bootDrainEnabled resolves the kill switch. Default ON: the drain exists +// precisely so that an UNATTENDED restart cannot silently re-arm the backlog +// ratchet, so it must not depend on anyone remembering to enable it. The env +// switch exists for maintenance windows that manage the drain themselves. +func bootDrainEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("GC_DOLT_BOOT_DRAIN"))) { + case "0", "off", "false", "no": + return false + } + return true +} + +// bootDrainBudget bounds the whole pass. A full-fleet drain after a long +// outage measured ~6 minutes worst case (nine stores, one at 8.5k commits); +// the default leaves headroom without letting a wedged remote hold the boot +// hostage indefinitely. +func bootDrainBudget() time.Duration { + if raw := strings.TrimSpace(os.Getenv("GC_DOLT_BOOT_DRAIN_BUDGET")); raw != "" { + if d, err := time.ParseDuration(raw); err == nil && d > 0 { + return d + } + } + return 10 * time.Minute +} diff --git a/cmd/gc/dolt_boot_drain_test.go b/cmd/gc/dolt_boot_drain_test.go new file mode 100644 index 0000000000..d5411e1ca7 --- /dev/null +++ b/cmd/gc/dolt_boot_drain_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// makeBootDrainDB lays a minimal dolt-shaped database dir: .dolt/repo_state.json. +func makeBootDrainDB(t *testing.T, root, name, head string, remotes map[string]any) { + t.Helper() + dir := filepath.Join(root, name, ".dolt") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + state := map[string]any{"head": head, "remotes": remotes} + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "repo_state.json"), raw, 0o644); err != nil { + t.Fatal(err) + } +} + +type bootDrainCall struct { + dir, remote, branch string +} + +func stubBootDrainPush(t *testing.T, calls *[]bootDrainCall, fail map[string]error, advance time.Duration) func() { + t.Helper() + prevPush := managedDoltBootDrainPushFn + prevNow := bootDrainNowFn + // A fake clock instead of wall sleeps: each push "takes" advance. The + // resource census forbids growing the fixed-sleep ledger, and it is right — + // a slept test is a flaky test. + now := time.Unix(1_700_000_000, 0) + bootDrainNowFn = func() time.Time { return now } + managedDoltBootDrainPushFn = func(dbDir, remote, branch string, _ time.Duration) error { + *calls = append(*calls, bootDrainCall{dbDir, remote, branch}) + now = now.Add(advance) + if err, ok := fail[filepath.Base(dbDir)]; ok { + return err + } + return nil + } + return func() { managedDoltBootDrainPushFn = prevPush; bootDrainNowFn = prevNow } +} + +// A database with an origin remote is pushed at ITS OWN checked-out branch — +// never a guessed one. A database with no remote is skipped without noise; a +// non-database directory is ignored entirely. +func TestBootDrainPushesEachRemotedDatabaseAtItsOwnBranch(t *testing.T) { + root := t.TempDir() + makeBootDrainDB(t, root, "hq", "refs/heads/main", map[string]any{"origin": map[string]any{}}) + makeBootDrainDB(t, root, "vr", "refs/heads/work", map[string]any{"origin": map[string]any{}}) + makeBootDrainDB(t, root, "local-only", "refs/heads/main", map[string]any{}) + if err := os.MkdirAll(filepath.Join(root, "not-a-db"), 0o755); err != nil { + t.Fatal(err) + } + + var calls []bootDrainCall + defer stubBootDrainPush(t, &calls, nil, 0)() + var out strings.Builder + report := runManagedDoltBootDrain(root, time.Minute, &out) + + if len(calls) != 2 { + t.Fatalf("pushes = %d, want 2 (got %+v)", len(calls), calls) + } + byBase := map[string]string{} + for _, c := range calls { + byBase[filepath.Base(c.dir)] = c.branch + if c.remote != "origin" { + t.Errorf("remote = %q, want origin", c.remote) + } + } + if byBase["hq"] != "main" || byBase["vr"] != "work" { + t.Errorf("branches = %v; a push at a guessed branch would hand the remote a lineage nobody chose", byBase) + } + var skipped int + for _, r := range report.Results { + if r.Skipped != "" { + skipped++ + } + } + if skipped != 1 { + t.Errorf("skipped = %d, want 1 (the remoteless db)", skipped) + } +} + +// A failing push must NOT block the boot or the remaining databases: five of +// nine fleet stores have hit remote-side archive corruption, and a store with +// a dead remote still needs its local server. +func TestBootDrainFailureIsLoudAndNonBlocking(t *testing.T) { + root := t.TempDir() + makeBootDrainDB(t, root, "aa-broken", "refs/heads/main", map[string]any{"origin": map[string]any{}}) + makeBootDrainDB(t, root, "zz-healthy", "refs/heads/main", map[string]any{"origin": map[string]any{}}) + + var calls []bootDrainCall + defer stubBootDrainPush(t, &calls, map[string]error{"aa-broken": fmt.Errorf("Blob not found: abc.darc")}, 0)() + var out strings.Builder + report := runManagedDoltBootDrain(root, time.Minute, &out) + + if len(calls) != 2 { + t.Fatalf("the failure stopped the sweep: pushes = %d, want 2", len(calls)) + } + if !strings.Contains(out.String(), "FAILED") || !strings.Contains(out.String(), "aa-broken") { + t.Errorf("failure not reported loudly; out = %q", out.String()) + } + var failed int + for _, r := range report.Results { + if r.Err != "" { + failed++ + } + } + if failed != 1 { + t.Errorf("failed = %d, want 1", failed) + } +} + +// Budget exhaustion stops the sweep AND names what it did not attempt — +// a truncated sweep that reads as complete is the vp-g2m4 shape. +func TestBootDrainBudgetExhaustionNamesTheUnattempted(t *testing.T) { + root := t.TempDir() + for _, name := range []string{"aa", "bb", "cc"} { + makeBootDrainDB(t, root, name, "refs/heads/main", map[string]any{"origin": map[string]any{}}) + } + var calls []bootDrainCall + defer stubBootDrainPush(t, &calls, nil, 60*time.Millisecond)() + var out strings.Builder + report := runManagedDoltBootDrain(root, 90*time.Millisecond, &out) + + if len(calls) >= 3 { + t.Fatalf("budget did not stop the sweep: %d pushes", len(calls)) + } + if !report.Exhaust { + t.Error("report.Exhaust = false, want true") + } + if !strings.Contains(out.String(), "NOT attempted") { + t.Errorf("unattempted stores not named; out = %q", out.String()) + } +} + +// An unreadable repo_state is UNKNOWN: skipped with a reason, never pushed at +// a guessed branch (RULE 1 — unknown must not share a path with "fine"). +func TestBootDrainUnreadableStateIsSkippedNotGuessed(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "mangled", ".dolt") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "repo_state.json"), []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + var calls []bootDrainCall + defer stubBootDrainPush(t, &calls, nil, 0)() + var out strings.Builder + runManagedDoltBootDrain(root, time.Minute, &out) + if len(calls) != 0 { + t.Fatalf("a mangled state was pushed anyway: %+v", calls) + } + if !strings.Contains(out.String(), "mangled skipped") { + t.Errorf("skip not reported; out = %q", out.String()) + } +} + +// The kill switch: default ON (an unattended restart must not silently re-arm +// the ratchet), explicit off values honored. +func TestBootDrainKillSwitch(t *testing.T) { + for value, want := range map[string]bool{"": true, "1": true, "on": true, "0": false, "off": false, "false": false, "no": false} { + t.Run("value="+value, func(t *testing.T) { + t.Setenv("GC_DOLT_BOOT_DRAIN", value) + if got := bootDrainEnabled(); got != want { + t.Errorf("bootDrainEnabled() with %q = %v, want %v", value, got, want) + } + }) + } +} diff --git a/cmd/gc/dolt_start_managed.go b/cmd/gc/dolt_start_managed.go index 74787c08df..47814470f2 100644 --- a/cmd/gc/dolt_start_managed.go +++ b/cmd/gc/dolt_start_managed.go @@ -186,6 +186,15 @@ func startManagedDoltProcessWithOptions(cityPath, host, port, user, logLevel str return report, fmt.Errorf("refusing to start dolt sql-server for %s: %w", layout.DataDir, err) } + // Drain the off-box backlog BEFORE the server starts (vp-6hb8). This is + // the only moment a CLI push is safe (the lock-free wait above proved no + // server owns the store) and the only path with no listener deadline in + // front of it. See dolt_boot_drain.go for the trap this closes. Failures + // never block the boot. + if bootDrainEnabled() { + runManagedDoltBootDrain(layout.DataDir, bootDrainBudget(), os.Stderr) + } + currentPort := portNum // retryWindow is resolved once before the loop so an in-progress // city.toml edit cannot change the wait policy mid-flight. diff --git a/internal/testenv/testdata/gc_env_read_baseline.golden b/internal/testenv/testdata/gc_env_read_baseline.golden index 0acd10960e..7b6a37a2a7 100644 --- a/internal/testenv/testdata/gc_env_read_baseline.golden +++ b/internal/testenv/testdata/gc_env_read_baseline.golden @@ -52,6 +52,8 @@ GC_DISABLE_USAGE_METRICS GC_DOLT GC_DOLT_ARCHIVE_LEVEL GC_DOLT_AUTO_GC_ENABLED +GC_DOLT_BOOT_DRAIN +GC_DOLT_BOOT_DRAIN_BUDGET GC_DOLT_CRED_CMD GC_DOLT_DATABASE GC_DOLT_HOST