From 752a51debf26c2dcd9c1d651f1c70b049cdd0e47 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:52:37 +0530 Subject: [PATCH 01/22] fix(security): keep accounts off the engine's container names (GW-2) Creating a branch starts by removing the container of that name, so a branch called standby killed the live HA standby. CheckNewName refuses main, standby, restore, restore-check, agent-* and *-before-* wherever a name comes in from outside: POST /api/branches (masked too), every import target and its default, pipeline targets, and fox branch create. The engine's own creation goes through Create and is unaffected. Create and RestoreExport also refuse when a container of that name is already there, which the storage check cannot see. Co-Authored-By: Claude Opus 5.5 --- cmd/fox/main.go | 1 + internal/branch/branch.go | 50 +++++++++++++++++++- internal/branch/branch_before.go | 6 +-- internal/branch/export.go | 3 ++ internal/branch/import.go | 7 +++ internal/branch/names_test.go | 68 ++++++++++++++++++++++++++++ internal/branch/pipeline.go | 4 ++ internal/controlplane/server.go | 19 +++++++- internal/controlplane/server_test.go | 19 ++++++++ 9 files changed, 169 insertions(+), 8 deletions(-) diff --git a/cmd/fox/main.go b/cmd/fox/main.go index 3556160c..70861413 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -784,6 +784,7 @@ func branchCmd(args []string) { fmt.Println("usage: fox branch create [--from ] [--masked]") os.Exit(2) } + must(branch.CheckNewName(name)) started := time.Now() if hasFlag(args[1:], "--masked") { maskedBranchCreate(name, optValue(args[1:], "--from"), started) diff --git a/internal/branch/branch.go b/internal/branch/branch.go index 8c6e95c0..6faaea4b 100644 --- a/internal/branch/branch.go +++ b/internal/branch/branch.go @@ -878,6 +878,9 @@ func Create(name, parent string) error { if activeStorage().exists(name) { return fmt.Errorf("branch %q already exists", name) } + if err := checkNoContainer(name, ContainerState(name)); err != nil { + return err + } // A masked branch being made, or one that failed, is copied by nobody; // a copy of an open one is masked too (maskedbranch.go). if err := inheritMasked(name, parent); err != nil { @@ -1279,8 +1282,8 @@ func agentBranch(id string) string { return "agent-" + id } var ErrBadName = errors.New("invalid name: use letters, digits, '.', '-' and '_' (at most 63, starting with a letter or digit)") // safeNameRe is every name the engine passes to Docker, ZFS or btrfs. Stricter -// rules for new names live where names are made (the control plane's nameRe); -// this is the floor under every path that acts on an existing one. A '/' would +// rules for new names live where names are made (the control plane's nameRe, +// and CheckNewName); this is the floor under every path that acts on an existing one. A '/' would // walk out of the branch directory on btrfs (filepath.Join), '@' would name a // ZFS snapshot of another dataset, and "..", a space or a leading '-' mean // something to one tool or another. @@ -1293,6 +1296,49 @@ func checkName(name string) error { return nil } +// ErrReservedName is returned for a new branch named like one the engine makes +// for itself. +var ErrReservedName = errors.New("reserved branch name") + +// engineName reports whether name is one the engine's own containers and +// branches use: the primary, the HA standby, the two restores, an agent's. +func engineName(name string) bool { + switch name { + case "main", "standby", "restore", restoreCheckContainer: + return true + } + return strings.HasPrefix(name, "agent-") +} + +// CheckNewName is the rule for a name a person or an account picks. A branch's +// container is named after it and creating one starts by removing whatever +// holds that name, so a branch called "standby" used to kill the live HA +// standby. The engine makes these names itself by calling Create directly, and +// checkName must go on accepting them, so the refusal belongs here, where names +// come in from outside. Compared in lower case: an import target is not held to +// the control plane's lower-case nameRe, and Agent-x is still an agent's name +// to anyone reading it. +func CheckNewName(name string) error { + if err := checkName(name); err != nil { + return err + } + if n := strings.ToLower(name); engineName(n) || strings.Contains(n, "-before-") { + return fmt.Errorf("%w: %q is kept for the engine (main, standby, restore, restore-check, agent-*, *-before-*), choose another name", ErrReservedName, name) + } + return nil +} + +// checkNoContainer refuses a new branch whose container name is already taken, +// given that container's state: the storage check cannot see a container that +// has no branch behind it, like the standby's or a restore's, and starting the +// new one would remove it. +func checkNoContainer(name, state string) error { + if state == "absent" { + return nil + } + return fmt.Errorf("%w: a container named %s is already there (%s), and making this branch would remove it", ErrBranchExists, container(name), state) +} + // Exists reports whether a branch's data exists, running or not. func Exists(name string) bool { return checkName(name) == nil && activeStorage().exists(name) } diff --git a/internal/branch/branch_before.go b/internal/branch/branch_before.go index ab3ee9e3..3e631556 100644 --- a/internal/branch/branch_before.go +++ b/internal/branch/branch_before.go @@ -106,11 +106,9 @@ func validateBeforeName(name string) error { if !beforeNameRe.MatchString(name) { return fmt.Errorf("%w: invalid branch name %q (use lowercase letters, digits, dashes)", ErrInvalidRequest, name) } - switch { - case name == "main" || name == "restore" || name == "standby": + // Not CheckNewName: a name ending in -before- is exactly what this makes. + if engineName(name) { return fmt.Errorf("%w: %q is a reserved branch name", ErrInvalidRequest, name) - case strings.HasPrefix(name, "agent-"): - return fmt.Errorf("%w: names starting with agent- are reserved for agent branches", ErrInvalidRequest) } return nil } diff --git a/internal/branch/export.go b/internal/branch/export.go index 1325a0ce..fe20e5f7 100644 --- a/internal/branch/export.go +++ b/internal/branch/export.go @@ -458,6 +458,9 @@ func RestoreExport(archive, from, into string) error { if store.exists(into) { return fmt.Errorf("branch %q already exists — restore into a new name", into) } + if err := checkNoContainer(into, ContainerState(into)); err != nil { + return err + } dir, err := os.MkdirTemp("", "fox-restore-") if err != nil { return err diff --git a/internal/branch/import.go b/internal/branch/import.go index bf682597..0788af72 100644 --- a/internal/branch/import.go +++ b/internal/branch/import.go @@ -144,6 +144,10 @@ func importInstance(p *Progress, target, defName, desc string, a As, load func(s if target == "" { target = defName } + // After the default is filled in: it comes from the source's name. + if err := CheckNewName(target); err != nil { + return "", err + } p.Logf("Creating instance %q…\n", target) if err := Create(target, "main"); err != nil { return "", fmt.Errorf("create target instance %q: %w", target, err) @@ -203,6 +207,9 @@ func ImportContinuousToAs(p *Progress, source, target string, a As) (string, err if target == "" { target = defaultTargetName(source) } + if err := CheckNewName(target); err != nil { + return "", err + } // Every client below, and the subscription's walreceiver, is given the // source's own password or an empty one, never the container's (GW-1). source = noAmbientPassword(source) diff --git a/internal/branch/names_test.go b/internal/branch/names_test.go index 995887e4..7c666702 100644 --- a/internal/branch/names_test.go +++ b/internal/branch/names_test.go @@ -4,6 +4,7 @@ package branch import ( "errors" + "strings" "testing" ) @@ -67,3 +68,70 @@ func TestPublishedPortsAreLoopback(t *testing.T) { } } } + +// Any account could name a branch "standby", and making it began by removing +// the live HA standby's container. +func TestCheckNewNameRefusesEngineNames(t *testing.T) { + for _, n := range []string{"main", "standby", "restore", "restore-check", "agent-zz", "agent-", + "main-before-606", "x-before-y", "Standby", "AGENT-zz", "X-Before-1"} { + if err := CheckNewName(n); !errors.Is(err, ErrReservedName) { + t.Errorf("CheckNewName(%q) = %v, want ErrReservedName", n, err) + } + } + for _, n := range []string{"qa", "feature-123", "import-db", "agents", "mainline", "standby2", "restored", "before", "x-before"} { + if err := CheckNewName(n); err != nil { + t.Errorf("CheckNewName(%q) = %v, want ok", n, err) + } + } + if err := CheckNewName("x/../main"); !errors.Is(err, ErrBadName) { + t.Errorf("a bad name: %v, want ErrBadName", err) + } + // The floor under the engine's own names is not raised. + for _, n := range []string{"main", "standby", "restore", "restore-check", "agent-zz", "main-before-606"} { + if err := checkName(n); err != nil { + t.Errorf("checkName(%q) = %v: the engine uses this name", n, err) + } + } +} + +// Each way a name comes in from outside is refused before anything is run, so +// these need no Docker. +func TestNewNameEntryPointsRefuseEngineNames(t *testing.T) { + p := &Progress{} + check := func(what string, err error) { + t.Helper() + if !errors.Is(err, ErrReservedName) { + t.Errorf("%s: %v, want ErrReservedName", what, err) + } + } + _, err := ImportToAs(p, "postgres://h/d", "standby", As{}) + check("import target", err) + _, err = ImportToAs(p, "mongodb://h/d", "main-before-3", As{}) + check("mongodb target", err) + _, err = ImportReaderToAs(p, strings.NewReader(""), srcSQL, "x.sql", "agent-zz", As{}) + check("upload target", err) + _, err = ImportContinuousToAs(p, "postgres://h/d", "restore-check", As{}) + check("continuous target", err) +} + +func TestCheckNoContainer(t *testing.T) { + if err := checkNoContainer("qa", "absent"); err != nil { + t.Errorf("no container: %v", err) + } + for _, state := range []string{"running", "exited", "created"} { + if err := checkNoContainer("standby", state); !errors.Is(err, ErrBranchExists) { + t.Errorf("a %s container: %v, want ErrBranchExists", state, err) + } + } +} + +// branch-before makes the *-before-* names, so it must keep accepting them, +// and it now refuses the name it had missed. +func TestBeforeNameFollowsEngineNames(t *testing.T) { + if err := validateBeforeName("restore-check"); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("restore-check: %v", err) + } + if err := validateBeforeName("main-before-12"); err != nil { + t.Errorf("main-before-12: %v", err) + } +} diff --git a/internal/branch/pipeline.go b/internal/branch/pipeline.go index e3b0d7f7..3c753324 100644 --- a/internal/branch/pipeline.go +++ b/internal/branch/pipeline.go @@ -94,5 +94,9 @@ func RunPipelineAs(p *Progress, spec PipelineSpec, branch string, a As) (RunResu if runPipeline == nil { return RunResult{}, ErrPipelinesNotLicensed } + // The engine begins by deleting whatever has this name. + if err := CheckNewName(branch); err != nil { + return RunResult{}, err + } return runPipeline(p, spec, branch, a) } diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 7469fa7d..449ba691 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -39,6 +39,14 @@ import ( var nameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,40}$`) +// checkNewBranchName is the rule for a branch name an account picks. +func checkNewBranchName(name string) error { + if !nameRe.MatchString(name) { + return fmt.Errorf("invalid name (use lowercase letters, digits, dashes)") + } + return branch.CheckNewName(name) +} + // ruleIDRe matches a Blackbox policy rule id (bb.policy_rules.rule_id). var ruleIDRe = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) @@ -298,8 +306,8 @@ func registerAPI(mux *http.ServeMux) { Masked bool `json:"masked"` // a masked branch (masking step 5): made in the background } _ = json.NewDecoder(r.Body).Decode(&body) - if !nameRe.MatchString(body.Name) { - writeErr(w, 400, fmt.Errorf("invalid name (use lowercase letters, digits, dashes)")) + if err := checkNewBranchName(body.Name); err != nil { + writeErr(w, 400, err) return } if body.From != "" && !nameRe.MatchString(body.From) { @@ -1236,6 +1244,13 @@ func registerPipelines(mux *http.ServeMux, store *auth.Store) { return } target := pipelineBranch(pl) + // Here as well as in the engine: a run that got as far as the engine + // has already claimed the name, and is recorded as owning it even when + // it fails. + if err := branch.CheckNewName(target); err != nil { + send("error", map[string]string{"message": fmt.Sprintf("rename the pipeline: %v", err)}) + return + } // A spec saved before sources were checked may still name a file. if err := checkPipelineSource(spec.Source); err != nil { send("error", map[string]string{"message": err.Error()}) diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go index 9a327716..9869f539 100644 --- a/internal/controlplane/server_test.go +++ b/internal/controlplane/server_test.go @@ -3,12 +3,14 @@ package controlplane import ( + "errors" "net/http" "net/http/httptest" "strings" "testing" "testing/fstest" + "github.com/thefoxbyte/foxbyte/internal/branch" "github.com/thefoxbyte/foxbyte/internal/httpx" ) @@ -108,3 +110,20 @@ func TestTheConsolePageCarriesItsPolicy(t *testing.T) { } } } + +// POST /api/branches: the shape rule and the engine's names are both refused. +func TestCheckNewBranchName(t *testing.T) { + for _, n := range []string{"qa", "feature-123"} { + if err := checkNewBranchName(n); err != nil { + t.Errorf("%q: %v", n, err) + } + } + for _, n := range []string{"main", "standby", "restore", "restore-check", "agent-zz", "main-before-1"} { + if err := checkNewBranchName(n); !errors.Is(err, branch.ErrReservedName) { + t.Errorf("%q: %v, want ErrReservedName", n, err) + } + } + if err := checkNewBranchName("Bad"); err == nil || errors.Is(err, branch.ErrReservedName) { + t.Errorf("Bad: %v", err) + } +} From 62cf649cff14d6634eda4fd5d8ab7eadd98ab3cc Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:53:15 +0530 Subject: [PATCH 02/22] fix(gateway): cap what a client can make us allocate before auth (PERF-3) Before authenticating, a client's claimed length was enough to get up to 16 MiB allocated per message and 1 MiB per startup packet. The password message is now capped at 1 KiB and the startup packet at 10000 bytes, Postgres's own limit, both checked on the header and answered with SQLSTATE 08P01. Reads from the backend keep their limit and the proxied session is untouched. A SASLContinue shorter than its 4-byte code no longer indexes out of range. Co-Authored-By: Claude Opus 5.5 --- internal/proxy/auth.go | 33 +++++++- internal/proxy/proxy.go | 11 ++- internal/proxy/proxy_test.go | 154 +++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 6 deletions(-) diff --git a/internal/proxy/auth.go b/internal/proxy/auth.go index 8516c01a..194ea95e 100644 --- a/internal/proxy/auth.go +++ b/internal/proxy/auth.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/binary" + "errors" "fmt" "io" "net" @@ -26,14 +27,35 @@ import ( // --- low-level message framing (post-startup Postgres protocol) --- +const ( + // maxBackendMessage bounds a message from a branch's own Postgres. + maxBackendMessage = 1 << 24 + + // maxPasswordMessage bounds the one message read from a client before it has + // authenticated, length field included. It is sent in the clear-text form + // only, and holds an API key of well under 100 bytes; SCRAM is spoken to the + // backend, never to a client. Without this, each connection anyone opened + // could have the gateway allocate 16 MiB by claiming to send that much. + maxPasswordMessage = 1024 +) + +// errBadMsgLength is a message whose length field is out of bounds. +var errBadMsgLength = errors.New("bad message length") + func readMsg(c net.Conn) (byte, []byte, error) { + return readMsgMax(c, maxBackendMessage) +} + +// readMsgMax is readMsg for a peer held to a smaller limit, checked before +// anything is allocated for the body. +func readMsgMax(c net.Conn, limit uint32) (byte, []byte, error) { h := make([]byte, 5) if _, err := io.ReadFull(c, h); err != nil { return 0, nil, err } n := binary.BigEndian.Uint32(h[1:5]) - if n < 4 || n > 1<<24 { - return 0, nil, fmt.Errorf("bad message length %d", n) + if n < 4 || n > limit { + return 0, nil, fmt.Errorf("%w: %d", errBadMsgLength, n) } body := make([]byte, n-4) if _, err := io.ReadFull(c, body); err != nil { @@ -61,8 +83,11 @@ func requestClientPassword(client net.Conn) (string, error) { if err := writeMsg(client, 'R', req); err != nil { return "", err } - typ, body, err := readMsg(client) + typ, body, err := readMsgMax(client, maxPasswordMessage) if err != nil { + if errors.Is(err, errBadMsgLength) { + sendError(client, "08P01", "invalid password message length") // protocol_violation + } return "", err } if typ != 'p' { @@ -172,7 +197,7 @@ func scramSHA256(backend net.Conn, mechanisms []byte, password string) error { if err != nil { return err } - if typ != 'R' || binary.BigEndian.Uint32(body[:4]) != 11 { + if typ != 'R' || len(body) < 4 || binary.BigEndian.Uint32(body[:4]) != 11 { return fmt.Errorf("expected SASLContinue") } serverFirst := string(body[4:]) diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index c3e44f51..e9d8078e 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -85,6 +85,11 @@ const ( codeSSL = 80877103 // SSLRequest codeGSS = 80877104 // GSSENCRequest codeCancel = 80877102 // CancelRequest + + // maxStartupPacket is the longest startup packet accepted, length field + // included: Postgres's own MAX_STARTUP_PACKET_LENGTH, so no client that + // works with Postgres is refused here. + maxStartupPacket = 10000 ) // errHandledCancel signals that readStartup already handled a CancelRequest, so @@ -512,8 +517,10 @@ func readStartup(client net.Conn) (net.Conn, map[string]string, error) { if _, err := io.ReadFull(client, header); err != nil { return client, nil, err } - msgLen := int(binary.BigEndian.Uint32(header)) - if msgLen < 8 || msgLen > 1<<20 { + // A uint32 kept whole: on a 32-bit build int() could turn a huge length negative. + msgLen := binary.BigEndian.Uint32(header) + if msgLen < 8 || msgLen > maxStartupPacket { + sendError(client, "08P01", "invalid length of startup packet") // protocol_violation return client, nil, fmt.Errorf("bad startup length %d", msgLen) } body := make([]byte, msgLen-4) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 2d8c5e9b..e0ffcbc1 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -16,6 +16,8 @@ import ( "io" "math/big" "net" + "os" + "strings" "testing" "time" ) @@ -193,3 +195,155 @@ func TestSendError(t *testing.T) { t.Errorf("ErrorResponse missing SQLSTATE code: %q", got) } } + +// readErrorResponse reads one ErrorResponse and returns its fields as text. +func readErrorResponse(t *testing.T, c net.Conn) string { + t.Helper() + h := make([]byte, 5) + if _, err := io.ReadFull(c, h); err != nil { + t.Fatalf("reading the error: %v", err) + } + if h[0] != 'E' { + t.Fatalf("expected an ErrorResponse, got %q", h[0]) + } + body := make([]byte, binary.BigEndian.Uint32(h[1:])-4) + if _, err := io.ReadFull(c, body); err != nil { + t.Fatalf("reading the error body: %v", err) + } + return string(body) +} + +// A startup packet that claims too much is refused on its header alone: the +// test never sends the body, so a reader that waited for it would time out. +func TestReadStartupRefusesOversizedPacket(t *testing.T) { + for _, n := range []uint32{maxStartupPacket + 1, 1 << 20, 1<<32 - 1, 4} { + client, server := net.Pipe() + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + _ = client.SetDeadline(time.Now().Add(3 * time.Second)) + go func() { + h := make([]byte, 4) + binary.BigEndian.PutUint32(h, n) + _, _ = client.Write(h) + }() + errc := make(chan error, 1) + go func() { _, _, err := readStartup(server); errc <- err }() + if got := readErrorResponse(t, client); !bytes.Contains([]byte(got), []byte("08P01")) { + t.Errorf("length %d: error %q, want SQLSTATE 08P01", n, got) + } + if err := <-errc; err == nil || errors.Is(err, os.ErrDeadlineExceeded) { + t.Errorf("length %d: readStartup = %v", n, err) + } + client.Close() + server.Close() + } +} + +// The largest packet Postgres itself would take still gets through. +func TestReadStartupAcceptsLargestPacket(t *testing.T) { + params := map[string]string{"user": "u", "database": "main"} + params["options"] = strings.Repeat("x", maxStartupPacket-len(buildStartup(params))-len("options")-2) + pkt := buildStartup(params) + if len(pkt) != maxStartupPacket { + t.Fatalf("test packet is %d bytes, want %d", len(pkt), maxStartupPacket) + } + client, server := net.Pipe() + defer client.Close() + defer server.Close() + go func() { _, _ = client.Write(pkt) }() + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + _, got, err := readStartup(server) + if err != nil { + t.Fatalf("readStartup: %v", err) + } + if got["database"] != "main" { + t.Errorf("database = %q", got["database"]) + } +} + +// startPasswordRead runs requestClientPassword against a pipe and consumes the +// AuthenticationCleartextPassword request it sends first. +func startPasswordRead(t *testing.T) (net.Conn, chan error, *string) { + t.Helper() + client, server := net.Pipe() + t.Cleanup(func() { client.Close(); server.Close() }) + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + _ = client.SetDeadline(time.Now().Add(3 * time.Second)) + errc := make(chan error, 1) + got := new(string) + go func() { + var err error + *got, err = requestClientPassword(server) + errc <- err + }() + if _, err := io.ReadFull(client, make([]byte, 9)); err != nil { + t.Fatalf("reading the auth request: %v", err) + } + return client, errc, got +} + +// A client that has not authenticated is held to a small password message, on +// the header alone (no body is ever sent here) and with a real protocol error. +func TestPasswordMessageIsCapped(t *testing.T) { + for _, n := range []uint32{maxPasswordMessage + 1, 1 << 24, 3} { + client, errc, _ := startPasswordRead(t) + h := []byte{'p', 0, 0, 0, 0} + binary.BigEndian.PutUint32(h[1:], n) + if _, err := client.Write(h); err != nil { + t.Fatal(err) + } + if got := readErrorResponse(t, client); !bytes.Contains([]byte(got), []byte("08P01")) { + t.Errorf("length %d: error %q, want SQLSTATE 08P01", n, got) + } + if err := <-errc; !errors.Is(err, errBadMsgLength) { + t.Errorf("length %d: %v, want errBadMsgLength", n, err) + } + } +} + +// An API key, and anything up to the limit, is still read. +func TestPasswordMessageWithinLimit(t *testing.T) { + for _, secret := range []string{"key_" + strings.Repeat("a", 64), strings.Repeat("k", maxPasswordMessage-5)} { + client, errc, got := startPasswordRead(t) + if err := writeMsg(client, 'p', append([]byte(secret), 0)); err != nil { + t.Fatal(err) + } + if err := <-errc; err != nil { + t.Fatalf("%d-byte secret: %v", len(secret), err) + } + if *got != secret { + t.Errorf("read %q, want %q", *got, secret) + } + } +} + +// The limit is the client's alone: a backend may send much larger messages. +func TestBackendMessagesKeepTheirLimit(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + defer server.Close() + body := make([]byte, maxStartupPacket*4) + go func() { _ = writeMsg(client, 'S', body) }() + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + typ, got, err := readMsg(server) + if err != nil || typ != 'S' || len(got) != len(body) { + t.Fatalf("readMsg: %q, %d bytes, %v", typ, len(got), err) + } +} + +// A SASLContinue too short to hold its code used to index out of range. +func TestScramRefusesShortSASLContinue(t *testing.T) { + for _, short := range [][]byte{{}, {0, 0, 11}} { + gateway, backend := net.Pipe() + _ = gateway.SetDeadline(time.Now().Add(3 * time.Second)) + _ = backend.SetDeadline(time.Now().Add(3 * time.Second)) + go func() { + _, _, _ = readMsg(backend) // SASLInitialResponse + _ = writeMsg(backend, 'R', short) + }() + if err := scramSHA256(gateway, []byte("SCRAM-SHA-256\x00\x00"), "pw"); err == nil { + t.Errorf("a %d-byte SASLContinue was accepted", len(short)) + } + gateway.Close() + backend.Close() + } +} From 2e789c882c359f27d9a7e3b6e834add451eb1ba2 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:55:05 +0530 Subject: [PATCH 03/22] fix(security): check the release signature in fox setup too Client.Release fetched and parsed SHA256SUMS without ever looking at SHA256SUMS.sig, so fox setup installed an engine (which runs as root) on checksums alone and then printed that it was verified. Only Resolve checked. Both now read checksums through verifiedSums, which verifies the signature before parsing. A missing or bad signature, or a build with no release key, is an error: refreshEngineBinary keeps the installer's engine, and FOX_NO_REFRESH and FOX_NO_VERIFY stay the only deliberate ways round it. Co-Authored-By: Claude Opus 5.5 --- internal/host/host.go | 8 ++-- internal/update/select.go | 74 +++++++++++++++++++++------------- internal/update/update_test.go | 65 +++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 31 deletions(-) diff --git a/internal/host/host.go b/internal/host/host.go index 81634678..beeba3bc 100644 --- a/internal/host/host.go +++ b/internal/host/host.go @@ -266,8 +266,10 @@ func refreshEngineBinary(arch string) string { } // The engine is installed into the VM and run as root, so it is downloaded - // through the same verified path as `fox update`: the release's SHA256SUMS - // decides, and a file that doesn't match is never kept. + // through the same verified path as `fox update`: the release's signed + // SHA256SUMS decides, and a file that doesn't match is never kept. A release + // that doesn't verify (or a build with no release key) lands in keep below, + // so setup carries on with the installer's engine; it never goes unchecked. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() c := update.NewClient(os.Getenv, filepath.Join(cacheDir(), "update-check.json")) @@ -291,7 +293,7 @@ func refreshEngineBinary(arch string) string { _ = os.Remove(path) return keep("%s in release %s is not a Linux binary", asset, offer.Release.Tag) } - fmt.Printf("Engine %s from %s verified against SHA256SUMS.\n", asset, offer.Release.Tag) + fmt.Printf("Engine %s from %s verified against the signed SHA256SUMS.\n", asset, offer.Release.Tag) return path } diff --git a/internal/update/select.go b/internal/update/select.go index ec7af9d7..8a17e31f 100644 --- a/internal/update/select.go +++ b/internal/update/select.go @@ -5,6 +5,7 @@ package update import ( "bytes" "context" + "errors" "fmt" "sort" "strings" @@ -173,7 +174,8 @@ func Candidates(rels []Release, current Version, t Target, pin string) ([]Releas // // tag is a release tag, or "" / "latest" for the newest published release that // has every file this target needs. The returned Offer carries the release's -// SHA256SUMS, so Download verifies what it fetches. +// SHA256SUMS, checked against the release signature, so Download verifies what +// it fetches. There is no unsigned path: a missing or bad signature is an error. func (c *Client) Release(ctx context.Context, tag string, t Target) (*Offer, error) { rels, err := c.ListReleases(ctx) if err != nil { @@ -222,14 +224,9 @@ func (c *Client) Release(ctx context.Context, tag string, t Target) (*Offer, err } r := usable[0] - sa, _ := r.Asset(SumsAsset) - body, err := c.fetchSmall(ctx, sa.URL, 1<<20) - if err != nil { - return nil, fmt.Errorf("%s: fetching %s: %w", r.Tag, SumsAsset, err) - } - sums, err := ParseChecksums(bytes.NewReader(body)) + sums, err := c.verifiedSums(ctx, r) if err != nil { - return nil, fmt.Errorf("%s: %w", r.Tag, err) + return nil, err } for _, name := range required { if _, ok := sums[name]; !ok { @@ -240,6 +237,42 @@ func (c *Client) Release(ctx context.Context, tag string, t Target) (*Offer, err return &Offer{Release: r, Version: v, Sums: sums, Required: required}, nil } +// signatureError is a release whose SHA256SUMS failed its signature check, as +// opposed to one that could not be fetched. +type signatureError struct{ error } + +func (e signatureError) Unwrap() error { return e.error } + +// verifiedSums is the only way a release's checksums are read: it fetches +// SHA256SUMS and its signature, and parses the list only once the signature +// checks out, because nothing in SHA256SUMS is trusted before that. +func (c *Client) verifiedSums(ctx context.Context, r Release) (Checksums, error) { + sa, ok := r.Asset(SumsAsset) + if !ok { + return nil, fmt.Errorf("release %s has no %s", r.Tag, SumsAsset) + } + sigAsset, ok := r.Asset(SigAsset) + if !ok { + return nil, fmt.Errorf("release %s has no %s, so it cannot be verified", r.Tag, SigAsset) + } + body, err := c.fetchSmall(ctx, sa.URL, 1<<20) + if err != nil { + return nil, fmt.Errorf("%s: fetching %s: %w", r.Tag, SumsAsset, err) + } + sig, err := c.fetchSmall(ctx, sigAsset.URL, 1024) + if err != nil { + return nil, fmt.Errorf("%s: fetching %s: %w", r.Tag, SigAsset, err) + } + if err := VerifySums(body, sig); err != nil { + return nil, signatureError{fmt.Errorf("release %s: %w", r.Tag, err)} + } + sums, err := ParseChecksums(bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("%s: %w", r.Tag, err) + } + return sums, nil +} + // Available is the check `fox start` makes: the newest release worth telling the // user about, or nil when this install is up to date. // @@ -293,29 +326,14 @@ func (c *Client) Resolve(ctx context.Context, current Version, t Target, pin str if ctx.Err() != nil { return nil, ctx.Err() } - sa, _ := r.Asset(SumsAsset) - body, err := c.fetchSmall(ctx, sa.URL, 1<<20) - if err != nil { - lastErr = err - continue - } - // Nothing in SHA256SUMS is trusted until its signature checks out. - sigAsset, _ := r.Asset(SigAsset) - sig, err := c.fetchSmall(ctx, sigAsset.URL, 1024) + sums, err := c.verifiedSums(ctx, r) if err != nil { - lastErr = err - continue - } - if err := VerifySums(body, sig); err != nil { // Not "nothing new": a release that exists and does not verify is // either forged or broken, and saying "up to date" would hide it. - sigErr = fmt.Errorf("release %s: %w", r.Tag, err) - lastErr = sigErr - continue - } - sums, err := ParseChecksums(bytes.NewReader(body)) - if err != nil { - lastErr = fmt.Errorf("%s: %w", r.Tag, err) + if errors.As(err, new(signatureError)) { + sigErr = err + } + lastErr = err continue } var unlisted []string diff --git a/internal/update/update_test.go b/internal/update/update_test.go index aacda2f9..570dcd66 100644 --- a/internal/update/update_test.go +++ b/internal/update/update_test.go @@ -11,6 +11,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -503,6 +504,70 @@ func TestReleaseForSetup(t *testing.T) { } } +// `fox setup` installs an engine that runs as root, so Release holds its +// checksums to the same signature as Resolve: nothing comes back from a release +// that doesn't verify, whichever way it fails. +func TestReleaseRequiresTheSignature(t *testing.T) { + linux := Target{GOOS: "linux", HostArch: "amd64"} + ctx := context.Background() + _, other, _ := ed25519.GenerateKey(nil) + + t.Run("good", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "good"}) + o, err := f.client("").Release(ctx, "v0.9.1", linux) + if err != nil || o.Sums[engLinuxAMD64] == "" { + t.Fatalf("release = %+v, %v", o, err) + } + }) + t.Run("signed by another key", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "forged"}) + f.files["/dl/v0.9.1/SHA256SUMS.sig"] = ed25519.Sign(other, f.files["/dl/v0.9.1/SHA256SUMS"]) + for _, tag := range []string{"latest", "v0.9.1"} { + if o, err := f.client("").Release(ctx, tag, linux); o != nil || err == nil || + !strings.Contains(err.Error(), "not signed with the FoxByte release key") { + t.Errorf("%s: release = %+v, %v; want the signature error", tag, o, err) + } + } + }) + t.Run("altered after signing", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "good"}) + f.files["/dl/v0.9.1/SHA256SUMS"] = append(f.files["/dl/v0.9.1/SHA256SUMS"], '\n') + if o, err := f.client("").Release(ctx, "latest", linux); o != nil || err == nil { + t.Errorf("release = %+v, %v; want refused", o, err) + } + }) + t.Run("signature not attached", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "unsigned"}) + f.releases[0].Assets = f.releases[0].Assets[:len(f.releases[0].Assets)-1] + for _, tag := range []string{"latest", "v0.9.1"} { + if o, err := f.client("").Release(ctx, tag, linux); o != nil || err == nil { + t.Errorf("%s: release = %+v, %v; want refused", tag, o, err) + } + } + }) + t.Run("signature attached but unavailable", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "unsigned"}) + delete(f.files, "/dl/v0.9.1/SHA256SUMS.sig") + if o, err := f.client("").Release(ctx, "latest", linux); o != nil || err == nil || + !strings.Contains(err.Error(), SigAsset) { + t.Errorf("release = %+v, %v; want an error naming %s", o, err, SigAsset) + } + }) + t.Run("build with no release key", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "good"}) + releasePublicKey = "" // newFakeGitHub's cleanup restores it + if o, err := f.client("").Release(ctx, "latest", linux); o != nil || !errors.Is(err, ErrNoReleaseKey) { + t.Errorf("release = %+v, %v; want ErrNoReleaseKey", o, err) + } + }) +} + func TestShouldCheck(t *testing.T) { env := map[string]string{} get := func(k string) string { return env[k] } From cc6e724470d53cecc94a6945bf23d0e41bcfe321 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:55:05 +0530 Subject: [PATCH 04/22] feat(install): find the distro image in its own checksum listing The distro workflow is about to publish SHA256SUMS.distro instead of adding to SHA256SUMS. The Windows installer looks the image up there first and falls back to SHA256SUMS, which is where every earlier release lists it. The new listing only ever answers for the distro image. Co-Authored-By: Claude Opus 5.5 --- deploy/install.Tests.ps1 | 20 +++++++++++++++++++ deploy/install.ps1 | 42 +++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/deploy/install.Tests.ps1 b/deploy/install.Tests.ps1 index 97190f45..e8050a4d 100644 --- a/deploy/install.Tests.ps1 +++ b/deploy/install.Tests.ps1 @@ -86,6 +86,7 @@ Describe 'FoxByte release verification' { BeforeEach { $env:FOX_NO_VERIFY = $null Set-Variable -Name FoxSums -Scope Script -Value $script:listing + Set-Variable -Name FoxDistroSums -Scope Script -Value '' } AfterAll { $env:FOX_NO_VERIFY = $null } @@ -97,6 +98,25 @@ Describe 'FoxByte release verification' { Get-FoxChecksum 'foxbyte-distro.tar.gz' | Should -Be ('b' * 64) } + It 'reads the distro image from its own listing' { + Set-Variable -Name FoxDistroSums -Scope Script -Value (('c' * 64) + " foxbyte-distro.tar.zst`n") + Get-FoxChecksum 'foxbyte-distro.tar.zst' | Should -Be ('c' * 64) + } + + It 'prefers the distro listing over an older entry in SHA256SUMS' { + Set-Variable -Name FoxDistroSums -Scope Script -Value (('c' * 64) + " foxbyte-distro.tar.gz`n") + Get-FoxChecksum 'foxbyte-distro.tar.gz' | Should -Be ('c' * 64) + } + + It 'never lets the distro listing answer for another file' { + Set-Variable -Name FoxDistroSums -Scope Script -Value (('c' * 64) + " fox-windows-amd64.exe`n") + Get-FoxChecksum 'fox-windows-amd64.exe' | Should -Be ('a' * 64) + } + + It 'refuses a distro image neither listing names' { + { Get-FoxChecksum 'foxbyte-distro.tar.zst' } | Should -Throw '*not listed in SHA256SUMS*' + } + It 'refuses an asset the listing does not name' { { Get-FoxChecksum 'fox-linux-amd64' } | Should -Throw '*not listed in SHA256SUMS*' } diff --git a/deploy/install.ps1 b/deploy/install.ps1 index 4025a09a..47c562a4 100644 --- a/deploy/install.ps1 +++ b/deploy/install.ps1 @@ -280,7 +280,12 @@ function Assert-UpstreamChecksum([string]$Path, [string]$SumsUrl, [string]$Name) # download), not authorship -- signatures would be needed for that. The files # below are executed as root inside the distro, so an unverifiable one is not # installed. FOX_NO_VERIFY=1 skips the check deliberately. +# +# The distro image is built by its own workflow, on its own schedule, so its +# checksum is in SHA256SUMS.distro. Releases made before that file existed list +# the image in SHA256SUMS, which is why both are looked at for it, and only for it. $script:FoxSums = $null +$script:FoxDistroSums = $null function Get-FoxSums { if ($env:FOX_NO_VERIFY -eq '1') { return $null } @@ -295,16 +300,43 @@ function Get-FoxSums { return $script:FoxSums } +# Get-FoxDistroSums returns the distro image's own listing, or '' for a release +# without one (the distro workflow has not finished, or the release predates it). +function Get-FoxDistroSums { + if ($null -ne $script:FoxDistroSums) { return $script:FoxDistroSums } + try { + $timeout = $script:WebTimeout + $body = (Invoke-WebRequest -UseBasicParsing -Uri (Get-FoxAsset 'SHA256SUMS.distro') @timeout).Content + $script:FoxDistroSums = if ($body -is [byte[]]) { [System.Text.Encoding]::UTF8.GetString($body) } else { [string]$body } + } catch { + $script:FoxDistroSums = '' + } + return $script:FoxDistroSums +} + +# Find-FoxChecksum returns the SHA256 a listing gives for $Name, or $null. +function Find-FoxChecksum([string]$Listing, [string]$Name) { + foreach ($line in ($Listing -split "`n")) { + if ($line -match '^([0-9a-fA-F]{64})\s+\*?(.+?)\s*$' -and $Matches[2] -eq $Name) { + return $Matches[1].ToLower() + } + } + return $null +} + # Get-FoxChecksum returns the expected SHA256 for a release asset, or $null when -# verification is switched off. +# verification is switched off. The distro's own listing goes first, so a distro +# rebuilt for an older release is checked against its current hash; it can never +# answer for any other file. function Get-FoxChecksum([string]$Name) { $sums = Get-FoxSums if ($null -eq $sums) { return $null } - foreach ($line in ($sums -split "`n")) { - if ($line -match '^([0-9a-fA-F]{64})\s+\*?(.+?)\s*$' -and $Matches[2] -eq $Name) { - return $Matches[1].ToLower() - } + if ($Name -like 'foxbyte-distro.*') { + $want = Find-FoxChecksum (Get-FoxDistroSums) $Name + if ($want) { return $want } } + $want = Find-FoxChecksum $sums $Name + if ($want) { return $want } throw "$Name is not listed in SHA256SUMS for $Version -- nothing was installed.`nSet FOX_NO_VERIFY=1 to install without checking (not recommended)." } From e8e152e7e2e50171a04faad6df6fb344149e49f5 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:55:06 +0530 Subject: [PATCH 05/22] fix(security): never sign checksums read back from a release release-checksums.sh downloaded the published SHA256SUMS, merged its own lines in and signed the result, so anyone able to upload a release asset could get any hash signed with the release key. Each job now signs only what it hashed from its own build: release.yml owns SHA256SUMS, which still lists every file fox update and install.sh need, so released clients are unaffected, and wsl-distro.yml, which runs on its own schedule, publishes SHA256SUMS.distro. wsl-zfs.yml stops rewriting SHA256SUMS, which it did without the key, breaking the signature; no client reads the ZFS bundle's checksum, so it now sits in a .sha256 file beside the bundle. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 6 +- .github/workflows/release.yml | 11 +- .github/workflows/wsl-distro.yml | 19 ++-- .github/workflows/wsl-zfs.yml | 15 ++- README.md | 4 +- docs/windows-setup.md | 2 +- internal/update/checksums.go | 4 +- scripts/release-checksums.sh | 99 +++++++----------- scripts/test_release_checksums.sh | 166 +++++++++++++++++------------- 9 files changed, 164 insertions(+), 162 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b713c9eb..44412948 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,9 @@ jobs: # occasional impossible value rather than a failing test. - name: Test (race detector) run: go test -race ./... - # The release workflows share one SHA256SUMS; this checks the script that - # merges into it never drops another job's entries. - - name: Release checksum merging + # Each release workflow signs only its own build's checksums; this checks + # the script never signs anything read back from a release. + - name: Release checksum signing run: bash scripts/test_release_checksums.sh # The release job's check that every file `fox update` needs was published. - name: Release assets for fox update diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ebd90dcd..c962cfed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -285,11 +285,10 @@ jobs: # only integrity guarantee is TLS, and these assets are executed as root # inside the user's distro. # - # SHA256SUMS is shared with wsl-distro.yml, which runs on the same tag and - # can finish before or after this job. The script merges this job's entries - # into the published file by exact name, keeps the other job's, and checks - # again after uploading — so neither job can drop the other's checksums - # (v0.8.1 lost the distro image's entry that way). + # SHA256SUMS belongs to this job alone and lists only what it built here; + # wsl-distro.yml, on its own schedule, signs SHA256SUMS.distro for the + # distro image. Merging them meant signing lines read back from the + # release, so anyone able to upload an asset could get a hash signed. - name: Publish checksums env: GH_TOKEN: ${{ github.token }} @@ -298,7 +297,7 @@ jobs: FOX_RELEASE_SIGNING_KEY: ${{ secrets.FOX_RELEASE_SIGNING_KEY }} run: | cd dist - bash ../scripts/release-checksums.sh "$TAG" fox-* foxbyte-docker-context.tar.gz foxbyte-sbom.spdx.json + bash ../scripts/release-checksums.sh "$TAG" SHA256SUMS fox-* foxbyte-docker-context.tar.gz foxbyte-sbom.spdx.json - name: Attach binaries env: diff --git a/.github/workflows/wsl-distro.yml b/.github/workflows/wsl-distro.yml index a9309796..3db24a6e 100644 --- a/.github/workflows/wsl-distro.yml +++ b/.github/workflows/wsl-distro.yml @@ -73,10 +73,9 @@ jobs: gh release view "$TAG" >/dev/null 2>&1 || gh release create "$TAG" --prerelease --generate-notes --title "$TAG" - # SHA256SUMS is signed with the release key (audit v2 G22): fox update and - # install.sh refuse a release whose signature does not check. A missing - # secret fails the release here, rather than publishing one nobody can - # install. (cmd/releasesign also refuses a key fox would not accept.) + # This job's checksums are signed with the release key (audit v2 G22). A + # missing secret fails the job here, rather than publishing an image with + # no signature. (cmd/releasesign also refuses a key fox would not accept.) - name: Build the release signer env: FOX_RELEASE_SIGNING_KEY: ${{ secrets.FOX_RELEASE_SIGNING_KEY }} @@ -84,11 +83,11 @@ jobs: test -n "$FOX_RELEASE_SIGNING_KEY" || { echo "::error::the FOX_RELEASE_SIGNING_KEY secret is not set (see cmd/releasesign)"; exit 1; } go build -o "$RUNNER_TEMP/releasesign" ./cmd/releasesign - # Merged, not replaced: release.yml publishes sums for the binaries into - # the same SHA256SUMS. Every asset the installer or setup downloads must be - # listed, or verification silently skips it. The shared script matches - # entries by exact name and re-checks after uploading, so whichever job - # finishes last keeps the other's entries. + # A signed file of this job's own, because this workflow can run at any + # time for an existing tag and release.yml owns SHA256SUMS. Only the hash + # of the image built above is signed; nothing is read back from the + # release. install.ps1 looks the image up here, then in SHA256SUMS for + # releases made before this file existed. - name: Publish checksums env: GH_TOKEN: ${{ github.token }} @@ -97,7 +96,7 @@ jobs: FOX_RELEASE_SIGNING_KEY: ${{ secrets.FOX_RELEASE_SIGNING_KEY }} run: | cd dist - bash ../scripts/release-checksums.sh "$TAG" foxbyte-distro.tar.zst + bash ../scripts/release-checksums.sh "$TAG" SHA256SUMS.distro foxbyte-distro.tar.zst - name: Attach the distro image env: diff --git a/.github/workflows/wsl-zfs.yml b/.github/workflows/wsl-zfs.yml index 0495400d..3c88dc02 100644 --- a/.github/workflows/wsl-zfs.yml +++ b/.github/workflows/wsl-zfs.yml @@ -59,12 +59,11 @@ jobs: set -eux gh release upload "$TAG" dist/foxbyte-zfs-modules-*.tar.gz --clobber - # Keep SHA256SUMS complete, or setup skips verifying this bundle. + # A checksum beside each bundle, not in SHA256SUMS: that file is signed + # and belongs to release.yml, and rewriting it here without the key + # broke its signature, so fox update then refused the whole release. cd dist - gh release download "$TAG" -p SHA256SUMS -O existing.txt 2>/dev/null || : > existing.txt - sha256sum foxbyte-zfs-modules-*.tar.gz > mine.txt - awk '{print $2}' mine.txt > names.txt - grep -vFf names.txt existing.txt > SHA256SUMS || : > SHA256SUMS - cat mine.txt >> SHA256SUMS - sort -k2 -o SHA256SUMS SHA256SUMS - gh release upload "$TAG" SHA256SUMS --clobber + for f in foxbyte-zfs-modules-*.tar.gz; do + sha256sum "$f" > "$f.sha256" + gh release upload "$TAG" "$f.sha256" --clobber + done diff --git a/README.md b/README.md index 21cc1700..fcac7904 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,9 @@ Every release's `SHA256SUMS` is signed with the FoxByte release key it carries — which comes from this repository, not from the release — wherever `openssl` can do Ed25519 (OpenSSL 3; macOS's LibreSSL cannot, and the installer says so; `FOX_REQUIRE_SIGNATURE=1` refuses instead). `fox update` checks it on -every update and never installs a release that fails. Each file also has a +every update, `fox setup` on the engine it downloads, and neither installs a +release that fails. The Windows distro image, built by its own workflow, has +its own signed `SHA256SUMS.distro`. Each file also has a GitHub build attestation and the release an SBOM (`foxbyte-sbom.spdx.json`): ```bash diff --git a/docs/windows-setup.md b/docs/windows-setup.md index 772c586e..cba0a1c0 100644 --- a/docs/windows-setup.md +++ b/docs/windows-setup.md @@ -127,7 +127,7 @@ costs one ~2 MB bundle rather than another 740 MB image. **To cover a new kernel on an existing release**, run the **wsl-zfs** workflow manually with the kernel tag and the release tag. It builds just that bundle -(~1 hour), attaches it, and updates `SHA256SUMS` — no full rebuild. Add the +(~1 hour) and attaches it with a `.sha256` file beside it — no full rebuild. Add the kernel to the matrix as well, so future releases keep covering it. ## Building the ZFS bundle (maintainers) diff --git a/internal/update/checksums.go b/internal/update/checksums.go index 369af8ca..5102f08c 100644 --- a/internal/update/checksums.go +++ b/internal/update/checksums.go @@ -14,7 +14,9 @@ import ( ) // SumsAsset is the checksum file every release publishes -// (scripts/release-checksums.sh writes it). +// (scripts/release-checksums.sh writes it, in release.yml's job). It lists every +// file an update needs; the Windows distro image has a file of its own, which +// only install.ps1 reads. const SumsAsset = "SHA256SUMS" // Checksums maps a file name to its lowercase hex SHA-256. diff --git a/scripts/release-checksums.sh b/scripts/release-checksums.sh index 618b6f73..efc1a4b8 100755 --- a/scripts/release-checksums.sh +++ b/scripts/release-checksums.sh @@ -1,41 +1,42 @@ #!/usr/bin/env bash # SPDX-License-Identifier: AGPL-3.0-or-later # -# Publish this job's checksums into a GitHub release's SHA256SUMS without dropping -# anyone else's. +# Sign and publish the checksums of the files this job built, under a name only +# this job writes. # -# release-checksums.sh ... +# release-checksums.sh ... # -# Two workflows attach assets to the same release: release.yml (the fox and -# fox-verify binaries, the Docker context) and wsl-distro.yml (the Windows distro -# image). Both start from the same tag push and either can finish first. The -# installers verify every download against SHA256SUMS and silently skip any file -# that isn't listed, so a job that replaces the whole file with only its own -# entries quietly turns verification off for the other job's assets. That -# happened with v0.8.1. +# Two workflows attach assets to the same release, on their own schedules: +# release.yml (the fox and fox-verify binaries, the Docker context) publishes +# SHA256SUMS, and wsl-distro.yml (the Windows distro image) publishes +# SHA256SUMS.distro. Each file has its own signature, .sig. # -# So each job: -# 1. downloads the published SHA256SUMS (if any), -# 2. keeps every entry except those for the files it is publishing (matched by -# exact file name, not substring), -# 3. adds its own entries and uploads the result, -# 4. waits a moment, downloads again and checks its entries are still there — -# if the other job uploaded in between, it merges again (up to 3 times). +# This used to merge every job's entries into one SHA256SUMS: download the +# published file, add this job's lines, sign the result. That made the job a +# signing oracle, because anyone able to upload a release asset could put a line +# in the published file and have the release key sign it. So nothing read back +# from a release is ever signed here: the entries are hashed from the local +# files, which are this job's own build outputs, and a re-run simply replaces +# them. # -# Signing (audit v2 G22): with RELEASE_SIGN set to the releasesign tool -# (cmd/releasesign, which reads FOX_RELEASE_SIGNING_KEY), each upload of the -# merged SHA256SUMS goes with its signature, SHA256SUMS.sig, and step 4 also -# checks the published signature matches the published file. The two jobs can -# interleave (A's file, B's file, B's signature, A's signature), so a mismatch -# there is merged and signed again like any other race. +# The read-back afterwards only confirms what was uploaded is what is published +# (the release API can lag just behind an upload); it feeds nothing back in. +# +# RELEASE_SIGN is the releasesign tool (cmd/releasesign, which reads +# FOX_RELEASE_SIGNING_KEY). It is required: clients refuse an unsigned +# SHA256SUMS, so publishing one would only break the release. # # Needs gh (GH_TOKEN set) and sha256sum or shasum. Files may be given with a # directory; entries use the base name, as the installers expect. set -euo pipefail tag="${1:-}" -[ -n "$tag" ] && [ "$#" -ge 2 ] || { echo "usage: $0 ..." >&2; exit 2; } -shift +asset="${2:-}" +[ -n "$tag" ] && [ -n "$asset" ] && [ "$#" -ge 3 ] || { echo "usage: $0 ..." >&2; exit 2; } +shift 2 + +sign="${RELEASE_SIGN:-}" +[ -n "$sign" ] || { echo "error: RELEASE_SIGN is not set; a release's checksums must be signed" >&2; exit 2; } if command -v sha256sum >/dev/null 2>&1; then checksum() { sha256sum "$@"; } @@ -46,48 +47,26 @@ fi work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -# Our entries: " ". -checksum "$@" | awk '{ name = $2; sub(/.*\//, "", name); print $1 " " name }' > "$work/mine" +# Our entries, and nothing else: " ". +checksum "$@" | awk '{ name = $2; sub(/^\*/, "", name); sub(/.*\//, "", name); print $1 " " name }' | sort -k2 > "$work/$asset" + +"$sign" sign "$work/$asset" "$work/$asset.sig" +gh release upload "$tag" "$work/$asset" "$work/$asset.sig" --clobber -sign="${RELEASE_SIGN:-}" attempts="${CHECKSUM_ATTEMPTS:-3}" settle="${CHECKSUM_SETTLE_SECONDS:-10}" - +mkdir "$work/published" for attempt in $(seq 1 "$attempts"); do - # A release without SHA256SUMS yet starts from empty. - if ! gh release download "$tag" -p SHA256SUMS -O "$work/existing" --clobber >/dev/null 2>&1; then - : > "$work/existing" - fi - # Every published entry whose file we aren't publishing, then ours. - awk 'NR == FNR { mine[$2] = 1; next } NF >= 2 && !($2 in mine)' "$work/mine" "$work/existing" > "$work/SHA256SUMS" - cat "$work/mine" >> "$work/SHA256SUMS" - sort -k2 -o "$work/SHA256SUMS" "$work/SHA256SUMS" - - if [ -n "$sign" ]; then - "$sign" sign "$work/SHA256SUMS" "$work/SHA256SUMS.sig" - gh release upload "$tag" "$work/SHA256SUMS" "$work/SHA256SUMS.sig" --clobber - else - gh release upload "$tag" "$work/SHA256SUMS" --clobber - fi - sleep "$settle" - gh release download "$tag" -p SHA256SUMS -O "$work/published" --clobber - signed_ok=1 - if [ -n "$sign" ]; then - signed_ok=0 - if gh release download "$tag" -p SHA256SUMS.sig -O "$work/published.sig" --clobber >/dev/null 2>&1 && - "$sign" verify "$work/published" "$work/published.sig" >/dev/null 2>&1; then - signed_ok=1 - fi - fi - if [ "$signed_ok" = 1 ] && awk 'NR == FNR { want[$1 " " $2] = 1; total++; next } - (($1 " " $2) in want) && !seen[$1 " " $2]++ { found++ } - END { exit (found == total) ? 0 : 1 }' "$work/mine" "$work/published"; then - cat "$work/published" + if gh release download "$tag" -p "$asset" -O "$work/published/$asset" --clobber >/dev/null 2>&1 && + gh release download "$tag" -p "$asset.sig" -O "$work/published/$asset.sig" --clobber >/dev/null 2>&1 && + cmp -s "$work/$asset" "$work/published/$asset" && + cmp -s "$work/$asset.sig" "$work/published/$asset.sig"; then + cat "$work/$asset" exit 0 fi - echo "SHA256SUMS changed while publishing (another job uploaded at the same time); merging again ($attempt/$attempts)" >&2 + echo "$asset of $tag is not what was uploaded yet ($attempt/$attempts)" >&2 done -echo "error: could not publish checksums for: $*" >&2 +echo "error: $asset of $tag does not match what this job uploaded: another writer, or the upload failed" >&2 exit 1 diff --git a/scripts/test_release_checksums.sh b/scripts/test_release_checksums.sh index 6f0a0819..10c16a9e 100755 --- a/scripts/test_release_checksums.sh +++ b/scripts/test_release_checksums.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # # Tests scripts/release-checksums.sh against a fake `gh` that keeps the release's -# SHA256SUMS in a temporary directory. Runs anywhere (no GitHub access): +# assets in a temporary directory. Runs anywhere (no GitHub access): # bash scripts/test_release_checksums.sh set -uo pipefail @@ -20,14 +20,16 @@ assert_eq() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1"; printf ' got:\ # Supports exactly what the script uses: # gh release download -p -O --clobber # gh release upload ... --clobber -# STUB_RELEASE directory holding the release's SHA256SUMS -# STUB_OVERWRITE if set to a file, the first upload is immediately replaced by -# that file's content, as if another job uploaded at that moment +# STUB_RELEASE directory holding the release's assets +# STUB_LOG every call is appended here +# STUB_OVERWRITE if set to a file, each upload is immediately replaced by that +# file's content, as if somebody else wrote the asset # STUB_FAIL_UPLOAD make every upload fail mkdir -p "$tmp/bin" cat > "$tmp/bin/gh" <<'EOF' #!/usr/bin/env bash set -eu +echo "gh $*" >> "$STUB_LOG" [ "$1" = release ] || { echo "fake gh: unsupported: $*" >&2; exit 2; } case "$2" in download) @@ -39,108 +41,128 @@ download) upload) [ -z "${STUB_FAIL_UPLOAD:-}" ] || { echo "upload failed" >&2; exit 1; } shift 3 - for f in "$@"; do [ "$f" = --clobber ] || cp "$f" "$STUB_RELEASE/$(basename "$f")"; done - if [ -n "${STUB_OVERWRITE:-}" ] && [ ! -f "$STUB_RELEASE/.overwritten" ]; then - cp "$STUB_OVERWRITE" "$STUB_RELEASE/SHA256SUMS" - touch "$STUB_RELEASE/.overwritten" - fi + first="" + for f in "$@"; do + [ "$f" = --clobber ] && continue + cp "$f" "$STUB_RELEASE/$(basename "$f")" + [ -n "$first" ] || first="$(basename "$f")" + done + [ -z "${STUB_OVERWRITE:-}" ] || cp "$STUB_OVERWRITE" "$STUB_RELEASE/$first" ;; *) echo "fake gh: unsupported: $*" >&2; exit 2 ;; esac EOF chmod +x "$tmp/bin/gh" -export PATH="$tmp/bin:$PATH" CHECKSUM_SETTLE_SECONDS=0 + +# A stand-in for cmd/releasesign: the "signature" is the file's SHA-256, so the +# test can see exactly which bytes were signed. Every signed file is kept. +cat > "$tmp/bin/fakesign" <<'SIGNER' +#!/usr/bin/env bash +set -eu +h() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi; } +case "$1" in +sign) h "$2" > "$3"; cat "$2" >> "$STUB_SIGNED"; echo "signed" >> "$STUB_LOG" ;; +verify) [ "$(h "$2")" = "$(cat "$3")" ] ;; +esac +SIGNER +chmod +x "$tmp/bin/fakesign" +export PATH="$tmp/bin:$PATH" CHECKSUM_SETTLE_SECONDS=0 RELEASE_SIGN="$tmp/bin/fakesign" if command -v sha256sum >/dev/null 2>&1; then sum() { sha256sum "$1" | awk '{print $1}'; }; else sum() { shasum -a 256 "$1" | awk '{print $1}'; }; fi # Assets of the two jobs, in their own dist directories. mkdir -p "$tmp/rel" "$tmp/distro" for f in fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz; do echo "content of $f" > "$tmp/rel/$f"; done -echo "content of the distro image" > "$tmp/distro/foxbyte-distro.tar.gz" - -publish_release() { (cd "$tmp/rel" && bash "$script" v9.9.9 fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz) >/dev/null; } -publish_distro() { (cd "$tmp/distro" && bash "$script" v9.9.9 foxbyte-distro.tar.gz) >/dev/null; } -names() { awk '{print $2}' "$STUB_RELEASE/SHA256SUMS" | tr '\n' ' ' | sed 's/ $//'; } -new_release() { export STUB_RELEASE="$tmp/release-$1"; rm -rf "$STUB_RELEASE"; mkdir -p "$STUB_RELEASE"; unset STUB_OVERWRITE STUB_FAIL_UPLOAD; } +echo "content of the distro image" > "$tmp/distro/foxbyte-distro.tar.zst" + +publish_release() { (cd "$tmp/rel" && bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz) >/dev/null; } +publish_distro() { (cd "$tmp/distro" && bash "$script" v9.9.9 SHA256SUMS.distro foxbyte-distro.tar.zst) >/dev/null; } +names() { awk '{print $2}' "$STUB_RELEASE/$1" | tr '\n' ' ' | sed 's/ $//'; } +new_release() { + export STUB_RELEASE="$tmp/release-$1" STUB_LOG="$tmp/log-$1" STUB_SIGNED="$tmp/signed-$1" + rm -rf "$STUB_RELEASE" "$STUB_LOG" "$STUB_SIGNED" + mkdir -p "$STUB_RELEASE" + : > "$STUB_LOG" + : > "$STUB_SIGNED" + unset STUB_OVERWRITE STUB_FAIL_UPLOAD +} +sig_matches() { [ -f "$STUB_RELEASE/$1.sig" ] && [ "$(sum "$STUB_RELEASE/$1")" = "$(cat "$STUB_RELEASE/$1.sig")" ] && echo yes || echo no; } + +evil="$(printf 'e%.0s' $(seq 1 64))" echo "### release-checksums.sh" new_release first publish_release -assert_eq "a release without SHA256SUMS gets this job's entries" "$(names)" \ +assert_eq "the release job's file lists exactly what it built" "$(names SHA256SUMS)" \ "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz" +assert_eq "…and it is signed" "$(sig_matches SHA256SUMS)" "yes" publish_distro -assert_eq "the second job adds its entry and keeps the first job's" "$(names)" \ - "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-distro.tar.gz foxbyte-docker-context.tar.gz" -release_then_distro="$(cat "$STUB_RELEASE/SHA256SUMS")" +assert_eq "the distro job writes its own file" "$(names SHA256SUMS.distro)" "foxbyte-distro.tar.zst" +assert_eq "…signed on its own" "$(sig_matches SHA256SUMS.distro)" "yes" +assert_eq "…and leaves SHA256SUMS and its signature alone" "$(names SHA256SUMS)|$(sig_matches SHA256SUMS)" \ + "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz|yes" +release_then_distro="$(cat "$STUB_RELEASE/SHA256SUMS" "$STUB_RELEASE/SHA256SUMS.distro")" new_release reversed publish_distro publish_release -assert_eq "the result doesn't depend on which job finishes first" "$(cat "$STUB_RELEASE/SHA256SUMS")" "$release_then_distro" +assert_eq "the result doesn't depend on which job finishes first" \ + "$(cat "$STUB_RELEASE/SHA256SUMS" "$STUB_RELEASE/SHA256SUMS.distro")" "$release_then_distro" -echo "changed binary" > "$tmp/rel/fox-linux-amd64" +# The finding itself: whatever is already on the release must never be signed. +new_release oracle +printf '%s %s\n' "$evil" "fox-backdoor" > "$STUB_RELEASE/SHA256SUMS" +printf '%s %s\n' "$evil" "fox-linux-amd64" >> "$STUB_RELEASE/SHA256SUMS" publish_release -assert_eq "a re-run replaces its own entry without duplicating it" \ - "$(grep -c ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS")|$(grep ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS" | awk '{print $1}')|$(wc -l < "$STUB_RELEASE/SHA256SUMS" | tr -d ' ')" \ - "1|$(sum "$tmp/rel/fox-linux-amd64")|4" -assert_eq "…and leaves the other job's entry untouched" \ - "$(grep ' foxbyte-distro.tar.gz$' "$STUB_RELEASE/SHA256SUMS" | awk '{print $1}')" "$(sum "$tmp/distro/foxbyte-distro.tar.gz")" - -new_release exact -printf '%s %s\n' "$(printf 'a%.0s' $(seq 1 64))" "fox-linux-amd64.sig" > "$STUB_RELEASE/SHA256SUMS" +assert_eq "a line planted on the release is not signed" "$(grep -c "$evil" "$STUB_SIGNED")" "0" +assert_eq "…nor published" "$(grep -c "$evil" "$STUB_RELEASE/SHA256SUMS")" "0" +assert_eq "…and the planted file is replaced by the signed one" "$(names SHA256SUMS)|$(sig_matches SHA256SUMS)" \ + "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz|yes" +assert_eq "signing happens before anything is read from the release" \ + "$(grep -n 'download\|signed' "$STUB_LOG" | head -1 | sed 's/^[0-9]*://')" "signed" + +new_release rerun publish_release -assert_eq "entries are matched by exact name (fox-linux-amd64.sig survives)" "$(names)" \ - "fox-linux-amd64 fox-linux-amd64.sig fox-verify-linux-amd64 foxbyte-docker-context.tar.gz" - -new_release race -printf '%s %s\n' "$(printf 'b%.0s' $(seq 1 64))" "foxbyte-distro.tar.gz" > "$tmp/other-job.txt" -export STUB_OVERWRITE="$tmp/other-job.txt" -out="$( (cd "$tmp/rel" && bash "$script" v9.9.9 fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz) 2>&1 >/dev/null)" -assert_eq "another job uploading at the same moment is detected and merged again" \ - "$(echo "$out" | grep -c 'merging again')|$(names)" \ - "1|fox-linux-amd64 fox-verify-linux-amd64 foxbyte-distro.tar.gz foxbyte-docker-context.tar.gz" +echo "changed binary" > "$tmp/rel/fox-linux-amd64" +publish_release +assert_eq "a re-run replaces its own entries" \ + "$(grep -c ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS")|$(grep ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS" | awk '{print $1}')|$(sig_matches SHA256SUMS)" \ + "1|$(sum "$tmp/rel/fox-linux-amd64")|yes" + +new_release overwritten +printf '%s %s\n' "$evil" "fox-linux-amd64" > "$tmp/other-writer.txt" +export STUB_OVERWRITE="$tmp/other-writer.txt" +(cd "$tmp/rel" && CHECKSUM_ATTEMPTS=2 bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64) >/dev/null 2>&1 +assert_eq "somebody else's file in its place fails the job" "$?" "1" +assert_eq "…and is still not signed" "$(grep -c "$evil" "$STUB_SIGNED")" "0" +unset STUB_OVERWRITE new_release failing export STUB_FAIL_UPLOAD=1 -(cd "$tmp/rel" && bash "$script" v9.9.9 fox-linux-amd64) >/dev/null 2>&1 +(cd "$tmp/rel" && bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64) >/dev/null 2>&1 assert_eq "a failed upload fails the job" "$?" "1" unset STUB_FAIL_UPLOAD -assert_eq "missing arguments are a usage error" "$(bash "$script" v9.9.9 >/dev/null 2>&1; echo $?)" "2" +new_release nofile +(cd "$tmp/rel" && bash "$script" v9.9.9 SHA256SUMS no-such-file) >/dev/null 2>&1 +assert_eq "a file this job did not build fails the job" "$?:$(ls "$STUB_RELEASE" | wc -l | tr -d ' ')" "1:0" + +assert_eq "missing arguments are a usage error" "$(bash "$script" v9.9.9 SHA256SUMS >/dev/null 2>&1; echo $?)" "2" +assert_eq "no signer is an error, not an unsigned release" \ + "$(cd "$tmp/rel" && RELEASE_SIGN='' bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64 >/dev/null 2>&1; echo $?)" "2" echo -# --- signing --------------------------------------------------------------- -# A stand-in for cmd/releasesign: the "signature" is the file's SHA-256, so the -# test can see whether the published signature belongs to the published file. -cat > "$tmp/bin/fakesign" <<'SIGNER' -#!/usr/bin/env bash -set -eu -h() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi; } -case "$1" in -sign) h "$2" > "$3" ;; -verify) [ "$(h "$2")" = "$(cat "$3")" ] ;; -esac -SIGNER -chmod +x "$tmp/bin/fakesign" -sig_matches() { [ -f "$STUB_RELEASE/SHA256SUMS.sig" ] && [ "$(sum "$STUB_RELEASE/SHA256SUMS")" = "$(cat "$STUB_RELEASE/SHA256SUMS.sig")" ] && echo yes || echo no; } - -new_release signed -RELEASE_SIGN="$tmp/bin/fakesign" publish_release -RELEASE_SIGN="$tmp/bin/fakesign" publish_distro -assert_eq "signed: the published signature belongs to the final, merged SHA256SUMS" "$(sig_matches)" "yes" - -# Another job's upload lands between this job's file and its signature, leaving -# a signature for a file that is no longer there: merged and signed again. -new_release signed-race -RELEASE_SIGN="$tmp/bin/fakesign" publish_distro -printf '%s %s\n' "$(sum "$tmp/distro/foxbyte-distro.tar.gz")" foxbyte-distro.tar.gz > "$tmp/theirs" -export STUB_OVERWRITE="$tmp/theirs" -RELEASE_SIGN="$tmp/bin/fakesign" publish_release -unset STUB_OVERWRITE -assert_eq "signed: a signature left stale by another upload is redone" "$(sig_matches)|$(names)" \ - "yes|fox-linux-amd64 fox-verify-linux-amd64 foxbyte-distro.tar.gz foxbyte-docker-context.tar.gz" +# The workflows must stay on this path: nothing may read a release's checksums +# back in order to rewrite them. +wf="$here/../.github/workflows" +assert_eq "no workflow reads SHA256SUMS back from a release" \ + "$(grep -lE 'gh release download.*SHA256SUMS' "$wf"/*.yml | wc -l | tr -d ' ')" "0" +assert_eq "only release.yml publishes SHA256SUMS" \ + "$(grep -lE 'release-checksums\.sh "\$TAG" SHA256SUMS |gh release upload.* SHA256SUMS' "$wf"/*.yml | xargs -n1 basename | tr '\n' ' ' | sed 's/ $//')" "release.yml" +assert_eq "wsl-distro.yml publishes its own file" \ + "$(grep -c 'release-checksums\.sh "\$TAG" SHA256SUMS\.distro ' "$wf/wsl-distro.yml")" "1" echo "==== ${PASS} passed, ${FAIL} failed ====" [ "$FAIL" -eq 0 ] From 86f8b5c951499bca57d378c3ca73be291c5cc06a Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:55:27 +0530 Subject: [PATCH 06/22] fix(security): a pipeline's claim ends with its branch (API-2) ForgetBranch cleared the owner but left the row in pipeline_targets, so a pipeline's next run deleted whatever another account had since made under that name, and took it over. ForgetBranch now drops the claim, and a run replaces an existing target only if the pipeline made it and its user is still the recorded owner. fox pipeline run refuses a target an account owns. Co-Authored-By: Claude Opus 5.5 --- cmd/fox/main.go | 17 +++++++++++ internal/auth/accounts.go | 8 ++++- internal/auth/pipelines.go | 9 ++++++ internal/auth/setup_test.go | 52 +++++++++++++++++++++++++++++++++ internal/controlplane/server.go | 4 +-- 5 files changed, 87 insertions(+), 3 deletions(-) diff --git a/cmd/fox/main.go b/cmd/fox/main.go index 70861413..71854947 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -696,6 +696,11 @@ func pipelineCmd(args []string) { if target == "" { target = "pl-" + strings.TrimSuffix(filepath.Base(specPath), filepath.Ext(specPath)) } + // A run deletes and re-creates its target. A branch an account owns is not + // this command's to replace; deleting it first is a decision made out loud. + if branch.Exists(target) && accountOwnsBranch(target) { + must(fmt.Errorf("the branch %q belongs to an account: delete it first, or choose another name with --as", target)) + } res, err := branch.RunPipeline(&branch.Progress{Log: os.Stdout}, spec, target) must(err) if res.Failed { @@ -1483,6 +1488,18 @@ func forgetOwner(name string) { } } +// accountOwnsBranch reports whether an account is recorded as a branch's owner. +// Best-effort, like forgetOwner: with no store there are no accounts. +func accountOwnsBranch(name string) bool { + s, err := auth.OpenFromEnv() + if err != nil { + return false + } + defer s.Close() + _, owned := s.BranchOwner(name) + return owned +} + // userCmd is `fox user`: accounts, for whoever runs the engine (audit v2 G16). func userCmd(args []string) { usage := func() { diff --git a/internal/auth/accounts.go b/internal/auth/accounts.go index 2fd00481..1659aa1f 100644 --- a/internal/auth/accounts.go +++ b/internal/auth/accounts.go @@ -185,13 +185,19 @@ func (s *Store) BranchOwner(branch string) (int64, bool) { } // ForgetBranch drops what the store held about a branch that has been deleted: -// its owner, and any change request still waiting on it. One place decides what +// its owner, a pipeline's claim on its name, and any change request still +// waiting on it. One place decides what // forgetting a branch means, so every caller — the CLI, the API, the agent API — // gets the same thing. func (s *Store) ForgetBranch(branch string) error { if _, err := s.db.Exec(`DELETE FROM branch_owners WHERE branch=?`, branch); err != nil { return err } + // A claim that outlived the branch let its pipeline delete whatever somebody + // else later made under that name. + if _, err := s.db.Exec(`DELETE FROM pipeline_targets WHERE branch=?`, branch); err != nil { + return err + } // Readings go with the branch, unlike decided requests, which are history // and stay. A branch name can be reused, and a newly made branch that // inherited the old one's readings would report work it never did — diff --git a/internal/auth/pipelines.go b/internal/auth/pipelines.go index a94ff167..596fc429 100644 --- a/internal/auth/pipelines.go +++ b/internal/auth/pipelines.go @@ -141,3 +141,12 @@ func (s *Store) PipelineOwnsTarget(pipelineID, branch string) bool { err := s.db.QueryRow(`SELECT pipeline_id FROM pipeline_targets WHERE branch=?`, branch).Scan(&owner) return err == nil && owner == pipelineID } + +// PipelineMayReplace reports whether a run of pipelineID, which belongs to +// userID, may delete branch and make it again: the pipeline made the branch and +// the branch is still that account's. The claim alone is not enough, since a +// branch can change hands under the same name. +func (s *Store) PipelineMayReplace(pipelineID string, userID int64, branch string) bool { + owner, ok := s.BranchOwner(branch) + return ok && owner == userID && s.PipelineOwnsTarget(pipelineID, branch) +} diff --git a/internal/auth/setup_test.go b/internal/auth/setup_test.go index 4cd08b03..5806a70a 100644 --- a/internal/auth/setup_test.go +++ b/internal/auth/setup_test.go @@ -220,3 +220,55 @@ func TestPipelineTargets(t *testing.T) { t.Error("ownership reads back wrong") } } + +// A branch's name can be reused. A pipeline's claim used to outlive the branch, +// so its next run deleted whatever another account had since made under that +// name, and took it over. +func TestForgetBranchDropsThePipelineClaim(t *testing.T) { + s := testStore(t) + u, _ := s.CreateUser("a@x.com", "password1") + p1, _ := s.CreatePipeline(u.ID, "one", "{}") + p2, _ := s.CreatePipeline(u.ID, "two", "{}") + if err := s.ClaimPipelineTarget(p1.ID, "sales"); err != nil { + t.Fatal(err) + } + if err := s.ForgetBranch("sales"); err != nil { + t.Fatal(err) + } + if s.PipelineOwnsTarget(p1.ID, "sales") { + t.Error("the claim outlived the branch") + } + if err := s.ClaimPipelineTarget(p2.ID, "sales"); err != nil { + t.Errorf("the freed name could not be claimed: %v", err) + } +} + +func TestPipelineMayReplace(t *testing.T) { + s := testStore(t) + a, _ := s.CreateUser("a@x.com", "password1") + b, _ := s.CreateUser("b@x.com", "password1") + p, _ := s.CreatePipeline(a.ID, "one", "{}") + other, _ := s.CreatePipeline(a.ID, "two", "{}") + + if s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("allowed with neither a claim nor an owner") + } + if err := s.ClaimPipelineTarget(p.ID, "sales"); err != nil { + t.Fatal(err) + } + if s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("allowed on a branch with no recorded owner") + } + _ = s.SetBranchOwner("sales", a.ID) + if !s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("a pipeline could not refresh its own branch") + } + if s.PipelineMayReplace(other.ID, a.ID, "sales") { + t.Error("allowed for a pipeline that did not make the branch") + } + // The name changed hands while the claim stayed. + _ = s.SetBranchOwner("sales", b.ID) + if s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("allowed on a branch another account now owns") + } +} diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 449ba691..87345567 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -1257,8 +1257,8 @@ func registerPipelines(mux *http.ServeMux, store *auth.Store) { return } // A run deletes and re-creates its target, so it may only do that to - // a branch this pipeline made. - if branch.Exists(target) && !store.PipelineOwnsTarget(pl.ID, target) { + // a branch this pipeline made and this account still owns. + if branch.Exists(target) && !store.PipelineMayReplace(pl.ID, u.ID, target) { send("error", map[string]string{"message": fmt.Sprintf( "a branch named %q already exists and was not made by this pipeline — rename the pipeline, or delete that branch first", target)}) return From c99332567ff590d3beca1f20a8026d91927bf4a5 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:57:03 +0530 Subject: [PATCH 07/22] fix(security): check pipeline targets where the name comes in (GW-2) The dispatcher is handed engine names by its own tests and has no reason to judge them. fox pipeline run now refuses a reserved --as before the engine deletes that name, as the API already does. Co-Authored-By: Claude Opus 5.5 --- cmd/fox/main.go | 1 + internal/branch/pipeline.go | 4 ---- internal/controlplane/server.go | 5 ++--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cmd/fox/main.go b/cmd/fox/main.go index 71854947..12e2755f 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -696,6 +696,7 @@ func pipelineCmd(args []string) { if target == "" { target = "pl-" + strings.TrimSuffix(filepath.Base(specPath), filepath.Ext(specPath)) } + must(branch.CheckNewName(target)) // A run deletes and re-creates its target. A branch an account owns is not // this command's to replace; deleting it first is a decision made out loud. if branch.Exists(target) && accountOwnsBranch(target) { diff --git a/internal/branch/pipeline.go b/internal/branch/pipeline.go index 3c753324..e3b0d7f7 100644 --- a/internal/branch/pipeline.go +++ b/internal/branch/pipeline.go @@ -94,9 +94,5 @@ func RunPipelineAs(p *Progress, spec PipelineSpec, branch string, a As) (RunResu if runPipeline == nil { return RunResult{}, ErrPipelinesNotLicensed } - // The engine begins by deleting whatever has this name. - if err := CheckNewName(branch); err != nil { - return RunResult{}, err - } return runPipeline(p, spec, branch, a) } diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 87345567..4f325ef4 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -1244,9 +1244,8 @@ func registerPipelines(mux *http.ServeMux, store *auth.Store) { return } target := pipelineBranch(pl) - // Here as well as in the engine: a run that got as far as the engine - // has already claimed the name, and is recorded as owning it even when - // it fails. + // Before anything else: the engine begins by deleting whatever has this + // name, and a run is recorded as owning its target even when it fails. if err := branch.CheckNewName(target); err != nil { send("error", map[string]string{"message": fmt.Sprintf("rename the pipeline: %v", err)}) return From a8903c2f869f9b139e68760141d89e1bf980cc2b Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:57:28 +0530 Subject: [PATCH 08/22] perf(branch): one short inspect per branch, remembered for two seconds EnsureRunning and BackendAddr asked docker for the state twice and the IP once on every connection, each under the ten-minute tier. One inspect now answers both, on the probe tier, and a branch seen serving is kept for 2s with concurrent callers sharing the call. Every docker command that changes a container invalidates it where commands are run. Only serving is remembered, so no stale absent or stopped can exist; waking and recreating stand on a read of their own, and a docker that did not answer is ErrStateUnknown, never absent. Co-Authored-By: Claude Opus 5.5 --- internal/branch/branch.go | 101 +++++++-- internal/branch/branch_before.go | 4 +- internal/branch/maskedbranch.go | 1 + internal/branch/reads.go | 8 +- internal/branch/statecache.go | 218 ++++++++++++++++++ internal/branch/statecache_test.go | 342 +++++++++++++++++++++++++++++ 6 files changed, 655 insertions(+), 19 deletions(-) create mode 100644 internal/branch/statecache.go create mode 100644 internal/branch/statecache_test.go diff --git a/internal/branch/branch.go b/internal/branch/branch.go index 8c6e95c0..5bdcaf6a 100644 --- a/internal/branch/branch.go +++ b/internal/branch/branch.go @@ -173,8 +173,13 @@ func sudoCmd(d time.Duration, name string, args ...string) (*exec.Cmd, context.C ctx, cancel := context.WithTimeout(context.Background(), t) cmd := exec.CommandContext(ctx, "sudo", full...) // Without this, a killed child that leaves a pipe open keeps Wait blocked — - // the hang this is meant to end, one level down. + // the hang this is meant to end, one level down. Killing sudo does not kill + // the docker client under it, so a hung dockerd reaches this every time, and + // a probe measured in seconds must not then wait ten more. cmd.WaitDelay = 10 * time.Second + if t <= execProbe { + cmd.WaitDelay = time.Second + } return cmd, ctx, cancel } @@ -189,6 +194,7 @@ func execErr(ctx context.Context, d time.Duration, name string, err error) error // run executes a privileged command and streams output to the terminal. func run(name string, args ...string) error { + defer noteIfChange(name, args) cmd, ctx, cancel := sudoCmd(execLong, name, args...) defer cancel() cmd.Stdout = os.Stdout @@ -199,6 +205,7 @@ func run(name string, args ...string) error { // quiet executes a privileged command, discarding output and errors. Used for // best-effort cleanup where "not found" is not a failure. func quiet(name string, args ...string) { + defer noteIfChange(name, args) cmd, _, cancel := sudoCmd(execClean, name, args...) defer cancel() _ = cmd.Run() @@ -228,6 +235,7 @@ func execSudo(name string, args ...string) *exec.Cmd { // text, which is what it echoes on failure, carries no secret. Checked rather // than assumed: no capture call passes a password, a conninfo or a secret. func capture(name string, args ...string) (string, error) { + defer noteIfChange(name, args) cmd, ctx, cancel := sudoCmd(execQuick, name, args...) defer cancel() out, err := cmd.Output() @@ -819,6 +827,7 @@ func LedgerVerify(name string) (string, error) { // captureCombined runs a privileged command and returns its combined // stdout+stderr, so a failing psql includes the server's error text. func captureCombined(name string, args ...string) (string, error) { + defer noteIfChange(name, args) cmd, ctx, cancel := sudoCmd(execQuick, name, args...) defer cancel() out, err := cmd.CombinedOutput() @@ -868,6 +877,9 @@ func Create(name, parent string) error { if err := checkName(parent); err != nil { return err } + // Last, as well as at each docker command: the dataset is part of what the + // console lists. + defer noteStateChange() forgetBranchRoles(name) forgetCatalogScan(name) // Cloning a branch that doesn't exist fails deep in the storage layer with a @@ -918,6 +930,7 @@ func Delete(name string) error { if name == "main" { return fmt.Errorf("refusing to delete the primary branch 'main'") } + defer noteStateChange() // after the dataset goes, which the console lists forgetBranchRoles(name) forgetCatalogScan(name) forgetBranchUnmasks(name) @@ -1440,6 +1453,9 @@ func DeleteAgentBranch(agentID string) error { // BackendAddr returns host:port where a branch's Postgres is reachable, used by // the wire-protocol proxy to route connections. For "main" it resolves the // container currently serving as primary (which changes after an HA failover). +// +// PrimaryContainer is read each time and the memory is per container, so a +// failover run by another process moves "main" at once. func BackendAddr(name string) (string, error) { if name == "" { name = "main" @@ -1448,11 +1464,14 @@ func BackendAddr(name string) (string, error) { if name == "main" { cont = PrimaryContainer() } - ip, err := containerIP(cont) + f, err := containers.lookup(cont) if err != nil { return "", err } - return ip + ":5432", nil + if f.ip == "" { + return "", fmt.Errorf("container %q has no IP on the %q network", cont, network) + } + return f.addr(), nil } // primaryFile records which branch container currently serves as the "main" @@ -1477,6 +1496,7 @@ func primaryBranch() string { // setPrimary records the branch name now serving as primary. func setPrimary(name string) error { + defer noteStateChange() _ = os.MkdirAll(filepath.Dir(primaryFile()), 0o755) return os.WriteFile(primaryFile(), []byte(name), 0o644) } @@ -1547,8 +1567,14 @@ func StorageInfo() Storage { // --- auto-suspend / auto-resume --- // ContainerState returns "running", "exited"/"created", or "absent". -func ContainerState(name string) string { - out, err := capture("docker", "inspect", "-f", "{{.State.Status}}", container(name)) +// +// Always asks docker: the engine's own decisions are made on this, and +// statecache.go's memory is only for routing a connection. +func ContainerState(name string) string { return containerStateWithin(execQuick, name) } + +// containerStateWithin is ContainerState bounded by d. +func containerStateWithin(d time.Duration, name string) string { + out, err := captureWithin(d, "docker", "inspect", "-f", "{{.State.Status}}", container(name)) if err != nil { return "absent" } @@ -1615,6 +1641,39 @@ func Resume(name string) error { // idle one. var OnUse func(name string) +// OnWake, when set, is told a suspended branch is about to be resumed, and what +// state it was in. The gateway logs it. +var OnWake func(name, state string) + +// recreateBranch and resumeBranch are EnsureRunning's two ways of bringing a +// branch up, and branchExists the check before the first: variables so a test +// can see that a stale or missing answer never reaches them. +var ( + branchExists = func(name string) bool { return activeStorage().exists(name) } + + recreateBranch = func(name string) error { + if err := startContainer(name, name == "main"); err != nil { + return err + } + if err := waitReady(name); err != nil { + return err + } + // A new container, perhaps from a newer image: the SQL too. + syncIdentityBestEffort(name, true) + applyAuditBestEffort(name) + reconcileGuard(name) + return nil + } + + resumeBranch = func(name string) error { + if err := Resume(name); err != nil { + return err + } + reconcileGuard(name) + return nil + } +) + func EnsureRunning(name string) (string, error) { if err := checkName(name); err != nil { return "", err @@ -1634,30 +1693,38 @@ func EnsureRunning(name string) (string, error) { // and cause split-brain. Just route to the current primary. return BackendAddr("main") } - switch ContainerState(name) { + cont := container(name) + // The ordinary case, and the only one remembered answers can settle: it was + // serving a moment ago, and if that has stopped being true the dial fails + // and the caller comes back through here. + if f, err := containers.lookup(cont); err == nil && f.serving() { + return f.addr(), nil + } + // Anything else may start or recreate a container, so it stands on an + // answer asked for here and now. + f, err := probeContainer(cont) + if err != nil { + return "", fmt.Errorf("branch %q: %w", name, err) + } + switch f.state { case "running": // already up case "absent": // No container. If the dataset survives (e.g. after a full stop), // recreate the container on it; otherwise the branch truly doesn't exist. - if !activeStorage().exists(name) { + if !branchExists(name) { return "", fmt.Errorf("branch %q does not exist", name) } - if err := startContainer(name, name == "main"); err != nil { + if err := recreateBranch(name); err != nil { return "", err } - if err := waitReady(name); err != nil { - return "", err - } - // A new container, perhaps from a newer image: the SQL too. - syncIdentityBestEffort(name, true) - applyAuditBestEffort(name) - reconcileGuard(name) default: // exited, created, paused, ... - if err := Resume(name); err != nil { + if OnWake != nil { + OnWake(name, f.state) + } + if err := resumeBranch(name); err != nil { return "", err } - reconcileGuard(name) } return BackendAddr(name) } diff --git a/internal/branch/branch_before.go b/internal/branch/branch_before.go index ab3ee9e3..26d8090a 100644 --- a/internal/branch/branch_before.go +++ b/internal/branch/branch_before.go @@ -595,7 +595,9 @@ func BranchBeforeEntry(src string, entryID int64, newName string, logf func(form "-v", mountpoint(newName)+":/var/lib/postgresql/data", "--entrypoint", "bash", pgImage(), "-c", restoreBeforeScript) - if out, err := exec.Command("sudo", args...).CombinedOutput(); err != nil { + out, err := exec.Command("sudo", args...).CombinedOutput() + noteStateChange() + if err != nil { return fail(fmt.Errorf("starting the restore: %v: %s", err, strings.TrimSpace(string(out)))) } if err := waitPromoted(newName, envDurationOr("FOX_BRANCH_BEFORE_TIMEOUT", 15*time.Minute)); err != nil { diff --git a/internal/branch/maskedbranch.go b/internal/branch/maskedbranch.go index 5c1270d8..b0a66bda 100644 --- a/internal/branch/maskedbranch.go +++ b/internal/branch/maskedbranch.go @@ -143,6 +143,7 @@ func putMaskedBranch(r MaskedBranch) error { m[k] = v } m[r.Name] = r + defer noteStateChange() // the console shows a masked branch's state return saveMaskedBranches(m) } diff --git a/internal/branch/reads.go b/internal/branch/reads.go index 13e482c8..18b0154a 100644 --- a/internal/branch/reads.go +++ b/internal/branch/reads.go @@ -85,7 +85,13 @@ var ErrNoStandbyRead = errors.New("no standby to read from") // StandbyReadAddr is the backend a read-only session should be sent to, or why // it cannot be. func StandbyReadAddr(name string) (string, error) { - if err := standbyReadRefusal(name, primaryBranch(), ContainerState("standby"), IsStandbyNode()); err != nil { + // From the gateway's connect path, so on the short deadline. A refusal is + // all that can come of a poor answer, and the client tries again. + state := "absent" + if f, err := containers.lookup(container("standby")); err == nil { + state = f.state + } + if err := standbyReadRefusal(name, primaryBranch(), state, IsStandbyNode()); err != nil { return "", err } return BackendAddr("standby") diff --git a/internal/branch/statecache.go b/internal/branch/statecache.go new file mode 100644 index 00000000..58deb07f --- /dev/null +++ b/internal/branch/statecache.go @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Every gateway connection and console request used to ask docker where its +// branch was: an inspect for the state, the same inspect again inside +// EnsureRunning, another for the IP. Three `sudo docker` processes per +// connection, each allowed ten minutes, so a dockerd that stopped answering held +// every request for ten minutes. +// +// So one inspect now answers both questions, it gets seconds rather than +// minutes, and what it found is kept briefly for the callers behind it. +// +// The gateway, the control plane and the CLI are separate processes, and this +// memory is not shared between them. A suspend run elsewhere cannot clear it, so +// the rules below are what make keeping it safe: +// +// - Only "running, at this address" is ever kept. A branch seen stopped or +// absent is not remembered at all, so no stale "absent" can exist to send +// anybody down startContainer's `docker rm -f`. +// - What to do about a branch that is not running is decided by probeContainer, +// which always asks docker and shares its answer with nobody. +// - A docker that did not answer is ErrStateUnknown, never "absent". +// - An address that turns out not to work is forgotten and routed again +// (ForgetBackends; the gateway's connectBackend, the console's connectSession). + +// stateCacheTTL is how long a branch seen running is believed without asking +// again. Short because it is the window in which another process's suspend goes +// unnoticed; long enough that a burst of connections costs one inspect. +const stateCacheTTL = 2 * time.Second + +// ErrStateUnknown: docker could not say what state a container is in. Not a +// statement that the container is missing, and the difference matters: "absent" +// is what recreates one. +var ErrStateUnknown = errors.New("could not read the container's state from docker") + +// containerFacts is what one inspect says about a container. +type containerFacts struct { + state string // running | exited | created | paused | ... | absent + ip string // on the engine's docker network; "" when not attached +} + +// serving reports whether these facts can route a connection. +func (f containerFacts) serving() bool { return f.state == "running" && f.ip != "" } + +func (f containerFacts) addr() string { return f.ip + ":5432" } + +// inspectFormat puts a container's name, state and IP on one line. `index` +// rather than the dotted path containerIP uses, because a container not on the +// network has no such key and a dotted path would fail the whole inspect. +var inspectFormat = `{{.Name}}|{{.State.Status}}|{{with index .NetworkSettings.Networks "` + network + `"}}{{.IPAddress}}{{end}}` + +// inspectContainers is the docker call behind every answer here, a variable so +// a test can count it. docker prints what it found and fails over what it did +// not, so the output is worth reading even with an error. +var inspectContainers = func(d time.Duration, conts []string) (string, error) { + args := append([]string{"inspect", "--type", "container", "-f", inspectFormat}, conts...) + return captureWithin(d, "docker", args...) +} + +// parseFacts reads inspectFormat's lines, keyed by container name. +func parseFacts(out string) map[string]containerFacts { + facts := map[string]containerFacts{} + for _, ln := range strings.Split(out, "\n") { + f := strings.Split(strings.TrimSpace(ln), "|") + if len(f) != 3 || f[1] == "" { + continue + } + facts[strings.TrimPrefix(f[0], "/")] = containerFacts{state: f[1], ip: f[2]} + } + return facts +} + +// isNoSuchContainer tells docker saying a container does not exist from docker +// not saying anything. Only the first may be read as "absent". +func isNoSuchContainer(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), "no such") +} + +// probeContainer asks docker now, on a short deadline, and shares the answer +// with nobody. It is what a decision to start or recreate a container has to +// stand on. +func probeContainer(cont string) (containerFacts, error) { + out, err := inspectContainers(execProbe, []string{cont}) + if f, ok := parseFacts(out)[cont]; ok { + return f, nil + } + if isNoSuchContainer(err) { + return containerFacts{state: "absent"}, nil + } + if err == nil { + err = fmt.Errorf("docker inspect said nothing about %s", cont) + } + return containerFacts{}, fmt.Errorf("%w: %v", ErrStateUnknown, err) +} + +// stateGen counts the state changes this process has made. Whatever was read +// under an earlier value is out of date, however recently it was read. +var stateGen atomic.Uint64 + +// stateNow is the clock of everything in this file, so a test can move it. +var stateNow = time.Now + +// noteStateChange says this process has just changed a container, a dataset or +// which container is the primary. It goes after the change, because a read made +// while a `docker stop` is running still says "running". +func noteStateChange() { + containers.mu.Lock() + defer containers.mu.Unlock() + stateGen.Add(1) + clear(containers.entries) +} + +// ForgetBackends drops every address kept here, for a caller that has just +// failed to reach one. The next routing asks docker again. +func ForgetBackends() { noteStateChange() } + +// changesContainers recognises the docker commands that change what is running. +// Checked where every command is run, rather than at each place that stops or +// starts something, so that one added later cannot forget to. `docker run --rm` +// is a tool in a throwaway container, and the backup listing runs one a minute. +func changesContainers(name string, args []string) bool { + if name != "docker" || len(args) == 0 { + return false + } + switch args[0] { + case "start", "stop", "restart", "kill", "rm", "create", "pause", "unpause", "rename": + return true + case "run": + for _, a := range args[1:] { + if a == "--rm" { + return false + } + } + return true + } + return false +} + +// noteIfChange is noteStateChange for a command that has just finished, +// successfully or not: one that failed may still have done part of its work. +func noteIfChange(name string, args []string) { + if changesContainers(name, args) { + noteStateChange() + } +} + +// probeFlight is one inspect in progress, for the callers waiting on it. +type probeFlight struct { + gen uint64 + done chan struct{} + facts containerFacts + err error +} + +// containerCache is the per-container memory: at most one inspect at a time for +// a container, and its answer kept for stateCacheTTL if the container was +// serving. +type containerCache struct { + mu sync.Mutex + entries map[string]cachedFacts + flights map[string]*probeFlight +} + +type cachedFacts struct { + facts containerFacts + expires time.Time +} + +var containers = &containerCache{entries: map[string]cachedFacts{}, flights: map[string]*probeFlight{}} + +// lookup answers from memory or from the inspect already under way, and starts +// one if there is neither. +// +// A caller may only act on a serving answer. Anything else came from nowhere +// that a decision can stand on: it may be another caller's inspect from a moment +// ago. Those go on to probeContainer. +// +// A flight begun before a state change is not joined, and what it finds is not +// kept: it may have read the world from before the change. +func (c *containerCache) lookup(cont string) (containerFacts, error) { + c.mu.Lock() + if e, ok := c.entries[cont]; ok && stateNow().Before(e.expires) { + c.mu.Unlock() + return e.facts, nil + } + gen := stateGen.Load() + if fl, ok := c.flights[cont]; ok && fl.gen == gen { + c.mu.Unlock() + <-fl.done + return fl.facts, fl.err + } + fl := &probeFlight{gen: gen, done: make(chan struct{})} + c.flights[cont] = fl + c.mu.Unlock() + + fl.facts, fl.err = probeContainer(cont) + + c.mu.Lock() + if c.flights[cont] == fl { + delete(c.flights, cont) + } + if fl.err == nil && fl.facts.serving() && fl.gen == stateGen.Load() { + c.entries[cont] = cachedFacts{facts: fl.facts, expires: stateNow().Add(stateCacheTTL)} + } + c.mu.Unlock() + close(fl.done) + return fl.facts, fl.err +} diff --git a/internal/branch/statecache_test.go b/internal/branch/statecache_test.go new file mode 100644 index 00000000..1b1cf0a8 --- /dev/null +++ b/internal/branch/statecache_test.go @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeDocker stands in for inspectContainers: containers it knows answer with +// their state and IP, anything else the way docker says it has no such thing. +type fakeDocker struct { + mu sync.Mutex + state map[string]containerFacts + calls atomic.Int64 + fail error // when set, docker is not answering + gate chan struct{} +} + +func (d *fakeDocker) inspect(_ time.Duration, conts []string) (string, error) { + d.calls.Add(1) + if d.gate != nil { + <-d.gate + } + d.mu.Lock() + defer d.mu.Unlock() + if d.fail != nil { + return "", d.fail + } + var lines []string + var missing error + for _, c := range conts { + f, ok := d.state[c] + if !ok { + missing = fmt.Errorf("exit status 1: Error: No such object: %s", c) + continue + } + lines = append(lines, fmt.Sprintf("/%s|%s|%s", c, f.state, f.ip)) + } + return strings.Join(lines, "\n"), missing +} + +func (d *fakeDocker) set(cont, state, ip string) { + d.mu.Lock() + defer d.mu.Unlock() + d.state[cont] = containerFacts{state: state, ip: ip} +} + +func (d *fakeDocker) remove(cont string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.state, cont) +} + +// withFakeDocker installs a fake docker and a clean memory, and hands back the +// clock so a test can pass the TTL without sleeping. +func withFakeDocker(t *testing.T) (*fakeDocker, func(time.Duration)) { + t.Helper() + d := &fakeDocker{state: map[string]containerFacts{}} + oldInspect, oldNow := inspectContainers, stateNow + oldExists, oldRecreate, oldResume := branchExists, recreateBranch, resumeBranch + now := time.Unix(1_000_000, 0) + var clock sync.Mutex + inspectContainers = d.inspect + stateNow = func() time.Time { clock.Lock(); defer clock.Unlock(); return now } + branchExists = func(string) bool { return true } + recreateBranch = func(n string) error { t.Fatalf("recreated %q unexpectedly", n); return nil } + resumeBranch = func(n string) error { t.Fatalf("resumed %q unexpectedly", n); return nil } + noteStateChange() + t.Cleanup(func() { + inspectContainers, stateNow = oldInspect, oldNow + branchExists, recreateBranch, resumeBranch = oldExists, oldRecreate, oldResume + noteStateChange() + }) + return d, func(by time.Duration) { clock.Lock(); defer clock.Unlock(); now = now.Add(by) } +} + +func TestParseFacts(t *testing.T) { + got := parseFacts("/pg-a|running|172.18.0.5\n/pg-b|exited|\n\nnot a line") + if got["pg-a"] != (containerFacts{"running", "172.18.0.5"}) { + t.Errorf("pg-a = %+v", got["pg-a"]) + } + if got["pg-b"] != (containerFacts{"exited", ""}) { + t.Errorf("pg-b = %+v", got["pg-b"]) + } + if len(got) != 2 { + t.Errorf("%d containers parsed, want 2: %v", len(got), got) + } +} + +// Within the TTL a connection costs no docker call at all; past it, one. +func TestARunningBranchIsKeptForTheTTL(t *testing.T) { + d, advance := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + for range 5 { + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.1:5432" { + t.Fatalf("EnsureRunning = %q, %v", addr, err) + } + } + if n := d.calls.Load(); n != 1 { + t.Fatalf("%d docker calls for 5 connections, want 1", n) + } + advance(stateCacheTTL + time.Millisecond) + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + if n := d.calls.Load(); n != 2 { + t.Fatalf("%d docker calls after the TTL, want 2", n) + } +} + +// Anything the engine does to a container is seen at once, without waiting for +// the TTL. +func TestAStateChangeInvalidates(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + d.set("pg-a", "running", "10.0.0.9") + noteIfChange("docker", []string{"start", "pg-a"}) + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.9:5432" { + t.Fatalf("EnsureRunning = %q, %v, want the new address", addr, err) + } +} + +// The commands that change what is running, recognised where commands are run, +// so that none needs remembering at its call site. A tool in a throwaway +// container changes nothing anybody is routed to, and reads certainly do not. +func TestChangesContainers(t *testing.T) { + yes := [][]string{{"docker", "stop", "x"}, {"docker", "start", "x"}, {"docker", "restart", "x"}, {"docker", "kill", "-s", "KILL", "x"}, + {"docker", "rm", "-f", "-v", "x"}, {"docker", "run", "-d", "--name", "x", "img"}, {"docker", "rename", "x", "y"}} + no := [][]string{{"docker", "run", "--rm", "img"}, {"docker", "inspect", "x"}, {"docker", "exec", "x", "psql"}, {"docker", "ps"}, + {"docker", "network", "create", "n"}, {"docker"}, {"zfs", "stop"}} + for _, c := range yes { + if !changesContainers(c[0], c[1:]) { + t.Errorf("%v should invalidate", c) + } + } + for _, c := range no { + if changesContainers(c[0], c[1:]) { + t.Errorf("%v should not invalidate", c) + } + } +} + +// A hundred connections arriving together, with nothing remembered, share one +// inspect. +func TestConcurrentCallersShareOneInspect(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + d.gate = make(chan struct{}) + const n = 100 + var wg sync.WaitGroup + wg.Add(n) + errs := make(chan error, n) + for range n { + go func() { + defer wg.Done() + if addr, err := EnsureRunning("a"); err != nil || addr != "10.0.0.1:5432" { + errs <- fmt.Errorf("%q, %v", addr, err) + } + }() + } + // Every caller queued, or as many as will: then let the inspect finish. + for d.calls.Load() == 0 { + time.Sleep(time.Millisecond) + } + time.Sleep(20 * time.Millisecond) + close(d.gate) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + if got := d.calls.Load(); got != 1 { + t.Fatalf("%d inspects for %d concurrent connections, want 1", got, n) + } +} + +// An inspect that began before a state change read the world from before it, +// so nobody joins it afterwards and its answer is not kept. +func TestAnInspectFromBeforeAChangeIsNotKept(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + d.gate = make(chan struct{}) + first := make(chan string, 1) + go func() { + a, _ := EnsureRunning("a") + first <- a + }() + for d.calls.Load() == 0 { + time.Sleep(time.Millisecond) + } + // The change lands while that inspect is out. + d.set("pg-a", "running", "10.0.0.9") + noteStateChange() + close(d.gate) + <-first + d.gate = nil + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.9:5432" { + t.Fatalf("EnsureRunning = %q, %v: kept an answer from before the change", addr, err) + } +} + +// Another process stopped the branch: the memory still says running and the +// caller is given the old address, which fails to dial. ForgetBackends sends it +// back through the real path, which wakes the branch. +func TestAForgottenBackendIsWokenThroughTheRealPath(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + d.set("pg-a", "exited", "") // another process's suspend, unseen here + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.1:5432" { + t.Fatalf("within the TTL: %q, %v, want the remembered address", addr, err) + } + woken := 0 + resumeBranch = func(string) error { woken++; d.set("pg-a", "running", "10.0.0.2"); noteStateChange(); return nil } + ForgetBackends() + addr, err = EnsureRunning("a") + if err != nil || addr != "10.0.0.2:5432" || woken != 1 { + t.Fatalf("after forgetting: %q, %v, woken %d", addr, err, woken) + } +} + +// Nothing but a fresh answer from docker can recreate a container, whose first +// step is `docker rm -f`. +func TestAbsentIsOnlyBelievedFromAFreshAnswer(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + // Absent never goes into the memory: the only way to the destructive path + // is an inspect made during this very call. + d.remove("pg-a") + ForgetBackends() + before := d.calls.Load() + recreated := 0 + recreateBranch = func(string) error { recreated++; d.set("pg-a", "running", "10.0.0.3"); noteStateChange(); return nil } + if addr, err := EnsureRunning("a"); err != nil || addr != "10.0.0.3:5432" || recreated != 1 { + t.Fatalf("EnsureRunning = %q, %v, recreated %d", addr, err, recreated) + } + if n := d.calls.Load() - before; n < 2 { + t.Fatalf("recreated on %d docker calls: the decision needs its own", n) + } + containers.mu.Lock() + for cont, e := range containers.entries { + if !e.facts.serving() { + t.Errorf("%s remembered as %q", cont, e.facts.state) + } + } + containers.mu.Unlock() +} + +// A dockerd that has stopped answering looks like every container vanishing to +// anything that treats an error as "absent", and the next step of that would be +// removing them. It is an error, and nothing is touched. +func TestDockerNotAnsweringIsNeverAbsent(t *testing.T) { + d, _ := withFakeDocker(t) + d.fail = errors.New("docker did not finish within 5s and was stopped") + // withFakeDocker fails the test if either way up is taken. + _, err := EnsureRunning("a") + if !errors.Is(err, ErrStateUnknown) { + t.Fatalf("EnsureRunning = %v, want ErrStateUnknown", err) + } + if _, err := BackendAddr("a"); !errors.Is(err, ErrStateUnknown) { + t.Fatalf("BackendAddr = %v, want ErrStateUnknown", err) + } +} + +// Docker saying there is no such container, by contrast, is an answer. +func TestNoSuchContainerIsAbsent(t *testing.T) { + d, _ := withFakeDocker(t) + f, err := probeContainer("pg-gone") + if err != nil || f.state != "absent" { + t.Fatalf("probeContainer = %+v, %v", f, err) + } + if d.calls.Load() != 1 { + t.Fatalf("%d calls", d.calls.Load()) + } +} + +// A branch that does not exist is still reported as one. +func TestAMissingBranchSaysSo(t *testing.T) { + _, _ = withFakeDocker(t) + branchExists = func(string) bool { return false } + if _, err := EnsureRunning("a"); err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Fatalf("EnsureRunning = %v", err) + } +} + +// A failover changes which container "main" is. Read each time, so a promotion +// done by another process needs no invalidation here. +func TestMainFollowsThePrimaryContainer(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-main", "running", "10.0.0.1") + d.set("pg-standby", "running", "10.0.0.2") + if addr, err := BackendAddr("main"); err != nil || addr != "10.0.0.1:5432" { + t.Fatalf("BackendAddr = %q, %v", addr, err) + } + if err := setPrimary("standby"); err != nil { + t.Fatal(err) + } + defer setPrimary("main") + if addr, err := BackendAddr("main"); err != nil || addr != "10.0.0.2:5432" { + t.Fatalf("after failover BackendAddr = %q, %v", addr, err) + } +} + +// The measurement behind the change: docker processes for 100 connections to a +// running branch, before and after. Run with -v to read the numbers. +func TestConnectPathDockerCalls(t *testing.T) { + d, advance := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + const connections = 100 + for i := range connections { + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + if i%10 == 9 { // ten connections a second + advance(time.Second) + } + } + got := d.calls.Load() + legacy := int64(connections * 3) // state in the proxy, state again, then the IP + t.Logf("connect path: %d connections at 10/s: %d docker calls, was %d", connections, got, legacy) + if got > connections/10 { + t.Fatalf("%d docker calls", got) + } +} From 5753c07635e1d7258a7440f6b94863fe2d643d73 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:57:28 +0530 Subject: [PATCH 09/22] perf(gateway): recover from a stale backend address The gateway no longer asks for a branch's state before EnsureRunning asks again. A remembered address can be left behind by another process's suspend, so a failed dial forgets it and routes once more, which wakes a suspended branch without the client seeing an error. The console's sessions do the same. A docker that did not answer tells the client to come back rather than that the branch does not exist. Co-Authored-By: Claude Opus 5.5 --- internal/controlplane/server.go | 39 ++++++++- internal/controlplane/stale_test.go | 75 ++++++++++++++++ internal/proxy/proxy.go | 131 ++++++++++++++++++++-------- internal/proxy/stale_test.go | 121 +++++++++++++++++++++++++ 4 files changed, 331 insertions(+), 35 deletions(-) create mode 100644 internal/controlplane/stale_test.go create mode 100644 internal/proxy/stale_test.go diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 7469fa7d..964a7628 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -12,6 +12,7 @@ import ( "fmt" "io/fs" "log" + "net" "net/http" "net/url" "path" @@ -612,6 +613,42 @@ func registerAPI(mux *http.ServeMux) { // statement. const maskedStatementSQL = `'(hidden from masked accounts: a statement can hold real values)'::text` +// The console's way out to a branch, variables so a test can stand in for one +// that moved. +var ( + pgConnect = pgx.ConnectConfig + sessionRoute = branch.EnsureRunning +) + +// connectSession connects, and if the address could not be dialled, forgets it, +// routes the branch again and tries once more. The address may be seconds old +// (branch/statecache.go), and a suspend run by the gateway's idle reaper leaves +// it behind without this process hearing about it. Only for a failure to reach +// the address: a refused password or a full server is an answer from a backend +// that is there. +func connectSession(ctx context.Context, cfg *pgx.ConnConfig, name string) (*pgx.Conn, error) { + conn, err := pgConnect(ctx, cfg) + if err == nil || name == "" || !isDialFailure(err) { + return conn, err + } + branch.ForgetBackends() + addr, rerr := sessionRoute(name) + if rerr != nil { + return nil, err + } + again, perr := pgx.ParseConfig(fmt.Sprintf("postgres://%s/%s", addr, branch.Database)) + if perr != nil { + return nil, err + } + again.User, again.Password = cfg.User, cfg.Password + return pgConnect(ctx, again) +} + +func isDialFailure(err error) bool { + var op *net.OpError + return errors.As(err, &op) && op.Op == "dial" +} + func sqlEsc(s string) string { return strings.ReplaceAll(s, "'", "''") } // ledgerSQL builds a filtered, bounded query over bb.schema_ledger. Filter @@ -867,7 +904,7 @@ func openSession(ctx context.Context, addr string, as queryAs) (*pgx.Conn, error } cfg.User, cfg.Password = as.Actor, branch.UserRolePassword(as.Actor) } - conn, err := pgx.ConnectConfig(ctx, cfg) + conn, err := connectSession(ctx, cfg, as.Branch) if err != nil { return nil, err } diff --git a/internal/controlplane/stale_test.go b/internal/controlplane/stale_test.go new file mode 100644 index 00000000..8e06c509 --- /dev/null +++ b/internal/controlplane/stale_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controlplane + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +// A branch the gateway's reaper suspended leaves the console holding an address +// that no longer answers. The console routes again rather than showing an error. +func TestConsoleRecoversFromAStaleAddress(t *testing.T) { + oldConnect, oldRoute := pgConnect, sessionRoute + t.Cleanup(func() { pgConnect, sessionRoute = oldConnect, oldRoute }) + + var hosts []string + pgConnect = func(_ context.Context, c *pgx.ConnConfig) (*pgx.Conn, error) { + hosts = append(hosts, c.Host) + if c.Host == "10.0.0.1" { + return nil, &net.OpError{Op: "dial", Err: errors.New("no route to host")} + } + return nil, nil + } + sessionRoute = func(string) (string, error) { return "10.0.0.2:5432", nil } + + cfg, err := pgx.ParseConfig("postgres://u:p@10.0.0.1:5432/d") + if err != nil { + t.Fatal(err) + } + if _, err := connectSession(context.Background(), cfg, "dev"); err != nil { + t.Fatalf("a stale address reached the user: %v", err) + } + if len(hosts) != 2 || hosts[1] != "10.0.0.2" { + t.Fatalf("connected to %v, want the old address and then the new one", hosts) + } +} + +// An answer from a backend that is there is not a stale address. +func TestConsoleDoesNotRetryAnAnswer(t *testing.T) { + oldConnect, oldRoute := pgConnect, sessionRoute + t.Cleanup(func() { pgConnect, sessionRoute = oldConnect, oldRoute }) + calls := 0 + refused := errors.New("password authentication failed") + pgConnect = func(context.Context, *pgx.ConnConfig) (*pgx.Conn, error) { calls++; return nil, refused } + sessionRoute = func(string) (string, error) { t.Fatal("routed again after an answer"); return "", nil } + cfg, _ := pgx.ParseConfig("postgres://u:p@10.0.0.1:5432/d") + if _, err := connectSession(context.Background(), cfg, "dev"); !errors.Is(err, refused) || calls != 1 { + t.Fatalf("err %v after %d connects", err, calls) + } +} + +// The check has to recognise the error pgx really returns, not just one built +// to look like it. +func TestIsDialFailureOnARealRefusal(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip(err) + } + addr := ln.Addr().String() + ln.Close() // nothing listens there now + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = pgx.Connect(ctx, "postgres://u:p@"+addr+"/d?sslmode=disable") + if err == nil { + t.Skip("something answered on a closed port") + } + if !isDialFailure(err) { + t.Fatalf("not recognised as a dial failure: %T %v", err, err) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index c3e44f51..aaca0ddb 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -174,6 +174,7 @@ func relayStartupCaptureKey(client, backend net.Conn, addr string) (string, erro // auto-resuming suspended branches on connect and auto-suspending branches idle // for longer than idle (0 disables suspension). func Serve(addr string, idle time.Duration) error { + branch.OnWake = func(name, state string) { log.Printf("auto-resume: waking %s (was %s)", name, state) } if gatewayNoAuth() { log.Printf("gateway authentication DISABLED (FOX_GATEWAY_NOAUTH) — trusted/local mode") } else { @@ -350,43 +351,27 @@ func handle(client net.Conn) { return } touch(target) - var addr string - if standbyRead { - // Never EnsureRunning: the standby is not a branch, nothing may start - // it on a client's behalf, and a stepped-down primary must not be - // revived by somebody opening a read-only session. - a, err := branch.StandbyReadAddr(target) - if err != nil { - log.Printf("route dbname=%q: %v", branch.StandbyReadName(target), err) - // A refusal that retrying cannot change is a name problem; one - // that waiting can — a standby mid-rebuild, or promoted and not - // yet failed back — tells the client to come back instead. - code := "57P03" // cannot_connect_now - if errors.Is(err, branch.ErrNoStandbyRead) { - code = "3D000" // invalid_catalog_name - } - sendError(client, code, err.Error()) - return - } - addr = a - } else { - if st := branch.ContainerState(target); st != "running" && st != "absent" { - log.Printf("auto-resume: waking %s (was %s)", target, st) - } - a, err := branch.EnsureRunning(target) // resumes the branch if suspended - if err != nil { - log.Printf("route dbname=%q: %v", target, err) - if errors.Is(err, branch.ErrBranchClosed) { - sendError(client, "57P03", err.Error()) // cannot_connect_now: a masked branch being made - return - } - sendError(client, "3D000", err.Error()) // invalid_catalog_name - return + route := func() (string, error) { + if standbyRead { + // Never EnsureRunning: the standby is not a branch, nothing may + // start it on a client's behalf, and a stepped-down primary must + // not be revived by somebody opening a read-only session. + return standbyAddr(target) } - addr = a + return ensureRunning(target) // resumes the branch if suspended + } + addr, err := route() + if err != nil { + sendRouteError(client, target, standbyRead, err) + return } - backend, err := net.Dial("tcp", addr) + backend, addr, err := connectBackend(addr, route) if err != nil { + var refused *routeFailure + if errors.As(err, &refused) { + sendRouteError(client, target, standbyRead, refused.err) + return + } log.Printf("dial backend %s: %v", addr, err) sendError(client, "08006", fmt.Sprintf("could not reach branch %q", target)) return @@ -502,6 +487,84 @@ func handle(client net.Conn) { <-done } +// The dials are bounded, unlike the rest of a session. The first is short +// because a backend on the same docker bridge answers in a millisecond, and the +// one that does not is an address the container has left. The second has come +// through a fresh look at docker and is allowed more. +const ( + backendDialFirst = 3 * time.Second + backendDialRetry = 10 * time.Second +) + +// The gateway's ways out to the engine, variables so a test can stand in for a +// branch that moved. +var ( + ensureRunning = branch.EnsureRunning + standbyAddr = branch.StandbyReadAddr + forgetBackends = branch.ForgetBackends + dialBackend = net.DialTimeout +) + +// routeFailure is a retry that could not find the branch at all, which the +// client is told about as a routing problem rather than a dial failure. +type routeFailure struct{ err error } + +func (r *routeFailure) Error() string { return r.err.Error() } + +// connectBackend dials addr, and when that fails, routes again and dials once +// more. +// +// The address may be seconds old (branch/statecache.go): a suspend or a restart +// run by another process leaves it behind and nothing can tell this process. So +// a failed dial says the memory was wrong, and it is dropped before routing +// again, which then finds a suspended branch and wakes it. The client sees a +// slightly slower connect instead of an error. Once only: a backend that fails +// twice is down, and more tries would only make the client wait to hear it. +func connectBackend(addr string, route func() (string, error)) (net.Conn, string, error) { + backend, err := dialBackend("tcp", addr, backendDialFirst) + if err == nil { + return backend, addr, nil + } + log.Printf("dial backend %s: %v (looking the branch up again)", addr, err) + forgetBackends() + again, rerr := route() + if rerr != nil { + return nil, addr, &routeFailure{err: rerr} + } + backend, err = dialBackend("tcp", again, backendDialRetry) + if err != nil { + return nil, again, err + } + return backend, again, nil +} + +// sendRouteError tells the client why its branch could not be routed to. +func sendRouteError(client net.Conn, target string, standbyRead bool, err error) { + name := target + if standbyRead { + name = branch.StandbyReadName(target) + } + log.Printf("route dbname=%q: %v", name, err) + sendError(client, routeErrorCode(standbyRead, err), err.Error()) +} + +// routeErrorCode separates a refusal that retrying cannot change, which is a +// name problem, from one that waiting can — a standby mid-rebuild or promoted +// and not yet failed back, a masked branch being made, a docker that did not +// answer in time — which tells the client to come back instead. +func routeErrorCode(standbyRead bool, err error) string { + const cannotConnectNow, invalidCatalog = "57P03", "3D000" + switch { + case errors.Is(err, branch.ErrStateUnknown): + return cannotConnectNow + case standbyRead && !errors.Is(err, branch.ErrNoStandbyRead): + return cannotConnectNow + case errors.Is(err, branch.ErrBranchClosed): + return cannotConnectNow + } + return invalidCatalog +} + // readStartup reads the startup phase and returns the startup parameters (user, // database, ...). When TLS is configured it accepts an SSLRequest, upgrades the // connection, and returns the wrapped conn — the caller must use the returned diff --git a/internal/proxy/stale_test.go b/internal/proxy/stale_test.go new file mode 100644 index 00000000..9f605697 --- /dev/null +++ b/internal/proxy/stale_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxy + +import ( + "errors" + "net" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// stubDial records the addresses dialled and fails the ones in down. +func stubDial(t *testing.T, down ...string) *[]string { + t.Helper() + dialled := &[]string{} + old, oldForget := dialBackend, forgetBackends + dialBackend = func(_, addr string, _ time.Duration) (net.Conn, error) { + *dialled = append(*dialled, addr) + for _, d := range down { + if d == addr { + return nil, errors.New("connect: no route to host") + } + } + a, b := net.Pipe() + t.Cleanup(func() { a.Close(); b.Close() }) + return a, nil + } + t.Cleanup(func() { dialBackend, forgetBackends = old, oldForget }) + return dialled +} + +// A branch another process suspended or moved leaves a remembered address that +// no longer answers. The client must not hear about that: the address is +// forgotten, the branch routed again, and the connection made. +func TestAStaleAddressIsForgottenAndRoutedAgain(t *testing.T) { + dialled := stubDial(t, "10.0.0.1:5432") + forgot := 0 + forgetBackends = func() { forgot++ } + routes := 0 + conn, addr, err := connectBackend("10.0.0.1:5432", func() (string, error) { + routes++ + return "10.0.0.2:5432", nil + }) + if err != nil { + t.Fatalf("a stale address reached the client: %v", err) + } + defer conn.Close() + if addr != "10.0.0.2:5432" || routes != 1 || forgot != 1 { + t.Fatalf("addr %s after %d routes and %d forgets, want the new address after one of each", addr, routes, forgot) + } + if len(*dialled) != 2 { + t.Fatalf("dialled %v, want the old address and then the new one", *dialled) + } +} + +// A working address costs nothing but the dial. +func TestAGoodAddressIsNotRoutedAgain(t *testing.T) { + stubDial(t) + forgetBackends = func() { t.Fatal("forgot a working address") } + conn, _, err := connectBackend("10.0.0.1:5432", func() (string, error) { + t.Fatal("routed again after a good dial") + return "", nil + }) + if err != nil { + t.Fatal(err) + } + conn.Close() +} + +// One retry only: a backend that is really down is reported, not chased. +func TestABackendDownTwiceIsReported(t *testing.T) { + dialled := stubDial(t, "10.0.0.1:5432") + forgetBackends = func() {} + _, addr, err := connectBackend("10.0.0.1:5432", func() (string, error) { return "10.0.0.1:5432", nil }) + if err == nil { + t.Fatal("two failed dials were reported as a connection") + } + var refused *routeFailure + if errors.As(err, &refused) { + t.Fatalf("a dial failure was reported as a routing problem: %v", err) + } + if addr != "10.0.0.1:5432" || len(*dialled) != 2 { + t.Fatalf("addr %q, dialled %v: want exactly two tries", addr, *dialled) + } +} + +// A retry can find the branch gone, which is a routing answer. +func TestARouteFailureOnRetryIsARoutingAnswer(t *testing.T) { + stubDial(t, "10.0.0.1:5432") + forgetBackends = func() {} + gone := errors.New("branch does not exist") + _, _, err := connectBackend("10.0.0.1:5432", func() (string, error) { return "", gone }) + var refused *routeFailure + if !errors.As(err, &refused) || !errors.Is(refused.err, gone) { + t.Fatalf("err = %v, want the routing failure", err) + } +} + +func TestRouteErrorCodes(t *testing.T) { + cases := []struct { + name string + standby bool + err error + want string + }{ + {"no such branch", false, errors.New("branch does not exist"), "3D000"}, + {"masked branch being made", false, branch.ErrBranchClosed, "57P03"}, + // A docker that did not answer is not a branch that does not exist: + // the client should come back. + {"docker did not answer", false, branch.ErrStateUnknown, "57P03"}, + {"no standby here", true, branch.ErrNoStandbyRead, "3D000"}, + {"standby mid-rebuild", true, errors.New("rebuilding"), "57P03"}, + } + for _, c := range cases { + if got := routeErrorCode(c.standby, c.err); got != c.want { + t.Errorf("%s: %s, want %s", c.name, got, c.want) + } + } +} From 303867d11f6c54718363bea59a5844a766d6d938 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:57:51 +0530 Subject: [PATCH 10/22] fix(promote): refuse a request whose statements leave its transaction An entry's statement is the source's whole current_query(), which can hold a COMMIT, a SET ROLE or a SET bb.actor beside the change it was recorded for. Replayed onto the target, the first ended the one transaction a request is applied in and the others changed who the rest ran or was recorded as. Such a request is now refused when it is asked for and again when it is applied, from a Go reading of the message that follows the server's scanner, comments, quoting and continuations included. What it cannot read is refused. Audit TX-5. Co-Authored-By: Claude Opus 5.5 --- enterprise/promote/promote.go | 7 + enterprise/promote/replaycheck.go | 517 +++++++++++++++++++++++++ enterprise/promote/replaycheck_test.go | 118 ++++++ internal/branch/promote.go | 7 + internal/controlplane/promote.go | 2 +- 5 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 enterprise/promote/replaycheck.go create mode 100644 enterprise/promote/replaycheck_test.go diff --git a/enterprise/promote/promote.go b/enterprise/promote/promote.go index 227102cc..3b237c60 100644 --- a/enterprise/promote/promote.go +++ b/enterprise/promote/promote.go @@ -126,6 +126,9 @@ func build(source, target string, history []auth.ChangeRequest) (entries []branc return nil, 0, fmt.Errorf("%w: %q has no schema change that %q does not already have", branch.ErrNothingToPromote, source, target) } + if err := checkReplayable(entries); err != nil { + return nil, 0, err + } xids, err := runXids(source, forkAfterID) if err != nil { return nil, 0, fmt.Errorf("reading %q's Blackbox: %w", source, err) @@ -219,6 +222,10 @@ func apply(requestID int64, target, approver string, entries []branch.RequestEnt if len(entries) == 0 { return 0, branch.ErrNothingToPromote } + // Again here: what is applied is a stored snapshot, and may predate this check. + if err := checkReplayable(entries); err != nil { + return 0, err + } target, err := branch.ResolveBranch(target) if err != nil { return 0, err diff --git a/enterprise/promote/replaycheck.go b/enterprise/promote/replaycheck.go new file mode 100644 index 00000000..9a399f67 --- /dev/null +++ b/enterprise/promote/replaycheck.go @@ -0,0 +1,517 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package promote + +import ( + "errors" + "fmt" + "strings" + + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/brand" +) + +// What a request may carry. +// +// An entry's statement is the source's whole current_query(): everything its +// author sent in one message, which can be more than the change it was recorded +// for. Replayed onto the target, a COMMIT in it ends the one transaction the +// request is applied in, and a SET ROLE or SET bb.actor changes who the rest +// runs as, or is recorded as. So a request is refused if any statement in any +// entry would do that: when it is asked for, and again when it is applied, +// because what is applied is a stored snapshot. +// +// This is the Go counterpart of bb._sql_statements, and stricter, because here a +// misread is a way in rather than a wrong risk label. It follows the server's +// own lexer where that function does not: a line comment also ends at a carriage +// return, an E'' string keeps its escapes across a newline-separated +// continuation, an identifier runs through any '$' in it, and a semicolon inside +// parentheses or a BEGIN ATOMIC body does not end the statement. Text it cannot +// read is refused, not passed. +// +// What it cannot see is SQL assembled at run time, or held in a function that the +// same message calls. Those run with the applying role's rights and nothing more: +// a COMMIT there is an error inside a transaction block, and the role has no +// other role to become (apply). + +type tokenKind int + +const ( + tokWord tokenKind = iota // a keyword or unquoted name, in lower case + tokQuoted // a "quoted name", in lower case: settings ignore case + tokEscaped // a U&"name" with escapes, whose meaning is not worked out + tokString // any string literal + tokPunct // one character of anything else +) + +type token struct { + kind tokenKind + text string + // plain is a string token whose text is exactly its value: no escapes to + // interpret. Set for '…' and $…$ only. + plain bool +} + +func (t token) is(kind tokenKind, text string) bool { return t.kind == kind && t.text == text } + +var errUnreadable = errors.New("its quoting or comments do not close") + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' +} +func isDigit(c byte) bool { return c >= '0' && c <= '9' } +func isTagStart(c byte) bool { return c == '_' || c >= 0x80 || lower(c) >= 'a' && lower(c) <= 'z' } +func isIdentCont(c byte) bool { return isTagStart(c) || isDigit(c) || c == '$' } + +func lower(c byte) byte { + if c >= 'A' && c <= 'Z' { + return c + 'a' - 'A' + } + return c +} + +// lexSQL splits a message into tokens as the server's scanner would. +func lexSQL(q string) ([]token, error) { + if strings.IndexByte(q, 0) >= 0 { + return nil, errors.New("it holds a NUL byte") + } + var out []token + n := len(q) + for i := 0; i < n; { + c := q[i] + switch { + case isSpace(c): + i++ + case strings.HasPrefix(q[i:], "--"): + i = lineEnd(q, i) + case strings.HasPrefix(q[i:], "/*"): + end, err := blockCommentEnd(q, i) + if err != nil { + return nil, err + } + i = end + case c == '\'': + t, end, err := lexString(q, i, false) + if err != nil { + return nil, err + } + t.plain = true + out, i = append(out, t), end + case c == '"': + t, end, err := lexQuoted(q, i) + if err != nil { + return nil, err + } + out, i = append(out, t), end + case c == '$': + t, end, err := lexDollar(q, i) + if err != nil { + return nil, err + } + out, i = append(out, t), end + case isDigit(c): + for i < n && isDigit(q[i]) { + i++ + } + case isTagStart(c): + t, end, err := lexWord(q, i) + if err != nil { + return nil, err + } + out, i = append(out, t), end + default: + out = append(out, token{kind: tokPunct, text: string(c)}) + i++ + } + } + return out, nil +} + +func lineEnd(q string, i int) int { + for i < len(q) && q[i] != '\n' && q[i] != '\r' { + i++ + } + return i +} + +// blockCommentEnd finds the end of a /* comment */, which nests. +func blockCommentEnd(q string, i int) (int, error) { + depth := 1 + for i += 2; i < len(q); { + switch { + case strings.HasPrefix(q[i:], "/*"): + depth++ + i += 2 + case strings.HasPrefix(q[i:], "*/"): + depth-- + i += 2 + if depth == 0 { + return i, nil + } + default: + i++ + } + } + return 0, errUnreadable +} + +// lexWord reads a name or keyword, and the literal it prefixes if it is one: +// the name is taken whole first, so `note'x'` is not an E” string. +func lexWord(q string, i int) (token, int, error) { + start := i + for i < len(q) && isIdentCont(q[i]) { + i++ + } + word := strings.ToLower(q[start:i]) + if i < len(q) && q[i] == '\'' { + switch word { + case "e": + return lexString(q, i, true) + case "b", "x", "n": + return lexString(q, i, false) + } + } + if word == "u" && i+1 < len(q) && q[i] == '&' { + switch q[i+1] { + case '\'': + return lexString(q, i+1, false) + case '"': + t, end, err := lexQuoted(q, i+1) + if err == nil && strings.Contains(t.text, `\`) { + t.kind = tokEscaped + } + return t, end, err + } + } + return token{kind: tokWord, text: word}, i, nil +} + +// lexString reads a string from its opening quote, with any continuation: two +// literals separated by a newline are one, and the first's rules hold in both. +func lexString(q string, i int, escapes bool) (token, int, error) { + var b strings.Builder + for i++; i < len(q); { + c := q[i] + switch { + case escapes && c == '\\': + if i+1 >= len(q) { + return token{}, 0, errUnreadable + } + b.WriteString(q[i : i+2]) + i += 2 + case c == '\'' && i+1 < len(q) && q[i+1] == '\'': + b.WriteByte('\'') + i += 2 + case c == '\'': + next, ok := stringContinues(q, i+1) + if !ok { + return token{kind: tokString, text: b.String()}, i + 1, nil + } + i = next + 1 + default: + b.WriteByte(c) + i++ + } + } + return token{}, 0, errUnreadable +} + +// stringContinues reports whether what follows a closing quote carries on the +// string: whitespace holding a newline, then a quote. It returns that quote. +// Line comments count as whitespace here, block comments do not. +func stringContinues(q string, i int) (int, bool) { + newline := false + for i < len(q) { + switch { + case q[i] == '\n' || q[i] == '\r': + newline = true + i++ + case isSpace(q[i]): + i++ + case strings.HasPrefix(q[i:], "--"): + i = lineEnd(q, i) + default: + return i, newline && q[i] == '\'' + } + } + return i, false +} + +func lexQuoted(q string, i int) (token, int, error) { + var b strings.Builder + for i++; i < len(q); i++ { + if q[i] != '"' { + b.WriteByte(q[i]) + continue + } + if i+1 < len(q) && q[i+1] == '"' { + b.WriteByte('"') + i++ + continue + } + return token{kind: tokQuoted, text: strings.ToLower(b.String())}, i + 1, nil + } + return token{}, 0, errUnreadable +} + +// lexDollar reads a $tag$ string, or a parameter or stray dollar if it is not one. +func lexDollar(q string, i int) (token, int, error) { + j := i + 1 + if j < len(q) && isTagStart(q[j]) { + for j < len(q) && (isTagStart(q[j]) || isDigit(q[j])) { + j++ + } + } + if j < len(q) && q[j] == '$' { + delim := q[i : j+1] + end := strings.Index(q[j+1:], delim) + if end < 0 { + return token{}, 0, errUnreadable + } + return token{kind: tokString, text: q[j+1 : j+1+end], plain: true}, j + 1 + end + len(delim), nil + } + for j = i + 1; j < len(q) && isDigit(q[j]); j++ { + } + return token{kind: tokPunct, text: "$"}, j, nil +} + +// splitStatements cuts a message at its top-level semicolons. A semicolon inside +// parentheses belongs to a CREATE RULE, and one in a BEGIN ATOMIC body to the +// function: neither ends the statement, and the grammar allows neither +// transaction control. Anything else that hides a semicolon that way does not +// parse, and the server rejects the whole message before running any of it. +func splitStatements(toks []token) [][]token { + var out [][]token + start, parens, atomic := 0, 0, 0 + for i, t := range toks { + switch { + case t.is(tokPunct, "("): + parens++ + case t.is(tokPunct, ")"): + if parens > 0 { + parens-- + } + case parens > 0: + case t.is(tokWord, "begin") && atomic == 0 && i+1 < len(toks) && + toks[i+1].is(tokWord, "atomic") && isRoutine(toks[start:]): + atomic = 1 + case t.is(tokWord, "case") && atomic > 0: + atomic++ + case t.is(tokWord, "end") && atomic > 0: + atomic-- + case t.is(tokPunct, ";") && atomic == 0: + if i > start { + out = append(out, toks[start:i]) + } + start = i + 1 + } + } + if start < len(toks) { + out = append(out, toks[start:]) + } + return out +} + +// isRoutine reports whether a statement is CREATE [OR REPLACE] FUNCTION or +// PROCEDURE, the only ones BEGIN ATOMIC can belong to. +func isRoutine(s []token) bool { + word := func(i int) string { return wordAt(s, i) } + if word(0) != "create" { + return false + } + i := 1 + if word(1) == "or" && word(2) == "replace" { + i = 3 + } + return word(i) == "function" || word(i) == "procedure" +} + +func wordAt(s []token, i int) string { + if i < len(s) && s[i].kind == tokWord { + return s[i].text + } + return "" +} + +// transactionControl is every statement that starts, ends or rewinds a +// transaction, by its first word. A savepoint counts: rolling back to one undoes +// entries the request would still report as applied. +var transactionControl = map[string]bool{ + "begin": true, "start": true, "commit": true, "end": true, "rollback": true, "abort": true, + "savepoint": true, "release": true, +} + +// refusedSettings are the settings a replayed statement may not change. A role +// or session authorization is who the statement runs as, the bb.* settings (by +// prefix) and application_name are who the Blackbox records and whether the +// guardrail is overridden, and session_replication_role switches triggers off. +// The two encodings change how the server reads the entries after this one, and +// so whether this check read them the same way. +var refusedSettings = map[string]bool{ + "role": true, "session_authorization": true, "session_replication_role": true, "application_name": true, + "standard_conforming_strings": true, "client_encoding": true, "names": true, "transaction": true, +} + +func refusedSetting(name string) bool { + return refusedSettings[name] || strings.HasPrefix(name, "bb.") +} + +// statementRefusal says why one statement cannot be replayed, or "" if it can. +func statementRefusal(s []token) string { + head := wordAt(s, 0) + switch { + case transactionControl[head]: + return fmt.Sprintf("%s is transaction control, and a request is applied in one transaction", strings.ToUpper(head)) + case head == "prepare" && wordAt(s, 1) == "transaction": + return "PREPARE TRANSACTION is transaction control, and a request is applied in one transaction" + case head == "set": + return setRefusal(s[1:], "SET") + case head == "reset": + return setRefusal(s[1:], "RESET") + case head == "do": + for _, t := range s[1:] { + if t.kind == tokString { + if r := bodyRefusal(t.text); r != "" { + return "its DO block: " + r + } + } + } + } + return setConfigRefusal(s) +} + +// setRefusal judges what follows SET or RESET, down to the name of the setting. +func setRefusal(rest []token, verb string) string { + i := 0 + if w := wordAt(rest, 0); w == "local" || (w == "session" && wordAt(rest, 1) != "authorization" && wordAt(rest, 1) != "characteristics") { + i = 1 + } + var name strings.Builder + for ; i < len(rest); i++ { + if rest[i].kind != tokWord && rest[i].kind != tokQuoted { + return fmt.Sprintf("the setting its %s changes cannot be read", verb) + } + name.WriteString(rest[i].text) + if i+1 >= len(rest) || !rest[i+1].is(tokPunct, ".") { + break + } + name.WriteByte('.') + i++ + } + n := name.String() + switch { + case n == "": + return fmt.Sprintf("the setting its %s changes cannot be read", verb) + case n == "session" && i+1 < len(rest): + if w := wordAt(rest, i+1); w == "authorization" || w == "characteristics" { + return fmt.Sprintf("%s SESSION %s changes what the session is", verb, strings.ToUpper(w)) + } + case n == "all" && verb == "RESET": + return "RESET ALL clears the settings that attribute the request" + case refusedSetting(n): + return fmt.Sprintf("%s %s changes who it runs or is recorded as, or how the rest is read", verb, n) + } + return "" +} + +// setConfigRefusal finds set_config() anywhere in a statement: SET by another +// name, and common enough with a harmless first argument that only the refused +// settings, or one that cannot be read, stop it. +func setConfigRefusal(s []token) string { + for i, t := range s { + if (t.kind != tokWord && t.kind != tokQuoted) || t.text != "set_config" || + i+1 >= len(s) || !s[i+1].is(tokPunct, "(") { + continue + } + if i+3 >= len(s) || s[i+2].kind != tokString || !s[i+2].plain || !s[i+3].is(tokPunct, ",") { + return "the setting its set_config() changes cannot be read" + } + if name := strings.ToLower(s[i+2].text); refusedSetting(name) { + return fmt.Sprintf("set_config() on %s changes who it runs or is recorded as", name) + } + } + return "" +} + +// bodyRefusal looks inside a DO block for the same things, in literal form. A +// block is not split into statements: BEGIN and END are its own syntax there, so +// transaction control is left to the server, which refuses it inside a +// transaction block. A SET counts only where a PL/pgSQL statement can start, so +// that `UPDATE t SET role = 1` is left alone. +func bodyRefusal(body string) string { + toks, err := lexSQL(body) + if err != nil { + return "it cannot be read: " + err.Error() + } + return tokensRefusal(toks, 0) +} + +func tokensRefusal(toks []token, depth int) string { + if r := setConfigRefusal(toks); r != "" { + return r + } + for i, t := range toks { + if t.kind == tokWord && (t.text == "set" || t.text == "reset") && startsStatement(toks, i) { + if r := setRefusal(toks[i+1:], strings.ToUpper(t.text)); r != "" { + return r + } + } + // EXECUTE 'SET ROLE …': a string in the block may be SQL. One that does + // not lex is not SQL, so it says nothing either way. + if t.kind == tokString && depth < 2 { + if inner, err := lexSQL(t.text); err == nil { + if r := tokensRefusal(inner, depth+1); r != "" { + return r + } + } + } + } + return "" +} + +func startsStatement(toks []token, i int) bool { + if i == 0 || toks[i-1].is(tokPunct, ";") { + return true + } + switch wordAt(toks, i-1) { + case "begin", "then", "else", "loop": + return true + } + return false +} + +// replayRefusal says why a whole message cannot be replayed, or "" if it can. +func replayRefusal(statement string) string { + toks, err := lexSQL(statement) + if err != nil { + return "it cannot be read: " + err.Error() + } + for _, t := range toks { + if t.kind == tokEscaped || t.is(tokWord, "uescape") { + return `it uses a U&"…" name with escapes, whose meaning this check does not work out` + } + } + for _, s := range splitStatements(toks) { + if r := statementRefusal(s); r != "" { + return r + } + } + return "" +} + +// checkReplayable refuses a request holding any entry that cannot be replayed. +func checkReplayable(entries []branch.RequestEntry) error { + var bad []string + for _, e := range entries { + if r := replayRefusal(e.Statement); r != "" { + bad = append(bad, fmt.Sprintf("entry %d (%s %s): %s", e.ID, e.CommandTag, e.Object, r)) + } + } + if len(bad) == 0 { + return nil + } + return fmt.Errorf("%w:\n %s\nNothing was applied. Make the change again on the source without those statements "+ + "(`%s blackbox` shows what it ran), or apply it by hand", branch.ErrNotReplayable, strings.Join(bad, "\n "), brand.CLI) +} diff --git a/enterprise/promote/replaycheck_test.go b/enterprise/promote/replaycheck_test.go new file mode 100644 index 00000000..a8334396 --- /dev/null +++ b/enterprise/promote/replaycheck_test.go @@ -0,0 +1,118 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package promote + +import ( + "errors" + "strings" + "testing" + + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// Every class the replay must refuse, in the forms a message can hide it in. +func TestReplayRefusal(t *testing.T) { + for _, q := range []string{ + // Transaction control. + "COMMIT", "commit;", "END", "END WORK", "ABORT", "ROLLBACK", "ROLLBACK TO SAVEPOINT s", + "BEGIN", "BEGIN ISOLATION LEVEL SERIALIZABLE", "START TRANSACTION", "SAVEPOINT s", "RELEASE SAVEPOINT s", + "PREPARE TRANSACTION 'x'", "COMMIT PREPARED 'x'", "ROLLBACK PREPARED 'x'", + "SET TRANSACTION READ ONLY", "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + // Role and session. + "SET ROLE dbadmin", "set role none", "SET LOCAL ROLE dbadmin", "SET SESSION ROLE dbadmin", "SET role = dbadmin", + `SET "role" TO dbadmin`, `SET "ROLE" TO dbadmin`, "RESET ROLE", "SET SESSION AUTHORIZATION dbadmin", + "SET LOCAL SESSION AUTHORIZATION dbadmin", "SET session_authorization = dbadmin", "RESET SESSION AUTHORIZATION", + "RESET session_authorization", "SET session_replication_role = replica", + // Attribution and the guardrail's override. + "SET bb.actor = 'x'", "SET LOCAL bb.actor_kind = 'human'", "set BB.ALLOW_DESTRUCTIVE = on", "SET bb . actor = 'x'", + `SET "bb".actor = 'x'`, `SET "bb.actor" = 'x'`, `SET "BB"."Actor" = 'x'`, "RESET bb.actor", "RESET ALL", + "SET application_name = 'x'", "SET bb.policy_allow = 'r1'", + "SELECT set_config('bb.actor', 'x', true)", "SELECT pg_catalog.set_config('role', 'dbadmin', true)", + `SELECT "set_config"('bb.session', 'x', false)`, "SELECT set_config('BB.Actor', 'x', true)", + "SELECT set_config(lower('ROLE'), 'dbadmin', true)", "SELECT set_config(E'bb.actor', 'x', true)", + // What changes how later statements are read. + "SET standard_conforming_strings = off", "SET client_encoding = 'SJIS'", "SET NAMES 'SJIS'", + // Behind comments, or where one seems to be. + "/* hello */ COMMIT", "-- hello\nCOMMIT", "/* a /* nested */ b */ RESET ROLE", "COM/* */MIT;SET ROLE x", + "-- a comment ended by a carriage return\rCOMMIT", + "CREATE TABLE t (); /* */ COMMIT; --", + // After a literal that seems to swallow it. + "SELECT 'a'; COMMIT", "SELECT $$a$$; COMMIT", "SELECT $x$ $y$ $x$; COMMIT", `SELECT "a;"; COMMIT`, + "SELECT 'it''s'; COMMIT", "SELECT E'it\\'s'; COMMIT", + // E'' escapes carry across a continuation, so this string ends where it seems not to. + "SELECT E'a'\n'\\''; COMMIT; --'", + "SELECT E'a' -- comment\n'\\''; COMMIT; --'", + // A dollar after a number starts a quote; inside a name it does not. + "SELECT 1$$a$$; COMMIT", + "SELECT a$b$; COMMIT; --$b$", + // Multi-statement messages, wherever it falls. + "CREATE TABLE t (id int); COMMIT; DROP TABLE u", "COMMIT; CREATE TABLE t (id int)", + "CREATE TABLE t (id int);;\n;SET ROLE dbadmin", + // BEGIN ATOMIC is only a body after CREATE FUNCTION, and ends at its END. + "CREATE FUNCTION begin() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; COMMIT", + "CREATE FUNCTION f() RETURNS int BEGIN ATOMIC SELECT 1; END; COMMIT", + // A DO block is not a way round. + "DO $$ BEGIN SET ROLE dbadmin; END $$", "DO $$ BEGIN SET LOCAL bb.actor = 'x'; END $$", + "DO $$ BEGIN PERFORM set_config('bb.actor', 'x', true); END $$", + "DO $$ BEGIN EXECUTE 'SET ROLE dbadmin'; END $$", "DO LANGUAGE plpgsql 'BEGIN RESET ROLE; END'", + // What cannot be read is not passed. + "CREATE TABLE t (a text DEFAULT 'oops", `CREATE TABLE "t (a int)`, "CREATE TABLE t () /* open", "SELECT $x$ open", + "SELECT 1\x00; COMMIT", `SET U&"r\006Fle" = dbadmin`, `CREATE TABLE U&"t!0041" UESCAPE '!' ()`, + } { + if replayRefusal(q) == "" { + t.Errorf("not refused: %q", q) + } + } +} + +// Ordinary changes, and the words above where they mean nothing. +func TestReplayAccepted(t *testing.T) { + for _, q := range []string{ + "CREATE TABLE t (id int PRIMARY KEY, note text)", "ALTER TABLE t ADD COLUMN x int;", "DROP INDEX i", + "CREATE INDEX ON t (id)", "GRANT SELECT ON t TO bb_audit", "COMMENT ON TABLE t IS 'do not COMMIT this; ROLLBACK later'", + // Multi-statement without transaction control. + "CREATE TABLE a (id int); CREATE VIEW v AS SELECT id FROM a; ALTER TABLE a ADD b text", + "SET search_path = app; CREATE TABLE a (id int)", "SET LOCAL lock_timeout = '5s'; ALTER TABLE a ADD b int", + "SELECT pg_catalog.set_config('search_path', '', false); CREATE TABLE a (id int)", + // Only in comments or quoting. + "CREATE TABLE t (id int) -- then COMMIT;", "/* SET ROLE x; */ CREATE TABLE t (id int)", + "CREATE TABLE t (a text DEFAULT '; COMMIT; --')", "CREATE TABLE t (a text DEFAULT $$; RESET ROLE;$$)", + `CREATE TABLE "commit" (id int)`, `CREATE TABLE t ("set role" int); ALTER TABLE t RENAME "set role" TO "begin"`, + "ALTER TABLE t ADD COLUMN \"end\" int", + // A string the E'' continuation really does keep going. + "SELECT E'a'\n'\\'; COMMIT; --'", + // PREPARE of a query, a column called role, a routine body with BEGIN and END. + "PREPARE p AS SELECT 1; CREATE TABLE a (id int)", + "UPDATE t SET role = 'x'; ALTER TABLE t ADD y int", + "CREATE FUNCTION f() RETURNS int LANGUAGE plpgsql AS $$ BEGIN COMMIT; RETURN 1; END $$", + "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC SELECT 1; SELECT CASE WHEN true THEN 1 ELSE 2 END; END", + "CREATE OR REPLACE FUNCTION f() RETURNS int BEGIN ATOMIC SELECT 1; END; CREATE TABLE a (id int)", + "CREATE RULE r AS ON INSERT TO t DO ALSO (SELECT 1; NOTIFY x)", + "DO $$ BEGIN CREATE TABLE a (id int); EXCEPTION WHEN duplicate_table THEN NULL; END $$", + "DO $$ BEGIN UPDATE t SET role = 'x'; END $$", + "DO $$ DECLARE r regclass := 't'::regclass; BEGIN EXECUTE format('GRANT SELECT ON %s TO bb_audit', r); END $$", + `CREATE TABLE U&"plain" (id int)`, + } { + if r := replayRefusal(q); r != "" { + t.Errorf("refused %q: %s", q, r) + } + } +} + +// Refused by name, and before any container is looked for: this test has none. +func TestApplyRefusesAnUnsafeStatement(t *testing.T) { + _, err := apply(1, "main", "someone@example.com", []branch.RequestEntry{ + entry(1, "a", "human", "CREATE TABLE", "public.t", "CREATE TABLE t (id int)"), + entry(2, "a", "human", "CREATE TABLE", "public.u", "CREATE TABLE u (id int); COMMIT"), + }) + if !errors.Is(err, branch.ErrNotReplayable) { + t.Fatalf("err = %v", err) + } + for _, want := range []string{"entry 2", "public.u", "COMMIT", "Nothing was applied"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("%q does not say %q", err, want) + } + } +} diff --git a/internal/branch/promote.go b/internal/branch/promote.go index 251d8780..9e50c0fc 100644 --- a/internal/branch/promote.go +++ b/internal/branch/promote.go @@ -85,6 +85,13 @@ var ErrConflict = errors.New("both branches changed the same object since they s // target in a state that is neither the old schema nor the source's. var ErrBlockedByPolicy = errors.New("the policy gate blocks a statement in this request") +// ErrNotReplayable is returned when a statement in the request carries +// transaction control or changes who the rest runs or is recorded as: an entry +// holds its author's whole message, and such a statement in it would break the +// one unprivileged transaction a request is applied in. Refused when the request +// is made and again when it is applied. +var ErrNotReplayable = errors.New("a statement in this request cannot be replayed safely") + // MarshalEntries is the snapshot stored with a request. func MarshalEntries(entries []RequestEntry) (string, error) { b, err := json.Marshal(entries) diff --git a/internal/controlplane/promote.go b/internal/controlplane/promote.go index 5740e0c4..13c7cdad 100644 --- a/internal/controlplane/promote.go +++ b/internal/controlplane/promote.go @@ -219,7 +219,7 @@ func statusForRequestErr(err error) int { switch { case errors.Is(err, branch.ErrConflict): return 409 - case errors.Is(err, branch.ErrBlockedByPolicy): + case errors.Is(err, branch.ErrBlockedByPolicy), errors.Is(err, branch.ErrNotReplayable): return 422 case errors.Is(err, branch.ErrNothingToPromote): return 400 From 4ca6d3c3538a7920866831d37d15aa0fa8a41802 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:58:25 +0530 Subject: [PATCH 11/22] fix(security): keep the Blackbox's chain head in a row taken under the lock A transaction in REPEATABLE READ or SERIALIZABLE read the tail through a snapshot that could be older than the append lock, so two rows chained off one parent and verify reported tampering. Ids came from the column default, before the lock, so id order and chain order could differ. bb.ledger_head holds the last id and hash, taken with UPDATE ... RETURNING under the lock: a stale snapshot now gets a serialization error instead of a fork, ids follow the chain and a rolled-back entry leaves no gap. Seeded from the tail on an existing ledger, without touching a row. Co-Authored-By: Claude Opus 5.5 --- internal/ledger/chainhead_test.go | 73 +++++++++++++++++++++++++++++++ internal/ledger/ledger.sql | 47 +++++++++++++++++++- 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 internal/ledger/chainhead_test.go diff --git a/internal/ledger/chainhead_test.go b/internal/ledger/chainhead_test.go new file mode 100644 index 00000000..4ea83b32 --- /dev/null +++ b/internal/ledger/chainhead_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "strings" + "testing" +) + +// chainRowBody is bb.chain_row's source, as ledger.sql writes it. +func chainRowBody(t *testing.T) string { + t.Helper() + i := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb.chain_row()") + if i < 0 { + t.Fatal("bb.chain_row is gone") + } + body := Schema[i:] + return body[:strings.Index(body, "$$;")] +} + +// In REPEATABLE READ a snapshot can be older than the append lock, so the tail +// must not be read through it: two rows would chain off one parent and verify +// would call it tampering. The head row's UPDATE is what turns a stale snapshot +// into a serialization error. Proved against a real server; this guards the +// shape it rests on. +func TestChainHeadIsTakenByUpdateUnderTheLock(t *testing.T) { + body := chainRowBody(t) + lock := strings.Index(body, "pg_advisory_xact_lock(hashtext('bb.schema_ledger'))") + head := strings.Index(body, "UPDATE bb.ledger_head SET last_id = last_id + 1 RETURNING last_id, last_hash INTO NEW.id, prev;") + hash := strings.Index(body, "NEW.row_hash := bb._ledger_hash(NEW);") + if lock < 0 || head < 0 || hash < 0 || !(lock < head && head < hash) { + t.Errorf("bb.chain_row must lock, then take the head, then hash (lock %d, head %d, hash %d)", lock, head, hash) + } + if strings.Contains(body, "ORDER BY id DESC") { + t.Error("bb.chain_row reads the tail through the transaction's snapshot again") + } + if !strings.Contains(body, "UPDATE bb.ledger_head SET last_hash = NEW.row_hash;") { + t.Error("bb.chain_row no longer moves the head to the row it wrote") + } +} + +// The upgrade: the head and the function that needs it arrive together, under +// the lock, seeded from the existing tail, and a re-run leaves a head that is +// already right alone. Nothing may rewrite a row that exists. +func TestChainHeadUpgradesAnExistingLedger(t *testing.T) { + begin := strings.Index(Schema, "BEGIN;\nDO $$ BEGIN PERFORM pg_advisory_xact_lock(hashtext('bb.schema_ledger')); END $$;") + table := strings.Index(Schema, "CREATE TABLE IF NOT EXISTS bb.ledger_head") + fn := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb.chain_row()") + commit := strings.Index(Schema, "FOR EACH ROW EXECUTE FUNCTION bb.chain_row();\nCOMMIT;") + if begin < 0 || table < begin || fn < table || commit < fn { + t.Errorf("the head and bb.chain_row must be installed in one locked transaction (begin %d, table %d, function %d, commit %d)", begin, table, fn, commit) + } + for _, want := range []string{ + "(SELECT id, row_hash FROM bb.schema_ledger ORDER BY id DESC LIMIT 1) t ON true", + "ON CONFLICT (id) DO UPDATE SET last_id = EXCLUDED.last_id, last_hash = EXCLUDED.last_hash\n WHERE EXCLUDED.last_id > bb.ledger_head.last_id;", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } +} + +// A masked branch's ledger is emptied by the engine, past the triggers +// (enterprise/masking, freshBlackboxSQL), and verify expects the first row of +// a ledger to link to nothing. +func TestChainRestartsOnAnEmptiedLedger(t *testing.T) { + body := chainRowBody(t) + head := strings.Index(body, "UPDATE bb.ledger_head SET last_id") + empty := strings.Index(body, "IF NOT EXISTS (SELECT 1 FROM bb.schema_ledger) THEN\n prev := '';") + if empty < 0 || empty < head { + t.Error("an emptied ledger must start a new chain, judged after the head was taken") + } +} diff --git a/internal/ledger/ledger.sql b/internal/ledger/ledger.sql index 0a2bba3e..3ec4bec1 100644 --- a/internal/ledger/ledger.sql +++ b/internal/ledger/ledger.sql @@ -696,20 +696,65 @@ LANGUAGE sql IMMUTABLE AS $$ coalesce(r.risk,''), 'UTF8')), 'hex'); $$; +-- The head of the chain, one row: the last id handed out and its row_hash. The +-- tail used to be read with SELECT … ORDER BY id DESC, under whatever snapshot +-- the transaction had, and in REPEATABLE READ or SERIALIZABLE that snapshot can +-- be older than the lock: two rows chained off one parent, and verify called it +-- tampering. An UPDATE of a row changed since the snapshot fails with a +-- serialization error instead, so a stale writer retries rather than forks. +-- +-- One transaction, under the lock appends take: an append by the previous +-- bb.chain_row must not fall between reading the tail here and the new one +-- going in. Seeded from the tail on an existing ledger; nothing written so far +-- changes. The head only ever moves forward here: a tail ahead of it was +-- written by an older engine, and a tail behind it has lost rows, which the +-- next entry's link should go on showing. +BEGIN; +DO $$ BEGIN PERFORM pg_advisory_xact_lock(hashtext('bb.schema_ledger')); END $$; +CREATE TABLE IF NOT EXISTS bb.ledger_head ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + last_id bigint NOT NULL, + last_hash text NOT NULL +); +INSERT INTO bb.ledger_head (last_id, last_hash) +SELECT coalesce(t.id, 0), coalesce(t.row_hash, '') +FROM (SELECT 1) one LEFT JOIN LATERAL + (SELECT id, row_hash FROM bb.schema_ledger ORDER BY id DESC LIMIT 1) t ON true +ON CONFLICT (id) DO UPDATE SET last_id = EXCLUDED.last_id, last_hash = EXCLUDED.last_hash + WHERE EXCLUDED.last_id > bb.ledger_head.last_id; + +-- The id comes from the head too, after the lock, so id order is chain order +-- (the column's default is taken before the lock, by whoever got there first) +-- and a rolled-back entry leaves no gap. The default stays: it costs a sequence +-- value and keeps the sequence ahead of every id, should an older engine ever +-- write here again. +-- +-- A ledger emptied by the engine (a masked branch starts with a fresh one) starts +-- a new chain, with ids carrying on. Reading it is safe at this point: the UPDATE +-- above has proved this snapshot saw the head's latest change, and with it the +-- last entry. CREATE OR REPLACE FUNCTION bb.chain_row() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE prev text; BEGIN -- Serialize appends so two concurrent inserts can't chain off the same row. PERFORM pg_advisory_xact_lock(hashtext('bb.schema_ledger')); - SELECT row_hash INTO prev FROM bb.schema_ledger ORDER BY id DESC LIMIT 1; + UPDATE bb.ledger_head SET last_id = last_id + 1 RETURNING last_id, last_hash INTO NEW.id, prev; + IF NOT FOUND THEN + RAISE EXCEPTION 'bb.ledger_head has no row: restart the engine to restore it'; + END IF; + IF NOT EXISTS (SELECT 1 FROM bb.schema_ledger) THEN + prev := ''; + END IF; NEW.prev_hash := coalesce(prev, ''); NEW.row_hash := bb._ledger_hash(NEW); + UPDATE bb.ledger_head SET last_hash = NEW.row_hash; RETURN NEW; END; $$; CREATE OR REPLACE TRIGGER bb_chain BEFORE INSERT ON bb.schema_ledger FOR EACH ROW EXECUTE FUNCTION bb.chain_row(); +COMMIT; -- Append-only: the ledger records history, so history cannot be rewritten. CREATE OR REPLACE FUNCTION bb.deny_change() RETURNS trigger From f452613428b5efa7f030a2f87c04c2579a993005 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:58:32 +0530 Subject: [PATCH 12/22] fix(perf): verify the security log's checkpoint from its last anchor CheckpointSecurityLog reloaded and re-hashed every event on each checkpoint, so its cost followed the log's whole length. It now checks the last anchored event against its anchor and chains forward from it, a page at a time. The full check stays where it was: fox audit verify still reads every event against every anchor, and only that catches an old event edited in place. The doc comment and a test say exactly where the line is. Co-Authored-By: Claude Opus 5.5 --- internal/auth/audit.go | 13 +- internal/branch/seclog.go | 65 ++++++-- internal/branch/seclog_checkpoint_test.go | 172 ++++++++++++++++++++++ 3 files changed, 238 insertions(+), 12 deletions(-) create mode 100644 internal/branch/seclog_checkpoint_test.go diff --git a/internal/auth/audit.go b/internal/auth/audit.go index a6c6db9b..32f6cb2b 100644 --- a/internal/auth/audit.go +++ b/internal/auth/audit.go @@ -25,7 +25,9 @@ import ( // SQLite triggers refuse UPDATE and DELETE, and the checkpointer anchors the // chain's head in a signed file outside the store (branch.CheckpointSecurityLog), // so rewriting history — even recomputing every hash — breaks either the chain -// or an anchor. `fox audit verify` checks both. +// or an anchor. `fox audit verify` checks both over every event; a checkpoint +// checks only what has been added since the last anchor (see +// branch.CheckpointSecurityLog for exactly what that does and does not cover). // Event kinds. const ( @@ -435,8 +437,13 @@ func (s *Store) LastLicenseEvent() (kind, subject string, ok bool, err error) { // CheckEventChain recomputes the chain. It returns the id of the first event // that does not match, or 0 when all do. func CheckEventChain(evs []SecurityEvent) (int64, string) { - prev := "" - var lastID int64 + return CheckEventChainFrom(evs, 0, "") +} + +// CheckEventChainFrom recomputes the chain over evs, taking them to follow the +// event lastID, whose row hash is prev. Like CheckEventChain, it returns the id +// of the first event that does not match, or 0 when all do. +func CheckEventChainFrom(evs []SecurityEvent, lastID int64, prev string) (int64, string) { for _, e := range evs { switch { case e.ID != lastID+1: diff --git a/internal/branch/seclog.go b/internal/branch/seclog.go index ffe0a199..e53f9179 100644 --- a/internal/branch/seclog.go +++ b/internal/branch/seclog.go @@ -38,8 +38,30 @@ const ( secAnchorAlgorithm = "sha256-chain-v1" // LastRowHash is the head of auth.EventHash's chain ) +// secLogCheckPageSize is how many events a checkpoint holds at once. A variable +// so a test can reach a page boundary without writing thousands of events. +var secLogCheckPageSize = 5000 + // CheckpointSecurityLog anchors the security log's events added since its last // anchor. It returns nil when there is nothing new. +// +// It verifies incrementally, so its cost follows what is new rather than how +// long the log has grown. Before anchoring it checks: +// - that the last anchored event is still there, is unchanged, and has the +// hash the anchor signed; +// - that every event since chains from that hash, with no id missing and no +// event changed. +// +// So it catches any edit after the last anchor, and any rewrite of older +// events that was re-hashed through to the present, because that changes the +// hash the new events chain from. +// +// It does not re-read events before the last anchored one. An older event +// edited in place, with the rows after it left alone, is not noticed here and +// this will go on anchoring; nor are the older anchors or any signature looked +// at. `fox audit verify` (VerifySecurityLog) checks every event against every +// anchor and is what catches those: this is a guard against anchoring a log +// already known to be bad, and passing it says nothing about the older rows. func CheckpointSecurityLog(store *auth.Store) (*ledger.Anchor, error) { dir := SecurityLogAnchorDir() anchors, err := ledger.LoadAnchors(dir) @@ -53,21 +75,46 @@ func CheckpointSecurityLog(store *auth.Store) (*ledger.Anchor, error) { lastTo, prev = a.ToID, a.LastRowHash } } - fresh, err := store.SecurityEvents(lastTo, 0) - if err != nil || len(fresh) == 0 { + page, err := store.SecurityEvents(lastTo, secLogCheckPageSize) + if err != nil || len(page) == 0 { return nil, err } // Refuse to anchor a log that no longer chains: that would bless it. - all, err := store.SecurityEvents(0, 0) - if err != nil { - return nil, err - } - if id, why := auth.CheckEventChain(all); id != 0 { + refuse := func(why string) (*ledger.Anchor, error) { return nil, fmt.Errorf("not anchoring the security log: %s — inspect with: %s audit verify", why, brand.CLI) } - head := fresh[len(fresh)-1] + if lastTo > 0 { + anchored, err := store.SecurityEvents(lastTo-1, 1) + if err != nil { + return nil, err + } + if len(anchored) != 1 || anchored[0].ID != lastTo { + return refuse(fmt.Sprintf("event %d, the last one anchored, is gone", lastTo)) + } + if e := anchored[0]; e.RowHash != prev || auth.EventHash(e) != e.RowHash { + return refuse(fmt.Sprintf("event %d no longer matches its anchor", lastTo)) + } + } + var head auth.SecurityEvent + n, after, hash := 0, lastTo, prev + for { + if id, why := auth.CheckEventChainFrom(page, after, hash); id != 0 { + return refuse(why) + } + head = page[len(page)-1] + n, after, hash = n+len(page), head.ID, head.RowHash + if len(page) < secLogCheckPageSize { + break + } + if page, err = store.SecurityEvents(after, secLogCheckPageSize); err != nil { + return nil, err + } + if len(page) == 0 { + break + } + } a := ledger.Anchor{Format: secAnchorFormat, Algorithm: secAnchorAlgorithm, Branch: secLogAnchorDir, - CheckpointID: head.ID, FromID: lastTo + 1, ToID: head.ID, EntryCount: len(fresh), + CheckpointID: head.ID, FromID: lastTo + 1, ToID: head.ID, EntryCount: n, LastRowHash: head.RowHash, MerkleRoot: head.RowHash, PrevRoot: prev, CreatedAt: time.Now().UTC()} key, err := ledger.LoadOrCreateSigningKey(AnchorKeyPath()) if err != nil { diff --git a/internal/branch/seclog_checkpoint_test.go b/internal/branch/seclog_checkpoint_test.go new file mode 100644 index 00000000..44d66df0 --- /dev/null +++ b/internal/branch/seclog_checkpoint_test.go @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "database/sql" + "fmt" + "path/filepath" + "testing" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/brand" + "github.com/thefoxbyte/foxbyte/internal/ledger" +) + +// tamperEvent changes one row directly in the store, past the append-only +// trigger: what someone with the file and SQLite can do. +func tamperEvent(t *testing.T, path, set string, id int64) { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, q := range []string{ + `DROP TRIGGER security_events_no_update`, + fmt.Sprintf(`UPDATE security_events SET %s WHERE id = %d`, set, id), + `CREATE TRIGGER security_events_no_update BEFORE UPDATE ON security_events + BEGIN SELECT RAISE(ABORT, 'security_events is append-only'); END`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } +} + +func logWith(t *testing.T, events int) (*auth.Store, string) { + t.Helper() + t.Setenv(brand.EnvName("ANCHOR_DIR"), t.TempDir()) + t.Setenv(brand.EnvName("ANCHOR_KEY"), filepath.Join(t.TempDir(), "signing.key")) + path := filepath.Join(t.TempDir(), "a.db") + s, err := auth.Open(auth.Config{DBPath: path, WebOrigin: "http://x"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + addEvents(s, events) + return s, path +} + +func addEvents(s *auth.Store, n int) { + for i := 0; i < n; i++ { + s.Audit(auth.EvLoginOK, "a@x.com", "", "127.0.0.1", "") + } +} + +// A log with its first n events already anchored. +func anchoredLog(t *testing.T, n int) (*auth.Store, string) { + t.Helper() + s, path := logWith(t, n) + if a, err := CheckpointSecurityLog(s); err != nil || a == nil || a.ToID != int64(n) { + t.Fatalf("the first anchor: %+v, %v", a, err) + } + return s, path +} + +func fullVerify(t *testing.T, s *auth.Store) ledger.Report { + t.Helper() + evs, err := s.SecurityEvents(0, 0) + if err != nil { + t.Fatal(err) + } + anchors, err := ledger.LoadAnchors(SecurityLogAnchorDir()) + if err != nil { + t.Fatal(err) + } + return VerifySecurityLog(evs, anchors, nil) +} + +// A checkpoint anchors what is new and leaves a genuine log verifying. +func TestCheckpointAnchorsOnlyWhatIsNew(t *testing.T) { + s, _ := anchoredLog(t, 4) + addEvents(s, 2) + a, err := CheckpointSecurityLog(s) + if err != nil || a == nil || a.FromID != 5 || a.ToID != 6 || a.EntryCount != 2 { + t.Fatalf("the second anchor: %+v, %v", a, err) + } + if rep := fullVerify(t, s); !rep.Intact || rep.UnanchoredRows != 0 { + t.Fatalf("a genuine log: %+v", rep) + } + if again, err := CheckpointSecurityLog(s); err != nil || again != nil { + t.Fatalf("with nothing new: %+v, %v", again, err) + } +} + +// What the incremental check catches, and the line it does not cross: an edit +// before the last anchor, left in place, is passed by the checkpoint and caught +// by the full verify. Asserted in both directions so the documented coverage +// cannot quietly stop being true. +func TestCheckpointChecksFromTheLastAnchor(t *testing.T) { + t.Run("an edit since the anchor is refused", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 2) + tamperEvent(t, path, "actor = 'nobody'", 5) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("anchored over an edited event: %+v, %v", a, err) + } + }) + t.Run("an edit to the anchored event is refused", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 1) + tamperEvent(t, path, "actor = 'nobody'", 4) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("anchored past an edited anchor event: %+v, %v", a, err) + } + }) + t.Run("a rewrite re-hashed up to the new events is refused", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 1) + // Event 5 chaining from something other than the anchored hash is what + // recomputing every hash after an old edit leaves behind. + tamperEvent(t, path, "prev_hash = 'rewritten'", 5) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("anchored a chain not following its anchor: %+v, %v", a, err) + } + }) + t.Run("an edit before the anchor is the full verify's to catch", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 1) + tamperEvent(t, path, "actor = 'nobody'", 2) + if a, err := CheckpointSecurityLog(s); err != nil || a == nil { + t.Fatalf("the incremental check looked before its anchor: %+v, %v", a, err) + } + if rep := fullVerify(t, s); rep.Intact || rep.FirstBrokenID != 2 { + t.Fatalf("the full verify missed an edit before the last anchor: %+v", rep) + } + }) + t.Run("the first anchor checks everything", func(t *testing.T) { + s, path := logWith(t, 3) + tamperEvent(t, path, "actor = 'nobody'", 1) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("a first anchor over an edited log: %+v, %v", a, err) + } + }) +} + +// The tail is read in pages, and every page is counted and checked. +func TestCheckpointPagesThroughTheTail(t *testing.T) { + old := secLogCheckPageSize + secLogCheckPageSize = 3 + t.Cleanup(func() { secLogCheckPageSize = old }) + + s, path := anchoredLog(t, 1) + addEvents(s, 7) // a boundary-straddling tail + a, err := CheckpointSecurityLog(s) + if err != nil || a == nil || a.EntryCount != 7 || a.ToID != 8 { + t.Fatalf("a tail longer than a page: %+v, %v", a, err) + } + addEvents(s, 6) // exactly two pages + a, err = CheckpointSecurityLog(s) + if err != nil || a == nil || a.EntryCount != 6 || a.ToID != 14 { + t.Fatalf("a tail of exact pages: %+v, %v", a, err) + } + if rep := fullVerify(t, s); !rep.Intact { + t.Fatalf("after paging: %+v", rep) + } + addEvents(s, 7) + tamperEvent(t, path, "actor = 'nobody'", 20) // in the third page + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("an edit in a later page: %+v, %v", a, err) + } +} From f76b2b7d78e676a42e49628d1eeb6a16b620c147 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:58:32 +0530 Subject: [PATCH 13/22] fix(perf): throttle and coalesce the Gateway's refusals per address Every invalid key was a chained row written under the write lock in a log that is never trimmed. Refusals now coalesce to one event per address per five minutes, carrying a count of what was suppressed, and an address that has used up its invalid keys is refused before any key is looked up. Only invalid keys count towards that, per address, so one address cannot lock out another and a valid key refused for its scope does not cost a client its connection. No retention or deletion is added: the log stays append-only. Co-Authored-By: Claude Opus 5.5 --- internal/auth/auth.go | 8 +- internal/auth/gateway.go | 219 ++++++++++++++++++++++++ internal/auth/gateway_test.go | 208 ++++++++++++++++++++++ internal/proxy/gateway_throttle_test.go | 115 +++++++++++++ internal/proxy/proxy.go | 12 +- 5 files changed, 558 insertions(+), 4 deletions(-) create mode 100644 internal/auth/gateway.go create mode 100644 internal/auth/gateway_test.go create mode 100644 internal/proxy/gateway_throttle_test.go diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 42a0bdee..8744f27f 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -57,6 +57,7 @@ type Store struct { db *sql.DB cfg Config throttle *throttle + gateway *gatewayGuard // historyTrimmed is when query history was last trimmed to its retention, // in Unix nanoseconds (history.go). historyTrimmed atomic.Int64 @@ -142,7 +143,9 @@ func Open(cfg Config) (*Store, error) { db.Close() return nil, err } - return &Store{db: db, cfg: cfg, throttle: newThrottle(loginMaxFails, loginWindow)}, nil + s := &Store{db: db, cfg: cfg, throttle: newThrottle(loginMaxFails, loginWindow)} + s.gateway = newGatewayGuard(s) + return s, nil } // initSchema creates and migrates the store. @@ -283,6 +286,9 @@ func (s *Store) Close() error { if s == nil || s.db == nil { return nil } + if s.gateway != nil { + s.gateway.close() + } return s.db.Close() } diff --git a/internal/auth/gateway.go b/internal/auth/gateway.go new file mode 100644 index 00000000..8166c448 --- /dev/null +++ b/internal/auth/gateway.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "fmt" + "net/netip" + "sync" + "time" +) + +// The Gateway's refusals, kept from flooding the security log. +// +// Every invalid key used to be one event, and one event is a chained row +// written under SQLite's write lock in a log that is never trimmed, so anyone +// who could reach the port could grow it without limit and queue every other +// writer behind them. Two things stop that, both per client address. +// +// The refusals are coalesced: one event per address per window, the first as +// it happens, carrying how many were suppressed since the event before it, so +// the log still says that an address was hammering the door and roughly how +// hard. A window's suppressed count is written when the window ends even if +// the address has gone quiet by then; a process killed inside a window loses +// that one count, never the event that opened it. +// +// And an address that has used up its invalid keys is refused before any key +// is looked up, until its window has passed. Only invalid keys count towards +// that. A valid key that is refused (a realtime key, a branch its owner may +// not open) is coalesced but never held against the address, because it proves +// the holder is a real client with a mistake to fix, and a success does not +// clear the count, because an attacker with one working key could then keep +// guessing for ever. +// +// Nothing here is keyed on anything an attacker can choose about someone else: +// a count belongs to the address that earned it, so refusals from one address +// never lock out another, and the check runs only when a connection opens, so +// a session already in progress is never ended by it. Clients sharing an +// address (behind one NAT) do share its count; that is the cost of the address +// being the only identity there is before a key has been checked. +// +// In memory and per process, like the sign-in throttle: a brake, not a record. +const ( + gatewayMaxFails = 30 + // gatewayWindow is both how long an address's invalid keys are counted and + // how often a refusing address can add a row to a log nothing trims: at + // most 288 rows a day per address, against thousands a second before. A + // client whose key was wrong is back within five minutes of fixing it. + gatewayWindow = 5 * time.Minute +) + +// gatewayGuard is the Gateway's throttle and its coalescer. +type gatewayGuard struct { + store *Store + limit *throttle + window time.Duration + now func() time.Time + + mu sync.Mutex + runs map[string]*refusalRun + timer *time.Timer // set while a run is waiting to be written + closed bool +} + +// refusalRun is one address's refusals of one sort in the current window: the +// event that opened it, and how many have been suppressed since. +type refusalRun struct { + first time.Time + suppressed int + kind, actor, subject, ip string + detail string +} + +func newGatewayGuard(s *Store) *gatewayGuard { + return &gatewayGuard{store: s, limit: newThrottle(gatewayMaxFails, gatewayWindow), + window: gatewayWindow, now: time.Now, runs: map[string]*refusalRun{}} +} + +// gatewayBucket is what an address is counted under. An IPv6 client is +// normally handed a whole /64, so counting single addresses would let it pick +// a fresh one for every attempt. +func gatewayBucket(ip string) string { + a, err := netip.ParseAddr(ip) + if err != nil { + return ip + } + a = a.Unmap() + if a.Is6() { + if p, err := a.Prefix(64); err == nil { + return p.String() + } + } + return a.String() +} + +// gatewayExempt is an address the throttle does not hold: this machine, whose +// one address is shared by every process on it and by anything forwarding +// connections to the Gateway, so holding it would let one of them lock out the +// rest. Its refusals are still coalesced. +func gatewayExempt(ip string) bool { + a, err := netip.ParseAddr(ip) + return err == nil && a.Unmap().IsLoopback() +} + +// GatewayThrottled reports whether an address has used up its invalid keys, +// and for how much longer. Called before a key is read: a held address costs +// no key lookup. A held attempt is added to the address's coalesced count. +func (s *Store) GatewayThrottled(database, ip string) (bool, time.Duration) { + if s == nil || s.gateway == nil || gatewayExempt(ip) { + return false, 0 + } + held, wait := s.gateway.limit.blocked(gatewayBucket(ip)) + if held { + s.gateway.note(EvGatewayRefused, "", database, ip, "invalid API key") + } + return held, wait +} + +// GatewayInvalidKey records a key that matched nothing: it counts towards the +// address's throttle, and the event is coalesced. +func (s *Store) GatewayInvalidKey(database, ip string) { + if s == nil || s.gateway == nil { + return + } + s.gateway.limit.fail(gatewayBucket(ip)) + s.gateway.note(EvGatewayRefused, "", database, ip, "invalid API key") +} + +// GatewayRefused records a valid key refused at the Gateway: coalesced like an +// invalid one, and never counted against the address. Runs are kept apart by +// kind and actor, so one person's refusals do not hide another's. +func (s *Store) GatewayRefused(kind, actor, subject, ip, detail string) { + if s == nil || s.gateway == nil { + return + } + s.gateway.note(kind, actor, subject, ip, detail) +} + +// note counts one refusal, and writes an event if it opens a window. +func (g *gatewayGuard) note(kind, actor, subject, ip, detail string) { + k := gatewayBucket(ip) + "\x00" + kind + "\x00" + actor + g.mu.Lock() + now := g.now() + r, ok := g.runs[k] + if ok && now.Sub(r.first) < g.window { + r.suppressed++ + g.mu.Unlock() + return + } + prior := 0 + if ok { + prior = r.suppressed + } + g.runs[k] = &refusalRun{first: now, kind: kind, actor: actor, subject: subject, ip: ip, detail: detail} + g.arm() + g.mu.Unlock() + g.store.Audit(kind, actor, subject, ip, coalescedDetail(detail, prior)) +} + +// coalescedDetail says how many refusals an event stands for beyond itself. +func coalescedDetail(detail string, suppressed int) string { + if suppressed == 0 { + return detail + } + return fmt.Sprintf("%s, and %d more refused from this address since the last record", detail, suppressed) +} + +// arm makes sure a timer is due to write what is waiting. The caller holds mu. +func (g *gatewayGuard) arm() { + if g.timer == nil && !g.closed { + g.timer = time.AfterFunc(g.window/4, g.tick) + } +} + +func (g *gatewayGuard) tick() { + g.flush(false) + g.mu.Lock() + defer g.mu.Unlock() + g.timer = nil + if len(g.runs) > 0 { + g.arm() + } +} + +// flush writes the suppressed counts of the windows that have ended, or of +// every window when all is set. A window that ended with nothing suppressed +// is forgotten; one that wrote a count opens the next window, so an address +// still gets at most one event per window however its refusals fall. +func (g *gatewayGuard) flush(all bool) { + g.mu.Lock() + now := g.now() + var due []refusalRun + for k, r := range g.runs { + if !all && now.Sub(r.first) < g.window { + continue + } + if r.suppressed == 0 { + delete(g.runs, k) + continue + } + due = append(due, *r) + r.first, r.suppressed = now, 0 + } + g.mu.Unlock() + for _, r := range due { + g.store.Audit(r.kind, r.actor, r.subject, r.ip, coalescedDetail(r.detail, r.suppressed)) + } +} + +// close writes what is waiting, while there is still a store to write it to. +func (g *gatewayGuard) close() { + g.mu.Lock() + g.closed = true + if g.timer != nil { + g.timer.Stop() + g.timer = nil + } + g.mu.Unlock() + g.flush(true) +} diff --git a/internal/auth/gateway_test.go b/internal/auth/gateway_test.go new file mode 100644 index 00000000..638ff8a3 --- /dev/null +++ b/internal/auth/gateway_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// A store whose gateway guard runs on a clock the test moves. +func guardedStore(t *testing.T) (*Store, *time.Time) { + t.Helper() + s := testStore(t) + now := time.Now() + clock := func() time.Time { return now } + s.gateway.now, s.gateway.limit.now = clock, clock + return s, &now +} + +func gatewayEvents(t *testing.T, s *Store) []SecurityEvent { + t.Helper() + evs, err := s.SecurityEvents(0, 0) + if err != nil { + t.Fatal(err) + } + return evs +} + +// One event per window however many refusals, the first when it happens, and +// the next one saying how many it stands for. +func TestGatewayRefusalsAreCoalesced(t *testing.T) { + s, now := guardedStore(t) + const ip = "203.0.113.7" + for i := 0; i < 1000; i++ { + s.GatewayInvalidKey("main", ip) + } + evs := gatewayEvents(t, s) + if len(evs) != 1 || evs[0].Kind != EvGatewayRefused || evs[0].IP != ip || evs[0].Detail != "invalid API key" { + t.Fatalf("after 1000 refusals in one window: %+v", evs) + } + *now = now.Add(gatewayWindow) + s.GatewayInvalidKey("main", ip) + evs = gatewayEvents(t, s) + if len(evs) != 2 || !strings.Contains(evs[1].Detail, "999 more") { + t.Fatalf("the next window's event: %+v", evs) + } + if id, why := CheckEventChain(evs); id != 0 { + t.Fatalf("the chain: %s", why) + } +} + +// An address that stops mid-window still gets its count written, without +// waiting for a refusal that may never come, and no extra event. +func TestGatewayCountIsWrittenWhenTheWindowEnds(t *testing.T) { + s, now := guardedStore(t) + for i := 0; i < 5; i++ { + s.GatewayInvalidKey("main", "203.0.113.7") + } + s.gateway.flush(false) + if n := len(gatewayEvents(t, s)); n != 1 { + t.Fatalf("written before the window ended: %d events", n) + } + *now = now.Add(gatewayWindow) + s.gateway.flush(false) + evs := gatewayEvents(t, s) + if len(evs) != 2 || !strings.Contains(evs[1].Detail, "4 more") { + t.Fatalf("after the window: %+v", evs) + } + // The count opened the next window: that one wrote nothing more, and it + // then goes away. + *now = now.Add(gatewayWindow) + s.gateway.flush(false) + if n := len(gatewayEvents(t, s)); n != 2 { + t.Fatalf("a quiet window added an event: %d", n) + } + if len(s.gateway.runs) != 0 { + t.Fatalf("a quiet address was remembered: %d runs", len(s.gateway.runs)) + } +} + +// Closing the store writes what was waiting. +func TestGatewayCountIsWrittenOnClose(t *testing.T) { + s, _ := guardedStore(t) + s.GatewayInvalidKey("main", "203.0.113.7") + s.GatewayInvalidKey("main", "203.0.113.7") + s.gateway.close() + evs := gatewayEvents(t, s) + if len(evs) != 2 || !strings.Contains(evs[1].Detail, "1 more") { + t.Fatalf("after close: %+v", evs) + } +} + +// A throttled address is held for its window, then let go, and its held +// attempts are counted without a row each. +func TestGatewayThrottleHoldsAnAddress(t *testing.T) { + s, now := guardedStore(t) + const ip = "203.0.113.7" + for i := 0; i < gatewayMaxFails; i++ { + if held, _ := s.GatewayThrottled("main", ip); held { + t.Fatalf("held after %d failures", i) + } + s.GatewayInvalidKey("main", ip) + } + for i := 0; i < 500; i++ { + if held, wait := s.GatewayThrottled("main", ip); !held || wait <= 0 { + t.Fatalf("not held once its failures were used up: %v %v", held, wait) + } + } + if n := len(gatewayEvents(t, s)); n != 1 { + t.Fatalf("held attempts wrote %d events", n) + } + *now = now.Add(gatewayWindow) + if held, _ := s.GatewayThrottled("main", ip); held { + t.Fatal("still held after its window") + } +} + +// One address's refusals cost nobody else anything: not another address, and +// not a run of refusals naming a database or an account. +func TestGatewayThrottleIsPerAddress(t *testing.T) { + s, _ := guardedStore(t) + for i := 0; i < gatewayMaxFails*2; i++ { + s.GatewayInvalidKey(fmt.Sprintf("db%d", i), "203.0.113.7") + } + if held, _ := s.GatewayThrottled("main", "203.0.113.8"); held { + t.Fatal("an attacker's refusals held another address") + } + if held, _ := s.GatewayThrottled("main", "2001:db8::1"); held { + t.Fatal("an attacker's refusals held another family's address") + } + if held, _ := s.GatewayThrottled("anything", "203.0.113.7"); !held { + t.Fatal("the attacker was not held whatever database it named") + } +} + +// A valid key that was refused is logged and coalesced, and is never held +// against the address: a client asking for a branch it may not open must not +// lose the branches it may. +func TestGatewayValidKeyRefusalsDoNotThrottle(t *testing.T) { + s, _ := guardedStore(t) + const ip = "203.0.113.7" + for i := 0; i < gatewayMaxFails*3; i++ { + s.GatewayRefused(EvDenied, "a@x.com", "secret", ip, "gateway") + s.GatewayRefused(EvGatewayRefused, "a@x.com", "secret", ip, "realtime key at the gateway") + } + if held, _ := s.GatewayThrottled("main", ip); held { + t.Fatal("refusals of a valid key throttled the address") + } + if n := len(gatewayEvents(t, s)); n != 2 { + t.Fatalf("want one event per kind in the window, got %d", n) + } + // One person's refusals do not hide another's. + s.GatewayRefused(EvDenied, "b@x.com", "secret", ip, "gateway") + if n := len(gatewayEvents(t, s)); n != 3 { + t.Fatalf("a second actor was folded into the first: %d events", n) + } +} + +// This machine's address is shared by everything on it and by anything +// forwarding to the Gateway, so it is coalesced and not held. +func TestGatewayDoesNotHoldLoopback(t *testing.T) { + s, _ := guardedStore(t) + for _, ip := range []string{"127.0.0.1", "::1"} { + for i := 0; i < gatewayMaxFails*2; i++ { + s.GatewayInvalidKey("main", ip) + } + if held, _ := s.GatewayThrottled("main", ip); held { + t.Errorf("%s was held", ip) + } + } + if n := len(gatewayEvents(t, s)); n != 2 { + t.Fatalf("loopback refusals: %d events, want one per address", n) + } +} + +func TestGatewayBucket(t *testing.T) { + for in, want := range map[string]string{ + "203.0.113.7": "203.0.113.7", + "::ffff:203.0.113.7": "203.0.113.7", + "2001:db8:1:2:3:4:5:6": "2001:db8:1:2::/64", + "2001:db8:1:2:aaaa:bbbb:c:d": "2001:db8:1:2::/64", + "2001:db8:1:3:aaaa:bbbb:c:d": "2001:db8:1:3::/64", + "not an address": "not an address", + } { + if got := gatewayBucket(in); got != want { + t.Errorf("gatewayBucket(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCheckEventChainFromTakesUpAfterAnAnchor(t *testing.T) { + s := testStore(t) + for i := 0; i < 6; i++ { + s.Audit(EvLoginOK, "a@x.com", "", "", "") + } + evs := gatewayEvents(t, s) + if id, why := CheckEventChainFrom(evs[3:], evs[2].ID, evs[2].RowHash); id != 0 { + t.Fatalf("a genuine tail: %s", why) + } + if id, _ := CheckEventChainFrom(evs[3:], evs[2].ID, "wrong"); id != evs[3].ID { + t.Fatalf("a tail not following the anchor: %d", id) + } + if id, _ := CheckEventChainFrom(evs[3:], evs[3].ID, evs[2].RowHash); id != evs[3].ID { + t.Fatalf("a tail with the wrong id: %d", id) + } +} diff --git a/internal/proxy/gateway_throttle_test.go b/internal/proxy/gateway_throttle_test.go new file mode 100644 index 00000000..d55fb5bc --- /dev/null +++ b/internal/proxy/gateway_throttle_test.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxy + +import ( + "bytes" + "crypto/tls" + "encoding/binary" + "io" + "net" + "path/filepath" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" +) + +// connectFrom runs one connection through handle from a public address, over +// TLS as any such client must be, and returns what the Gateway answered first +// and the SQLSTATE it finally refused with. +func connectFrom(t *testing.T, ip, password string) (first byte, code string) { + t.Helper() + client, server := net.Pipe() + defer client.Close() + done := make(chan struct{}) + go func() { + defer close(done) + remote, _ := net.ResolveTCPAddr("tcp", ip+":50000") + handle(addrConn{Conn: server, remote: remote}) + }() + _ = client.SetDeadline(time.Now().Add(10 * time.Second)) + ssl := make([]byte, 8) + binary.BigEndian.PutUint32(ssl[0:], 8) + binary.BigEndian.PutUint32(ssl[4:], codeSSL) + if _, err := client.Write(ssl); err != nil { + t.Fatal(err) + } + if _, err := io.ReadFull(client, ssl[:1]); err != nil || ssl[0] != 'S' { + t.Fatalf("SSLRequest: %q %v", ssl[0], err) + } + tc := tls.Client(client, &tls.Config{InsecureSkipVerify: true}) + if _, err := tc.Write(buildStartup(map[string]string{"user": "u", "database": "main"})); err != nil { + t.Fatal(err) + } + typ, body, err := readMsg(tc) + if err != nil { + t.Fatal(err) + } + first = typ + if typ == 'R' { // the password asked for + if err := writeMsg(tc, 'p', append([]byte(password), 0)); err != nil { + t.Fatal(err) + } + if typ, body, err = readMsg(tc); err != nil { + t.Fatal(err) + } + } + if typ == 'E' { + for _, f := range bytes.Split(body, []byte{0}) { + if len(f) > 1 && f[0] == 'C' { + code = string(f[1:]) + } + } + } + <-done + return first, code +} + +// The whole refusal path through handle: invalid keys coalesce into one event, +// an address that has used up its failures is turned away before it is even +// asked for a key, and another address is still asked. +func TestGatewayThrottlesInvalidKeys(t *testing.T) { + store, err := auth.Open(auth.Config{DBPath: filepath.Join(t.TempDir(), "a.db"), WebOrigin: "http://x"}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + prevStore, prevTLS := authStore, tlsConfig + authStore, tlsConfig = store, testTLSConfig(t) + defer func() { authStore, tlsConfig = prevStore, prevTLS }() + + const attacker = "203.0.113.7" + attempts := 0 + for ; attempts < 200; attempts++ { + first, code := connectFrom(t, attacker, "key_nonsense") + if first != 'R' { + if first != 'E' || code != "28000" { + t.Fatalf("attempt %d: first answer %q, code %q", attempts, first, code) + } + break + } + if code != "28P01" { + t.Fatalf("attempt %d: code %q, want 28P01", attempts, code) + } + } + if attempts == 200 { + t.Fatal("never throttled") + } + if first, code := connectFrom(t, "203.0.113.8", "key_nonsense"); first != 'R' || code != "28P01" { + t.Fatalf("another address: first %q, code %q", first, code) + } + evs, err := store.SecurityEvents(0, 0) + if err != nil { + t.Fatal(err) + } + n := 0 + for _, e := range evs { + if e.Kind == auth.EvGatewayRefused && e.IP == attacker { + n++ + } + } + if n != 1 || len(evs) != 2 { + t.Fatalf("after %d refusals from one address: %d events for it, %d in all, want 1 and 2", attempts, n, len(evs)) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index c3e44f51..292158fb 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -293,6 +293,12 @@ func handle(client net.Conn) { var agentKey bool var account auth.User if authStore != nil { + // Before the password is even asked for: an address that has used up + // its invalid keys costs no key lookup and no row in the security log. + if held, wait := authStore.GatewayThrottled(params["database"], remoteHost(client)); held { + sendError(client, "28000", fmt.Sprintf("too many invalid API keys from this address — try again in %s", wait.Round(time.Second))) + return + } key, err := requestClientPassword(client) if err != nil { log.Printf("gateway auth: %v", err) @@ -300,7 +306,7 @@ func handle(client net.Conn) { } u, scope, kind, ok := authStore.VerifyKeyKind(key) if !ok { - authStore.Audit(auth.EvGatewayRefused, "", params["database"], remoteHost(client), "invalid API key") + authStore.GatewayInvalidKey(params["database"], remoteHost(client)) sendError(client, "28P01", "invalid API key — use a key_ key as the password") return } @@ -311,7 +317,7 @@ func handle(client net.Conn) { // Refused before the branch is resolved, so such a key cannot even // wake one. if kind == auth.KindRealtime { - authStore.Audit(auth.EvGatewayRefused, u.Email, params["database"], remoteHost(client), "realtime key at the gateway") + authStore.GatewayRefused(auth.EvGatewayRefused, u.Email, params["database"], remoteHost(client), "realtime key at the gateway") sendError(client, "28P01", "this is a realtime key: it subscribes to the change feed and cannot run SQL — use a key_ key as the password") return } @@ -342,7 +348,7 @@ func handle(client net.Conn) { } actor = keyScope } else if gatewayACL != nil && !gatewayACL.Can(account, target, access.Use) { - authStore.Audit(auth.EvDenied, account.Email, target, remoteHost(client), "gateway") + authStore.GatewayRefused(auth.EvDenied, account.Email, target, remoteHost(client), "gateway") // Answered like a branch that does not exist, so a name is not // confirmed to someone who may not open it — and checked before the // branch is woken. From eaaae5e63ab5570f99d85205a3a0fe93ed120e1d Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:58:53 +0530 Subject: [PATCH 14/22] fix(promote): apply a request as db_client over pgx, in one transaction A request was applied by psql over stdin in a superuser's session, confined by SET LOCAL ROLE. psql read every statement as input of its own, so :names and backslash commands were live, and a statement that left the confinement had the superuser behind it. It now logs in as db_client, the role that owns what clients make, and runs in pgx.BeginFunc, each statement one message that only the server reads. The guardrail, the policy gate and the Blackbox fire as for any client, the actor still comes from bb.actor, and a failed apply says truly what was committed. Audit TX-5. Co-Authored-By: Claude Opus 5.5 --- enterprise/promote/promote.go | 115 +++++++++++++++++----- enterprise/promote/promote_test.go | 147 +++++++++++++++++++++++++++-- internal/branch/primitives.go | 26 ++++- 3 files changed, 254 insertions(+), 34 deletions(-) diff --git a/enterprise/promote/promote.go b/enterprise/promote/promote.go index 3b237c60..03afd2e8 100644 --- a/enterprise/promote/promote.go +++ b/enterprise/promote/promote.go @@ -10,8 +10,14 @@ package promote import ( + "context" + "errors" "fmt" "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/thefoxbyte/foxbyte/internal/auth" "github.com/thefoxbyte/foxbyte/internal/branch" @@ -218,6 +224,15 @@ func checkPolicyFor(target string, entries []branch.RequestEntry) error { // and the approver is recorded beside it: the Blackbox entry's session says which // request applied it, and the request says who approved it. One transaction means // the target is never left half-merged — either every statement is in, or none is. +// +// It logs in as the shared client role, over pgx. Ownership is why that role: +// applied as the superuser, a promoted table belongs to the admin role, and the +// next `ALTER TABLE` through the gateway fails with "must be owner of table", so +// running as what the gateway runs every client as (proxy.realUser) gives the +// object exactly the owner it has on the source. Logging in as it, rather than +// SET ROLE from a superuser's session, is why nothing in a statement can get back +// out. The Blackbox's attribution is unaffected: for a session logged in as this +// role it reads bb.actor, as it did for the superuser. func apply(requestID int64, target, approver string, entries []branch.RequestEntry) (int, error) { if len(entries) == 0 { return 0, branch.ErrNothingToPromote @@ -233,42 +248,96 @@ func apply(requestID int64, target, approver string, entries []branch.RequestEnt if _, err := branch.EnsureRunning(target); err != nil { return 0, err } - if err := branch.ExecScript(target, applyScript(requestID, approver, entries)); err != nil { + ctx := context.Background() + openCtx, cancel := context.WithTimeout(ctx, openTimeout) + defer cancel() + conn, err := branch.OpenClient(openCtx, target) + if err != nil { return 0, fmt.Errorf("applying the request to %q (nothing was committed): %w", target, err) } + defer conn.Close(ctx) + + replayed := false + err = pgx.BeginFunc(ctx, conn, func(tx pgx.Tx) error { + inTx := func() bool { return conn.PgConn().TxStatus() != 'I' } + if err := replay(ctx, tx, inTx, requestID, approver, entries); err != nil { + return err + } + replayed = true + return nil + }) + if err != nil { + return 0, applyFailure(target, replayed, err) + } return len(entries), nil } -// applyScript is the one transaction a request is applied in. -func applyScript(requestID int64, approver string, entries []branch.RequestEntry) string { - var b strings.Builder - b.WriteString("BEGIN;\n") +// openTimeout bounds the login. The statements themselves have no limit: an ALTER +// on a large table has always been allowed to take what it takes. +const openTimeout = 30 * time.Second + +// errTransactionEnded is a statement having ended the transaction it was given, +// which checkReplayable exists to prevent. If one ever gets past that, the apply +// stops there and says so, rather than report a rollback that did not happen. +var errTransactionEnded = errors.New("the statement ended the transaction it was applied in") + +// applyFailure says what a failed apply left behind. Every statement's error +// comes before the commit and rolls the transaction back, and a commit the server +// refused did too. Only a commit whose answer never arrived leaves it open. +func applyFailure(target string, replayed bool, err error) error { + var pgErr *pgconn.PgError + switch { + case errors.Is(err, errTransactionEnded): + return fmt.Errorf("applying the request to %q: part of it may be committed; compare the target with its Blackbox: %w", target, err) + case replayed && !errors.As(err, &pgErr) && !errors.Is(err, pgx.ErrTxCommitRollback): + return fmt.Errorf("applying the request to %q: every statement ran but the commit was not confirmed, so it may or may not be in; "+ + "look for this request's session in the target's Blackbox: %w", target, err) + } + return fmt.Errorf("applying the request to %q (nothing was committed): %w", target, err) +} + +// execer is the one call replay needs of a transaction. +type execer interface { + Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) +} + +// replay runs a request's statements in tx, each under its own author. inTx says +// whether the connection is still in that transaction. +// +// A statement goes to the server as its own message, exactly as recorded and with +// no arguments, which pgx sends by the simple protocol: one message may hold +// several statements, and nothing but the server reads it, so a :name or a +// backslash in it is only text. The settings are bound values for the same +// reason; set_config(…, true) is SET LOCAL. +func replay(ctx context.Context, tx execer, inTx func() bool, requestID int64, approver string, entries []branch.RequestEntry) error { // Set once for the whole transaction: which request this was, and that the - // engine applied it rather than a client. - fmt.Fprintf(&b, "SET LOCAL bb.session = %s;\n", branch.QuoteLiteral(fmt.Sprintf("request-%d", requestID))) - fmt.Fprintf(&b, "SET LOCAL application_name = %s;\n", branch.QuoteLiteral(brand.CLI+" request approve (by "+approver+")")) - // As the shared client role, which is what the gateway runs every client as - // (proxy.realUser). Ownership is the reason: applied as the superuser, a - // promoted table belongs to the admin role, and the next `ALTER TABLE` through - // the gateway fails with "must be owner of table" — the change arrives and then - // cannot be worked on. Running as the same role the gateway uses gives the - // object exactly the owner it has on the source. - // - // session_user stays the superuser, so the Blackbox's attribution — which reads - // session_user and falls back to bb.actor — is unaffected. - fmt.Fprintf(&b, "SET LOCAL ROLE %s;\n", branch.QuoteIdent(branch.ClientRole)) + // engine applied it rather than a client. standard_conforming_strings because + // the check read the statements with it on, whatever the role's default is. + if _, err := tx.Exec(ctx, "SELECT set_config('standard_conforming_strings', 'on', true), "+ + "set_config('bb.session', $1, true), set_config('application_name', $2, true)", + fmt.Sprintf("request-%d", requestID), brand.CLI+" request approve (by "+approver+")"); err != nil { + return fmt.Errorf("setting up the transaction: %w", err) + } for _, e := range entries { if e.SameRun != 0 { continue // its statement ran with the entry it shares it with } // Per statement: the actor is whoever ran it on the source, so the target's // record names them and not the approver. - fmt.Fprintf(&b, "SET LOCAL bb.actor = %s;\n", branch.QuoteLiteral(e.Actor)) + if _, err := tx.Exec(ctx, "SELECT set_config('bb.actor', $1, true)", e.Actor); err != nil { + return fmt.Errorf("entry %d: %w", e.ID, err) + } if e.ActorKind != "" { - fmt.Fprintf(&b, "SET LOCAL bb.actor_kind = %s;\n", branch.QuoteLiteral(e.ActorKind)) + if _, err := tx.Exec(ctx, "SELECT set_config('bb.actor_kind', $1, true)", e.ActorKind); err != nil { + return fmt.Errorf("entry %d: %w", e.ID, err) + } + } + if _, err := tx.Exec(ctx, strings.TrimSpace(e.Statement)); err != nil { + return fmt.Errorf("entry %d: %w", e.ID, err) + } + if !inTx() { + return fmt.Errorf("entry %d: %w", e.ID, errTransactionEnded) } - b.WriteString(strings.TrimRight(strings.TrimSpace(e.Statement), ";") + ";\n") } - b.WriteString("RESET ROLE;\nCOMMIT;\n") - return b.String() + return nil } diff --git a/enterprise/promote/promote_test.go b/enterprise/promote/promote_test.go index 6f296bb0..9f2785d4 100644 --- a/enterprise/promote/promote_test.go +++ b/enterprise/promote/promote_test.go @@ -5,9 +5,15 @@ package promote import ( + "context" + "errors" + "os" "strings" "testing" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/thefoxbyte/foxbyte/internal/auth" "github.com/thefoxbyte/foxbyte/internal/branch" ) @@ -70,9 +76,41 @@ func TestBuildRequestChecksNames(t *testing.T) { } } +// recordingTx stands in for the transaction: what replay asked of it, in order. +type recordingTx struct { + calls []execCall + failOn string +} + +type execCall struct { + sql string + args []any +} + +func (r *recordingTx) Exec(_ context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + r.calls = append(r.calls, execCall{sql, args}) + if r.failOn != "" && strings.Contains(sql, r.failOn) { + return pgconn.CommandTag{}, errors.New("boom") + } + return pgconn.CommandTag{}, nil +} + +// The statements replay sent, without the settings between them. +func (r *recordingTx) statements() []string { + var out []string + for _, c := range r.calls { + if !strings.HasPrefix(c.sql, "SELECT set_config(") { + out = append(out, c.sql) + } + } + return out +} + +func alwaysInTx() bool { return true } + // One run of two statements is two entries, each holding the run's whole -// text: the script runs it once, at the first, under its author. -func TestApplyScriptRunsASharedStatementOnce(t *testing.T) { +// text: replay runs it once, at the first, under its author. +func TestReplayRunsASharedStatementOnce(t *testing.T) { run := "CREATE TABLE t (id int); CREATE FUNCTION t_add(i int) RETURNS void LANGUAGE sql AS $$ INSERT INTO t VALUES (i) $$" entries := []branch.RequestEntry{ {ID: 171, Actor: "ada@example.com", CommandTag: "CREATE TABLE", Statement: run}, @@ -80,14 +118,105 @@ func TestApplyScriptRunsASharedStatementOnce(t *testing.T) { {ID: 173, Actor: "bob@example.com", CommandTag: "CREATE INDEX", Statement: "CREATE INDEX ON t (id)"}, } branch.ReplayOnce(entries, map[int64]int64{171: 900, 172: 900, 173: 901}) - s := applyScript(5, "carol@example.com", entries) - if n := strings.Count(s, "CREATE TABLE t (id int)"); n != 1 { - t.Fatalf("the shared run is in the script %d times:\n%s", n, s) + tx := &recordingTx{} + if err := replay(context.Background(), tx, alwaysInTx, 5, "carol@example.com", entries); err != nil { + t.Fatal(err) + } + got := tx.statements() + if len(got) != 2 || got[0] != run || got[1] != "CREATE INDEX ON t (id)" { + t.Fatalf("statements = %q", got) + } + var actors []any + for _, c := range tx.calls { + if strings.Contains(c.sql, "'bb.actor'") { + actors = append(actors, c.args...) + } + } + if len(actors) != 2 || actors[0] != "ada@example.com" || actors[1] != "bob@example.com" { + t.Fatalf("actors = %v", actors) } - if !strings.Contains(s, "CREATE INDEX ON t (id);") || !strings.Contains(s, "SET LOCAL bb.actor = 'bob@example.com'") { - t.Fatalf("the entry of its own is missing:\n%s", s) +} + +// The apply path is not psql: each statement is one Exec, verbatim and with no +// arguments, so nothing but the server reads it. The transaction is the +// caller's, and replay issues none of its own control. +func TestReplayNeverGoesThroughPsql(t *testing.T) { + stmt := "COMMENT ON TABLE t IS E':var \\\\! rm -rf / \\\\gexec'" + tx := &recordingTx{} + if err := replay(context.Background(), tx, alwaysInTx, 7, "carol@example.com", + []branch.RequestEntry{{ID: 1, Actor: "ada@example.com", Statement: " " + stmt + "\n"}}); err != nil { + t.Fatal(err) } - if strings.Index(s, "CREATE TABLE t") > strings.Index(s, "CREATE INDEX") { - t.Fatalf("out of order:\n%s", s) + if got := tx.statements(); len(got) != 1 || got[0] != stmt { + t.Fatalf("the statement was changed on its way: %q", got) + } + for _, c := range tx.calls { + if c.sql == stmt && len(c.args) != 0 { + t.Fatalf("the statement was sent with arguments: %v", c.args) + } + for _, control := range []string{"BEGIN", "COMMIT", "SET LOCAL", "RESET ROLE", "SET ROLE"} { + if strings.Contains(c.sql, control) { + t.Fatalf("replay issued %s itself: %q", control, c.sql) + } + } + } + src, err := os.ReadFile("promote.go") + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(src), "ExecScript(") { + t.Fatal("the apply path goes through branch.ExecScript, which is psql over stdin") + } +} + +// The statement after the one that fails is not attempted, and the failure +// names the entry. +func TestReplayStopsAtTheFirstFailure(t *testing.T) { + tx := &recordingTx{failOn: "CREATE VIEW"} + err := replay(context.Background(), tx, alwaysInTx, 1, "c", []branch.RequestEntry{ + {ID: 1, Statement: "CREATE TABLE a ()"}, {ID: 2, Statement: "CREATE VIEW b AS SELECT 1"}, {ID: 3, Statement: "CREATE TABLE c ()"}, + }) + if err == nil || !strings.Contains(err.Error(), "entry 2") { + t.Fatalf("err = %v", err) + } + if got := tx.statements(); len(got) != 2 { + t.Fatalf("carried on after the failure: %q", got) + } +} + +// If a statement ever did end the transaction, the next must not run. +func TestReplayStopsIfTheTransactionEnded(t *testing.T) { + tx := &recordingTx{} + err := replay(context.Background(), tx, func() bool { return false }, 1, "c", []branch.RequestEntry{ + {ID: 1, Statement: "CREATE TABLE a ()"}, {ID: 2, Statement: "CREATE TABLE b ()"}, + }) + if !errors.Is(err, errTransactionEnded) { + t.Fatalf("err = %v", err) + } + if got := tx.statements(); len(got) != 1 { + t.Fatalf("carried on outside the transaction: %q", got) + } +} + +// What a failure says was committed has to be true. +func TestApplyFailureSaysWhatWasCommitted(t *testing.T) { + refused := &pgconn.PgError{Message: "deferred constraint"} + lost := errors.New("connection reset") + cases := []struct { + name string + replayed bool + err error + want string + }{ + {"a statement failed", false, lost, "nothing was committed"}, + {"the server refused the commit", true, refused, "nothing was committed"}, + {"the server rolled back at commit", true, pgx.ErrTxCommitRollback, "nothing was committed"}, + {"the commit's answer never came", true, lost, "may or may not be in"}, + {"a statement ended the transaction", false, errTransactionEnded, "part of it may be committed"}, + } + for _, c := range cases { + if got := applyFailure("main", c.replayed, c.err).Error(); !strings.Contains(got, c.want) { + t.Errorf("%s: %q does not say %q", c.name, got, c.want) + } } } diff --git a/internal/branch/primitives.go b/internal/branch/primitives.go index 4ad7a8f1..56935d94 100644 --- a/internal/branch/primitives.go +++ b/internal/branch/primitives.go @@ -109,8 +109,8 @@ func PolicyQuery(name, sql string) ([]string, error) { return policyLines(name, func ResolveBranch(name string) (string, error) { return ledgerBranchName(name) } // ExecScript runs a multi-statement SQL script on a branch over stdin, aborting -// at the first error. Applying a change request is several statements that must -// stand or fall together, which ExecSQL's one statement at a time cannot do. +// at the first error, as the superuser. For the engine's own scripts only: psql +// reads its input as commands, so text a client wrote must not go through here. func ExecScript(target, sql string) error { return psqlStdin(target, sql) } // SetGuard enables or disables the Blackbox guardrail event trigger on a @@ -153,6 +153,28 @@ func OpenAs(ctx context.Context, name, account string) (*pgx.Conn, error) { return pgx.ConnectConfig(ctx, cfg) } +// OpenClient connects to the branch's database as the shared client role itself: +// what a change request is applied over. db_client owns what clients make, is no +// superuser and is a member of nothing, and the Blackbox takes its actor from +// bb.actor for a session logged in as it, so replayed statements have only a +// client's rights and are still recorded under whoever wrote them. +func OpenClient(ctx context.Context, name string) (*pgx.Conn, error) { + target := ServingBranch(name) + if err := EnsureAppRole(target); err != nil { + return nil, fmt.Errorf("setting %s's password on %s: %w", ClientRole, name, err) + } + ip, err := containerIP(container(target)) + if err != nil { + return nil, fmt.Errorf("finding %s: %w", name, err) + } + cfg, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:5432/%s", ip, pgDatabase)) + if err != nil { + return nil, err + } + cfg.User, cfg.Password = ClientRole, ClientRolePassword() + return pgx.ConnectConfig(ctx, cfg) +} + // QuoteIdent renders a name as a Postgres identifier, preserving its case. func QuoteIdent(s string) string { return sqlIdent(s) } From 94ff1e359ce67be4884fa1e487282126f8d59778 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:59:25 +0530 Subject: [PATCH 15/22] perf(console): serve status, branches and replication from one snapshot Each dashboard poll walked the branches, an inspect and a psql exec apiece, serially and per client. One read now serves everybody for 5s, single flight, out of date as soon as this process changes anything. Branches uses one docker inspect for all states and counts connections over the network, falling back to the exec. Access filtering stays after the shared list, per request. Docker's reads behind the snapshot are on the probe tier, and a caller with an earlier answer waits at most 4s for a new one. Co-Authored-By: Claude Opus 5.5 --- internal/branch/branch.go | 88 ++++++-- internal/branch/ha.go | 12 +- internal/branch/replication.go | 30 ++- internal/branch/snapshot.go | 157 ++++++++++++++ internal/branch/snapshot_test.go | 287 +++++++++++++++++++++++++ internal/controlplane/server.go | 51 ++++- internal/controlplane/snapshot_test.go | 111 ++++++++++ 7 files changed, 700 insertions(+), 36 deletions(-) create mode 100644 internal/branch/snapshot.go create mode 100644 internal/branch/snapshot_test.go create mode 100644 internal/controlplane/snapshot_test.go diff --git a/internal/branch/branch.go b/internal/branch/branch.go index 5bdcaf6a..3bf659ae 100644 --- a/internal/branch/branch.go +++ b/internal/branch/branch.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/jackc/pgx/v5" "github.com/thefoxbyte/foxbyte/internal/ledger" "github.com/thefoxbyte/foxbyte/internal/secrets" ) @@ -1518,37 +1519,94 @@ type BranchInfo struct { // Branches returns structured info for every branch (including main), enriched // with container state, connections, and copy-on-write size. +// +// One docker inspect for every branch's state and address, then each running +// branch's connection count, several at a time. A docker that does not answer +// shows every branch as absent, as it always did: this is a report, and nothing +// that removes a container reads it. func Branches() ([]BranchInfo, error) { - usages, err := activeStorage().usage() + usages, err := branchUsages() if err != nil { return nil, err } var infos []BranchInfo + var conts []string for _, u := range usages { name := u.Name if strings.Contains(name, "/") { continue // only direct children } - bi := BranchInfo{ + infos = append(infos, BranchInfo{ Masked: maskedStateOf(name), Name: name, Used: u.Used, Refer: u.Refer, Primary: name == "main", Agent: strings.HasPrefix(name, "agent-"), - State: ContainerState(name), + State: "absent", + }) + conts = append(conts, container(name)) + } + if len(conts) == 0 { + return infos, nil + } + out, _ := inspectContainers(execProbe, conts) // what it found, whatever else it says + facts := parseFacts(out) + forEachBounded(len(infos), func(i int) { + f, ok := facts[conts[i]] + if !ok { + return } - if bi.State == "running" { - bi.Port = "5432" // in-container port; branches are reached via the gateway - if c, err := ActiveConnections(name); err == nil { - bi.Connections = c + infos[i].State = f.state + if f.state == "running" { + infos[i].Port = "5432" // in-container port; branches are reached via the gateway + if c, err := countConnections(infos[i].Name, f.ip); err == nil { + infos[i].Connections = c } } - infos = append(infos, bi) - } + }) return infos, nil } +// branchUsages and countConnections are Branches' other two sources, variables +// so a test can count them. +var ( + branchUsages = func() ([]branchUsage, error) { return activeStorage().usage() } + + countConnections = func(name, ip string) (int, error) { + if ip != "" { + if n, err := connectionsOver(ip); err == nil { + return n, nil + } + } + return activeConnectionsWithin(execProbe, name) + } +) + +// connectionsOver reads a branch's connection count over the docker network, +// which stateConn already does as this role: no sudo, docker or psql process for +// a number the dashboard wants for every branch every few seconds. Its callers +// fall back to the exec, so a branch this cannot reach is no worse off. +func connectionsOver(ip string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cfg, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s@%s:5432/%s", pgUser, ip, pgDatabase)) + if err != nil { + return 0, err + } + cfg.Password = pgPass() + conn, err := pgx.ConnectConfig(ctx, cfg) + if err != nil { + return 0, err + } + defer conn.Close(context.Background()) + var n int + if err := conn.QueryRow(ctx, activeConnectionsSQL).Scan(&n); err != nil { + return 0, err + } + return n, nil +} + // Storage summarises pool usage for the dashboard. type Storage struct { Used string `json:"used"` @@ -1737,10 +1795,14 @@ func Wake(name string) error { // ActiveConnections returns the number of client connections currently open to a // branch (used to avoid suspending a branch that is in use). -func ActiveConnections(name string) (int, error) { - out, err := capture("docker", "exec", "--env-file", pgEnvFile(), container(name), - "psql", "-U", pgUser, "-d", pgDatabase, "-tAc", - "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend' AND pid<>pg_backend_pid();") +func ActiveConnections(name string) (int, error) { return activeConnectionsWithin(execQuick, name) } + +const activeConnectionsSQL = "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend' AND pid<>pg_backend_pid();" + +// activeConnectionsWithin is ActiveConnections bounded by d. +func activeConnectionsWithin(d time.Duration, name string) (int, error) { + out, err := captureWithin(d, "docker", "exec", "--env-file", pgEnvFile(), container(name), + "psql", "-U", pgUser, "-d", pgDatabase, "-tAc", activeConnectionsSQL) if err != nil { return 0, err } diff --git a/internal/branch/ha.go b/internal/branch/ha.go index 1c23d3a4..35a29517 100644 --- a/internal/branch/ha.go +++ b/internal/branch/ha.go @@ -29,12 +29,18 @@ type HAState struct { } // HAInfo reports current HA status. -func HAInfo() HAState { +func HAInfo() HAState { return haInfoWithin(execQuick) } + +// HAInfoProbe is HAInfo for the console's status, which must answer whatever the +// state of docker. +func HAInfoProbe() HAState { return haInfoWithin(execProbe) } + +func haInfoWithin(d time.Duration) HAState { st := HAState{Primary: "main", Sync: SyncOn()} if PrimaryContainer() == container("standby") { st.Primary = "standby" } - cs := ContainerState("standby") + cs := containerStateWithin(d, "standby") if cs == "absent" { return st } @@ -43,7 +49,7 @@ func HAInfo() HAState { // Both facts in one round trip, because this feeds the dashboard and the // status API: a second `docker exec psql` per call would double what the // page waits for to learn one extra bit. - out, _ := capture("docker", "exec", "--env-file", pgEnvFile(), PrimaryContainer(), + out, _ := captureWithin(d, "docker", "exec", "--env-file", pgEnvFile(), PrimaryContainer(), "psql", "-U", pgUser, "-d", pgDatabase, "-tAc", "SELECT (SELECT count(*) FROM pg_stat_replication WHERE state='streaming')::text"+ " || '|' || (current_setting('synchronous_standby_names') <> '')::text") diff --git a/internal/branch/replication.go b/internal/branch/replication.go index 15598ed7..cced9e07 100644 --- a/internal/branch/replication.go +++ b/internal/branch/replication.go @@ -59,22 +59,33 @@ func parseReplication(name string, lines func() ([]string, error)) (Replication, return r, nil } +// runningBranchNames and replicationOf are Replicating's sources, variables so a +// test can count them. +var ( + runningBranchNames = RunningBranches + replicationOf = ReplicationStatus +) + // Replicating lists the running branches that have a continuous import. func Replicating() ([]Replication, error) { - names, err := RunningBranches() + names, err := runningBranchNames() if err != nil { return nil, err } - out := []Replication{} - for _, n := range names { - if n == "main" { - continue // an import always lands in its own branch + found := make([]Replication, len(names)) + forEachBounded(len(names), func(i int) { + if names[i] == "main" { + return // an import always lands in its own branch } - r, err := ReplicationStatus(n) - if err != nil || !r.Replicating { - continue + if r, err := replicationOf(names[i]); err == nil { + found[i] = r + } + }) + out := []Replication{} + for _, r := range found { + if r.Replicating { + out = append(out, r) } - out = append(out, r) } return out, nil } @@ -90,6 +101,7 @@ func CutoverReplication(name string) (int, error) { if !r.Replicating { return 0, fmt.Errorf("%w: branch %q has no replication to cut over", ErrNotReplicating, name) } + defer noteStateChange() // the import page must stop listing it if err := ImportCutoverTo(&Progress{}, name); err != nil { return 0, err } diff --git a/internal/branch/snapshot.go b/internal/branch/snapshot.go new file mode 100644 index 00000000..a642d149 --- /dev/null +++ b/internal/branch/snapshot.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "fmt" + "slices" + "sync" + "time" +) + +// The dashboard asks what every branch is doing, and the import page which of +// them are replicating, every few seconds per open tab. Each answer was a +// serial walk over the branches, an inspect and a `docker exec psql` apiece, so +// the cost was tabs times branches, and twenty tabs on two branches was p95 1.2s. +// +// One answer now serves everybody for a few seconds. Nothing per-caller belongs +// in a Snapshot: the API filters what it is given by the caller's access after +// it comes out, and must go on doing that. +const ( + // SnapshotTTL is how old a shared answer may be. It bounds how long a change + // made by another process (the gateway suspending an idle branch, the CLI) + // is unseen on the dashboard. This process's own changes are seen at once. + SnapshotTTL = 5 * time.Second + // SnapshotWait is how long a caller holding the previous answer waits for a + // new one before showing the previous one instead. With no previous answer + // it waits for as long as the read takes, as it always did. + SnapshotWait = 4 * time.Second +) + +// Snapshot is one expensive answer shared between callers: at most one read at +// a time, kept for a TTL, and out of date the moment this process changes +// anything (noteStateChange), so the page that made a change shows it. +// +// A failed read is not kept: the next caller tries again. +type Snapshot[T any] struct { + ttl, wait time.Duration + read func() (T, error) + + mu sync.Mutex + val T + has bool + at time.Time + gen uint64 + flight *snapshotFlight[T] +} + +type snapshotFlight[T any] struct { + gen uint64 + done chan struct{} + val T + err error +} + +// NewSnapshot makes a Snapshot of read. +func NewSnapshot[T any](ttl, wait time.Duration, read func() (T, error)) *Snapshot[T] { + return &Snapshot[T]{ttl: ttl, wait: wait, read: read} +} + +// Get returns the shared answer, reading it first if it has gone out of date. +func (s *Snapshot[T]) Get() (T, error) { + s.mu.Lock() + gen := stateGen.Load() + if s.has && s.gen == gen && stateNow().Sub(s.at) < s.ttl { + v := s.val + s.mu.Unlock() + return v, nil + } + // A read begun before a change may have seen the world from before it, so + // nobody who comes after the change joins it. + fl := s.flight + if fl == nil || fl.gen != gen { + fl = &snapshotFlight[T]{gen: gen, done: make(chan struct{})} + s.flight = fl + go s.fetch(fl) + } + prev, hasPrev := s.val, s.has + s.mu.Unlock() + + if !hasPrev { + <-fl.done + return fl.val, fl.err + } + timer := time.NewTimer(s.wait) + defer timer.Stop() + select { + case <-fl.done: + return fl.val, fl.err + case <-timer.C: + return prev, nil + } +} + +func (s *Snapshot[T]) fetch(fl *snapshotFlight[T]) { + defer close(fl.done) + func() { + // On its own goroutine, where net/http's recovery does not reach: a + // panic here would take the whole control plane with it. + defer func() { + if p := recover(); p != nil { + fl.err = fmt.Errorf("reading a snapshot: %v", p) + } + }() + fl.val, fl.err = s.read() + }() + + s.mu.Lock() + defer s.mu.Unlock() + if s.flight == fl { + s.flight = nil + } + if fl.err == nil && fl.gen == stateGen.Load() { + s.val, s.has, s.at, s.gen = fl.val, true, stateNow(), fl.gen + } +} + +// branchesSnapshot and replicatingSnapshot are the shared answers the console's +// polling reads. +var ( + branchesSnapshot = NewSnapshot(SnapshotTTL, SnapshotWait, func() ([]BranchInfo, error) { return Branches() }) + replicatingSnapshot = NewSnapshot(SnapshotTTL, SnapshotWait, func() ([]Replication, error) { return Replicating() }) +) + +// BranchesSnapshot is Branches as of a few seconds ago, for callers polling it. +// For a display only: anything deciding what to do to a branch uses Branches or +// ContainerState. Every caller gets its own slice. +func BranchesSnapshot() ([]BranchInfo, error) { + bs, err := branchesSnapshot.Get() + return slices.Clone(bs), err +} + +// ReplicatingSnapshot is Replicating as of a few seconds ago, for callers +// polling it. Every caller gets its own slice. +func ReplicatingSnapshot() ([]Replication, error) { + rs, err := replicatingSnapshot.Get() + return slices.Clone(rs), err +} + +// maxProbeFanOut bounds how many branches are asked about at once, so a large +// install does not start every process together. +const maxProbeFanOut = 8 + +// forEachBounded runs f(0..n-1) with at most maxProbeFanOut running at a time, +// and returns when all have finished. +func forEachBounded(n int, f func(i int)) { + sem := make(chan struct{}, maxProbeFanOut) + var wg sync.WaitGroup + wg.Add(n) + for i := range n { + sem <- struct{}{} + go func() { + defer func() { <-sem; wg.Done() }() + f(i) + }() + } + wg.Wait() +} diff --git a/internal/branch/snapshot_test.go b/internal/branch/snapshot_test.go new file mode 100644 index 00000000..7051d8b4 --- /dev/null +++ b/internal/branch/snapshot_test.go @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestSnapshotIsSharedForItsTTL(t *testing.T) { + _, advance := withFakeDocker(t) + reads := 0 + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { reads++; return reads, nil }) + for range 10 { + if v, err := s.Get(); err != nil || v != 1 { + t.Fatalf("Get = %d, %v", v, err) + } + } + if reads != 1 { + t.Fatalf("%d reads for 10 callers, want 1", reads) + } + advance(SnapshotTTL) + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get after the TTL = %d, want a new read", v) + } +} + +// Create, delete and suspend have to show on the very next poll. +func TestSnapshotIsOutOfDateAfterAStateChange(t *testing.T) { + _, _ = withFakeDocker(t) + reads := 0 + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { reads++; return reads, nil }) + _, _ = s.Get() + noteIfChange("docker", []string{"stop", "pg-a"}) + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get = %d: a change this process made was not seen", v) + } +} + +func TestSnapshotConcurrentCallersShareOneRead(t *testing.T) { + _, _ = withFakeDocker(t) + var reads atomic.Int64 + gate := make(chan struct{}) + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { reads.Add(1); <-gate; return 7, nil }) + const n = 50 + var wg sync.WaitGroup + wg.Add(n) + for range n { + go func() { + defer wg.Done() + if v, err := s.Get(); err != nil || v != 7 { + t.Errorf("Get = %d, %v", v, err) + } + }() + } + for reads.Load() == 0 { + time.Sleep(time.Millisecond) + } + time.Sleep(20 * time.Millisecond) + close(gate) + wg.Wait() + if got := reads.Load(); got != 1 { + t.Fatalf("%d reads for %d concurrent callers, want 1", got, n) + } +} + +// A read begun before a change is neither joined nor kept. Otherwise the poll +// that follows a suspend could be given the list from before it. +func TestSnapshotDoesNotKeepAReadFromBeforeAChange(t *testing.T) { + _, _ = withFakeDocker(t) + var reads atomic.Int64 + gate := make(chan struct{}) + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { + if reads.Add(1) == 1 { + <-gate + return 1, nil // read before the change + } + return 2, nil + }) + first := make(chan int, 1) + go func() { v, _ := s.Get(); first <- v }() + for reads.Load() == 0 { + time.Sleep(time.Millisecond) + } + noteStateChange() + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get after the change = %d, want the read begun after it", v) + } + close(gate) + <-first + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get = %d: the older read replaced the newer", v) + } +} + +func TestSnapshotDoesNotKeepAFailure(t *testing.T) { + _, _ = withFakeDocker(t) + fail := true + boom := errors.New("boom") + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { + if fail { + return 0, boom + } + return 1, nil + }) + if _, err := s.Get(); !errors.Is(err, boom) { + t.Fatalf("err = %v", err) + } + fail = false + if v, err := s.Get(); err != nil || v != 1 { + t.Fatalf("Get = %d, %v: a failure was kept", v, err) + } +} + +// With an earlier answer to hand, nobody waits on a hung read for longer than +// SnapshotWait: this is what stands between a stuck docker and the dashboard. +func TestSnapshotServesThePreviousAnswerRatherThanHang(t *testing.T) { + _, _ = withFakeDocker(t) + var reads atomic.Int64 + gate := make(chan struct{}) + defer close(gate) + s := NewSnapshot(SnapshotTTL, 20*time.Millisecond, func() (int, error) { + if reads.Add(1) > 1 { + <-gate + } + return 1, nil + }) + if v, _ := s.Get(); v != 1 { + t.Fatal(v) + } + noteStateChange() // out of date: the next read will hang + done := make(chan int, 1) + go func() { v, _ := s.Get(); done <- v }() + select { + case v := <-done: + if v != 1 { + t.Fatalf("Get = %d, want the previous answer", v) + } + case <-time.After(5 * time.Second): + t.Fatal("hung on a read that will not finish") + } +} + +func TestSnapshotSurvivesAPanickingRead(t *testing.T) { + _, _ = withFakeDocker(t) + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { panic("no") }) + if _, err := s.Get(); err == nil { + t.Fatal("a panicking read returned no error") + } + s.read = func() (int, error) { return 1, nil } + if v, err := s.Get(); err != nil || v != 1 { + t.Fatalf("Get = %d, %v: the failed read stayed in flight", v, err) + } +} + +func TestForEachBoundedRunsEachOnce(t *testing.T) { + var running, peak atomic.Int64 + seen := make([]atomic.Int64, 40) + forEachBounded(len(seen), func(i int) { + now := running.Add(1) + for { + p := peak.Load() + if now <= p || peak.CompareAndSwap(p, now) { + break + } + } + time.Sleep(time.Millisecond) + running.Add(-1) + seen[i].Add(1) + }) + for i := range seen { + if seen[i].Load() != 1 { + t.Fatalf("index %d ran %d times", i, seen[i].Load()) + } + } + if peak.Load() > maxProbeFanOut { + t.Fatalf("%d ran at once, the bound is %d", peak.Load(), maxProbeFanOut) + } +} + +// branchesWorld installs the fakes behind Branches: a pool of datasets and a +// count of what is asked about them. +func branchesWorld(t *testing.T, datasets ...string) (*fakeDocker, *atomic.Int64, func(time.Duration)) { + t.Helper() + d, advance := withFakeDocker(t) + var counted atomic.Int64 + oldUsage, oldCount := branchUsages, countConnections + branchUsages = func() ([]branchUsage, error) { + var u []branchUsage + for _, n := range datasets { + u = append(u, branchUsage{Name: n, Used: "1M", Refer: "2M"}) + } + return u, nil + } + countConnections = func(string, string) (int, error) { counted.Add(1); return 3, nil } + t.Cleanup(func() { branchUsages, countConnections = oldUsage, oldCount }) + return d, &counted, advance +} + +// One inspect however many branches there are, and an answer of the same shape +// the serial walk gave: a stopped branch has no port or count, and one with no +// container is absent. +func TestBranchesUsesOneInspect(t *testing.T) { + d, counted, _ := branchesWorld(t, "main", "dev", "agent-x", "gone") + d.set("pg-main", "running", "10.0.0.1") + d.set("pg-dev", "exited", "") + d.set("pg-agent-x", "running", "10.0.0.3") + + got, err := Branches() + if err != nil { + t.Fatal(err) + } + if n := d.calls.Load(); n != 1 { + t.Fatalf("%d inspects for 4 branches, want 1", n) + } + if n := counted.Load(); n != 2 { + t.Fatalf("%d connection counts, want one per running branch", n) + } + want := []BranchInfo{ + {Name: "main", Primary: true, State: "running", Used: "1M", Refer: "2M", Connections: 3, Port: "5432"}, + {Name: "dev", State: "exited", Used: "1M", Refer: "2M"}, + {Name: "agent-x", Agent: true, State: "running", Used: "1M", Refer: "2M", Connections: 3, Port: "5432"}, + {Name: "gone", State: "absent", Used: "1M", Refer: "2M"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Branches =\n%+v\nwant\n%+v", got, want) + } +} + +// Callers filter and sort what they are given. One caller's doing that must +// not show up in another's list. +func TestBranchesSnapshotGivesEachCallerItsOwnSlice(t *testing.T) { + d, _, _ := branchesWorld(t, "main") + d.set("pg-main", "running", "10.0.0.1") + noteStateChange() // the package-level snapshot, between tests + a, err := BranchesSnapshot() + if err != nil { + t.Fatal(err) + } + a[0].Name = "changed by a caller" + b, _ := BranchesSnapshot() + if b[0].Name != "main" { + t.Fatalf("the shared list was changed through a caller's copy: %+v", b[0]) + } +} + +// The measurement behind the change. 20 tabs on 2 branches for a minute: each +// tab used to poll /api/status and /api/branches every 3s, and each of those +// walked the branches. Now 10s polls of one shared snapshot. Run with -v. +func TestDashboardDockerCalls(t *testing.T) { + d, counted, step := branchesWorld(t, "main", "dev") + d.set("pg-main", "running", "10.0.0.1") + d.set("pg-dev", "running", "10.0.0.2") + noteStateChange() + const clients, minute = 20, 60 + + polls := 0 + var clock time.Duration + for clock < minute*time.Second { + var wg sync.WaitGroup + wg.Add(clients * 2) + for range clients * 2 { // status and branches read the same snapshot + go func() { + defer wg.Done() + if _, err := BranchesSnapshot(); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + polls++ + step(10 * time.Second) + clock += 10 * time.Second + } + inspects, conns := d.calls.Load(), counted.Load() + // Per request before: an inspect and a psql exec per branch, serially. + legacy := clients * (minute / 3) * 2 * (2 + 2) + t.Logf("dashboard: %d tabs, 2 branches, 60s: %d docker inspects + %d connection counts over the network, was %d docker processes", + clients, inspects, conns, legacy) + if inspects > int64(polls)+1 { + t.Fatalf("%d inspects over %d polls: not shared", inspects, polls) + } +} diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 964a7628..f2a8bf34 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -204,8 +204,8 @@ func checkBranchName(next http.Handler) http.Handler { }) } -// backupHealth caches branch.CurrentBackupHealth for a minute: reading it lists -// the backups from a container, and the dashboard polls /api/status. +// backupHealth caches the backup judgement for a minute: reading it lists the +// backups from a container, and the dashboard polls /api/status. var backupHealth struct { sync.Mutex at time.Time @@ -216,11 +216,39 @@ func cachedBackupHealth() branch.BackupHealth { backupHealth.Lock() defer backupHealth.Unlock() if time.Since(backupHealth.at) > time.Minute { - backupHealth.h, backupHealth.at = branch.CurrentBackupHealth(), time.Now() + // Listing can start a container, so it gets more than a probe; still + // seconds, because this is answering a page. "Could not tell" is a + // verdict the dashboard already knows how to show. + backupHealth.h, backupHealth.at = branch.CurrentBackupHealthProbe(30*time.Second), time.Now() } return backupHealth.h } +// The console's polling reads these rather than walking the branches itself +// (branch/snapshot.go): variables so a test can count what is behind them. +var ( + listBranches = branch.BranchesSnapshot + listReplicating = branch.ReplicatingSnapshot +) + +// statusFacts is the engine's part of /api/status, the same for every caller. +type statusFacts struct { + branches []branch.BranchInfo + ha branch.HAState + backup branch.BackupHealth + storage branch.Storage +} + +var statusSnapshot = branch.NewSnapshot(branch.SnapshotTTL, branch.SnapshotWait, readStatusFacts) + +func readStatusFacts() (statusFacts, error) { + bs, err := listBranches() + if err != nil { + return statusFacts{}, err + } + return statusFacts{branches: bs, ha: branch.HAInfoProbe(), backup: cachedBackupHealth(), storage: branch.StorageInfo()}, nil +} + // isUpload is the one route whose body is a file rather than JSON; it has its // own, larger cap (maxUpload). func isUpload(r *http.Request) bool { return r.URL.Path == "/api/import/file" } @@ -237,14 +265,14 @@ func maxUpload() int64 { func registerAPI(mux *http.ServeMux) { mux.HandleFunc("GET /api/status", func(w http.ResponseWriter, r *http.Request) { - bs, err := branch.Branches() + st, err := statusSnapshot.Get() if err != nil { writeErr(w, 500, err) return } var nBranch, nAgent int mainReady := false - for _, b := range bs { + for _, b := range st.branches { switch { case b.Primary: mainReady = b.State == "running" @@ -264,9 +292,9 @@ func registerAPI(mux *http.ServeMux) { // and so does an Enterprise build with no licence. "edition": edition.Name(), "features": featureNames(), - "ha": branch.HAInfo(), - "backup": cachedBackupHealth(), - "storage": branch.StorageInfo(), + "ha": st.ha, + "backup": st.backup, + "storage": st.storage, "servers": map[string]bool{ "gateway": daemon.Alive("gateway"), "api": daemon.Alive("api"), @@ -275,13 +303,14 @@ func registerAPI(mux *http.ServeMux) { }) mux.HandleFunc("GET /api/branches", func(w http.ResponseWriter, r *http.Request) { - bs, err := branch.Branches() + bs, err := listBranches() if err != nil { writeErr(w, 500, err) return } // Only the branches the caller may reach: main, their own, and — for - // an admin — all of them. + // an admin — all of them. Here, per request, and never in the shared + // list above, which is the same for everybody. u, _ := auth.UserFrom(r.Context()) visible := []branch.BranchInfo{} for _, b := range bs { @@ -498,7 +527,7 @@ func registerAPI(mux *http.ServeMux) { // Continuous imports (logical replication into a branch): where each stands, // and the cutover that turns one into a standalone branch. mux.HandleFunc("GET /api/replication", func(w http.ResponseWriter, r *http.Request) { - list, err := branch.Replicating() + list, err := listReplicating() if err != nil { writeErr(w, 500, err) return diff --git a/internal/controlplane/snapshot_test.go b/internal/controlplane/snapshot_test.go new file mode 100644 index 00000000..d9106277 --- /dev/null +++ b/internal/controlplane/snapshot_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controlplane + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "sync/atomic" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// Everybody reads one shared list and each still sees only their own part of +// it. This is the property a snapshot could quietly lose: the list is cached, +// and who may see what must never be. +func TestSharedBranchListIsFilteredPerCaller(t *testing.T) { + s, admin, alice, bob := withACL(t) + var reads atomic.Int64 + snap := branch.NewSnapshot(time.Hour, time.Second, func() ([]branch.BranchInfo, error) { + reads.Add(1) + return []branch.BranchInfo{{Name: "main", Primary: true}, {Name: "alice-dev"}, {Name: "bobs-secret"}}, nil + }) + old := listBranches + listBranches = snap.Get + t.Cleanup(func() { listBranches = old }) + + api := http.NewServeMux() + registerAPI(api) + h := s.Authn(api) + + get := func(u auth.User) []string { + t.Helper() + key, _, err := s.CreateAPIKey(u.ID, "t") + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest("GET", "/api/branches", nil) + r.Header.Set("Authorization", "Bearer "+key) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("%s: %d %s", u.Email, w.Code, w.Body) + } + var out []branch.BranchInfo + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + names := []string{} + for _, b := range out { + names = append(names, b.Name) + } + sort.Strings(names) + return names + } + // Bob first, then the owner, then Bob again: an order that would show a + // result kept per list rather than per caller. + if got, want := get(bob), []string{"main"}; !reflect.DeepEqual(got, want) { + t.Errorf("bob sees %v, want %v", got, want) + } + if got, want := get(alice), []string{"alice-dev", "main"}; !reflect.DeepEqual(got, want) { + t.Errorf("alice sees %v, want %v", got, want) + } + if got, want := get(admin), []string{"alice-dev", "bobs-secret", "main"}; !reflect.DeepEqual(got, want) { + t.Errorf("the admin sees %v, want %v", got, want) + } + if got, want := get(bob), []string{"main"}; !reflect.DeepEqual(got, want) { + t.Errorf("bob sees %v afterwards, want %v", got, want) + } + if n := reads.Load(); n != 1 { + t.Errorf("%d reads of the branches for 4 requests, want 1", n) + } +} + +// And the same for what the import page polls. +func TestSharedReplicationListIsFilteredPerCaller(t *testing.T) { + s, _, alice, bob := withACL(t) + old := listReplicating + listReplicating = func() ([]branch.Replication, error) { + return []branch.Replication{{Branch: "alice-dev", Replicating: true}}, nil + } + t.Cleanup(func() { listReplicating = old }) + api := http.NewServeMux() + registerAPI(api) + h := s.Authn(api) + seen := func(u auth.User) int { + t.Helper() + key, _, _ := s.CreateAPIKey(u.ID, "t") + r := httptest.NewRequest("GET", "/api/replication", nil) + r.Header.Set("Authorization", "Bearer "+key) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("%s: %d %s", u.Email, w.Code, w.Body) + } + var out []branch.Replication + _ = json.Unmarshal(w.Body.Bytes(), &out) + return len(out) + } + if n := seen(bob); n != 0 { + t.Errorf("bob sees %d replications on a branch that is not his", n) + } + if n := seen(alice); n != 1 { + t.Errorf("alice sees %d, want 1", n) + } +} From 68e4f1d3692f1d58ca64d621d01e68ee55198e77 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 14:59:48 +0530 Subject: [PATCH 16/22] perf(web): poll after the last poll, every 10s, only when visible The dashboard and the import page polled on a fixed interval whether the last request had come back or anyone was looking. The next poll is now timed from the end of the last, a hidden tab does not poll, and a user's own action still refreshes at once; the newest refresh wins, so a poll already out cannot put the old list back. Co-Authored-By: Claude Opus 5.5 --- web/src/pages/Dashboard.tsx | 19 ++++++++------- web/src/pages/Import.tsx | 13 ++++------ web/src/usePolling.ts | 47 +++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 web/src/usePolling.ts diff --git a/web/src/pages/Dashboard.tsx b/web/src/pages/Dashboard.tsx index bdd19c77..699064eb 100644 --- a/web/src/pages/Dashboard.tsx +++ b/web/src/pages/Dashboard.tsx @@ -1,10 +1,11 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { getStatus, getBranches, createBranch, deleteBranch, suspendBranch, resumeBranch, listBackups, API, type Status, type Branch, type Backup, } from '../api' import { useConfirm } from '../confirm' import { useFeatures } from '../features' +import { usePolling } from '../usePolling' function Dot({ up }: { up: boolean }) { return @@ -131,19 +132,19 @@ export default function Dashboard() { const [err, setErr] = useState('') const [offline, setOffline] = useState(false) + // The newest refresh wins: a poll already on its way when the user suspends a + // branch can come back after the refresh that follows, carrying the old list. + const latest = useRef(0) const refresh = async () => { + const mine = ++latest.current try { const [s, b] = await Promise.all([getStatus(), getBranches()]) - setStatus(s); setBranches(b); setOffline(false) + if (mine === latest.current) { setStatus(s); setBranches(b); setOffline(false) } } catch { - setOffline(true) + if (mine === latest.current) setOffline(true) } } - useEffect(() => { - refresh() - const t = setInterval(refresh, 3000) - return () => clearInterval(t) - }, []) + usePolling(refresh, 10000) const act = async (fn: () => Promise) => { setErr('') @@ -182,7 +183,7 @@ export default function Dashboard() { return (

Dashboard

-

Live control plane · auto-refreshing every 3s.

+

Live control plane · auto-refreshing every 10s.

{tiles.map(([k, v], i) => ( diff --git a/web/src/pages/Import.tsx b/web/src/pages/Import.tsx index ecf33109..7b65536e 100644 --- a/web/src/pages/Import.tsx +++ b/web/src/pages/Import.tsx @@ -5,6 +5,7 @@ import { type ImportEvent, type ImportResult, type Replication, } from '../api' import { useConfirm } from '../confirm' +import { usePolling } from '../usePolling' type Line = { text: string; kind?: 'err' | 'ok' } type Prog = { done: number; total: number; label: string } @@ -186,14 +187,10 @@ function ContinuousImports() { const [done, setDone] = useState('') const [busy, setBusy] = useState('') - useEffect(() => { - let live = true - const load = () => listReplication().then(l => { if (live) { setList(l); setErr('') } }) - .catch(e => { if (live) setErr((e as Error).message) }) - load() - const t = setInterval(load, 4000) - return () => { live = false; clearInterval(t) } - }, []) + const live = useRef(true) + useEffect(() => { live.current = true; return () => { live.current = false } }, []) + usePolling(() => listReplication().then(l => { if (live.current) { setList(l); setErr('') } }) + .catch(e => { if (live.current) setErr((e as Error).message) }), 10000) const cutover = async (r: Replication) => { const ok = await confirm({ diff --git a/web/src/usePolling.ts b/web/src/usePolling.ts new file mode 100644 index 00000000..13f51175 --- /dev/null +++ b/web/src/usePolling.ts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import { useEffect, useRef } from 'react' + +// Run fn now and then every ms, for a page showing something that changes. +// +// setInterval does not know whether the last request came back: with a slow +// server the requests stack up, which makes the server slower, and every open +// tab did this whether anyone was looking at it or not. So the next poll is +// timed from the end of the last one, and a hidden tab does not poll at all and +// catches up the moment it is shown again. +// +// fn reports its own errors; one that throws anyway does not stop the polling. +export function usePolling(fn: () => unknown, ms: number) { + const latest = useRef(fn) + latest.current = fn + + useEffect(() => { + let stopped = false + let running = false + let timer: ReturnType | undefined + + const tick = async () => { + if (stopped || running || document.hidden) return + running = true + try { + await latest.current() + } catch { + // fn's to report + } + running = false + if (!stopped && !document.hidden) timer = setTimeout(tick, ms) + } + const onVisibility = () => { + if (document.hidden) return + clearTimeout(timer) + tick() + } + + document.addEventListener('visibilitychange', onVisibility) + tick() + return () => { + stopped = true + clearTimeout(timer) + document.removeEventListener('visibilitychange', onVisibility) + } + }, [ms]) +} From 8eba7bc5c09703031559604f8ec545c26e72015f Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 15:01:31 +0530 Subject: [PATCH 17/22] fix(security): put timeouts on the connection that records a refusal A blocked statement is recorded through a second connection, and a wait inside it is one Postgres cannot see as a deadlock: another transaction holding the Blackbox's append lock hung the refused client indefinitely. The guardrail, the data guards and the policy gate now go through a connection with lock_timeout=2s and statement_timeout=5s, and a failure to record is a WARNING: the refusal still happens. Client logins get lock_timeout=60s and idle_in_transaction_session_timeout =10min by default, so an abandoned transaction cannot stop the record, set once per role so an administrator's values stay. Co-Authored-By: Claude Opus 5.5 --- enterprise/schema/policy.sql | 2 +- enterprise/schema/schema_test.go | 5 +++ internal/ledger/blockedconn_test.go | 60 +++++++++++++++++++++++++++++ internal/ledger/datachanges.sql | 12 ++---- internal/ledger/ledger.sql | 58 ++++++++++++++++++++++++++-- 5 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 internal/ledger/blockedconn_test.go diff --git a/enterprise/schema/policy.sql b/enterprise/schema/policy.sql index 457e63f8..9759f44c 100644 --- a/enterprise/schema/policy.sql +++ b/enterprise/schema/policy.sql @@ -228,7 +228,7 @@ BEGIN -- the rollback the error below causes (the guardrail's mechanism). eval_id := NULL; BEGIN - conn := 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user; + conn := bb._blocked_conn(); -- Not when this transaction already wrote to Blackbox: the second connection -- would wait for this one forever (bb._holds_chain_lock). The evaluation row -- below takes no such lock and is still recorded. diff --git a/enterprise/schema/schema_test.go b/enterprise/schema/schema_test.go index 1c42c4be..776d6634 100644 --- a/enterprise/schema/schema_test.go +++ b/enterprise/schema/schema_test.go @@ -97,6 +97,11 @@ func TestTheGateChecksTheChainLockBeforeWriting(t *testing.T) { if blocked := strings.Index(schemaPolicy[lock:], "'BLOCKED','policy'"); blocked < 0 { t.Error("policy.sql (gate): no BLOCKED write follows the lock check") } + // And through the connection with timeouts, so no wait there can hang the + // statement being refused. + if !strings.Contains(schemaPolicy, "conn := bb._blocked_conn();") { + t.Error("policy.sql (gate): the blocked attempt must use bb._blocked_conn") + } } // Rules match the statement that runs, not everything a client sent with it. diff --git a/internal/ledger/blockedconn_test.go b/internal/ledger/blockedconn_test.go new file mode 100644 index 00000000..24c4a413 --- /dev/null +++ b/internal/ledger/blockedconn_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "strings" + "testing" +) + +// A blocked attempt is recorded through a second connection, and a wait there +// is invisible to Postgres' deadlock detector: another transaction holding the +// append lock used to hang the refused client. Every path goes through the one +// connection that carries timeouts, and failing to record stays a WARNING. +func TestBlockedRecordConnectionCarriesTimeouts(t *testing.T) { + if !strings.Contains(Schema, `options='-c lock_timeout=2s -c statement_timeout=5s'`) { + t.Error("the blocked-attempt connection lost its timeouts") + } + if !strings.Contains(Schema, "'host=/var/run/postgresql dbname='") { + t.Error("the blocked-attempt connection no longer uses the Unix socket") + } + i := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb._record_blocked(stmt text)") + if i < 0 { + t.Fatal("bb._record_blocked is gone") + } + body := Schema[i:] + body = body[:strings.Index(body, "$$;")] + if !strings.Contains(body, "EXCEPTION WHEN OTHERS THEN\n RAISE WARNING") { + t.Error("a failure to record a blocked attempt must be a WARNING, so the refusal still happens") + } + if strings.Contains(body, "SECURITY DEFINER") { + t.Error("bb._record_blocked must not run as its owner: a client could write entries through it") + } + + // Nothing talks to dblink except the helpers. + for what, sql := range map[string]string{"ledger.sql": Schema, "datachanges.sql": SchemaData} { + if n := strings.Count(sql, "PERFORM dblink_exec("); (what == "ledger.sql" && n != 1) || (what != "ledger.sql" && n != 0) { + t.Errorf("%s has %d direct dblink_exec call(s); the blocked path belongs in bb._record_blocked", what, n) + } + } + if n := strings.Count(SchemaData, "PERFORM bb._record_blocked(format("); n != 3 { + t.Errorf("datachanges.sql records %d blocked paths through the helper, want 3 (TRUNCATE, unqualified DML, impact)", n) + } +} + +// The defaults for client logins, and that an administrator's choice survives +// the re-run on every start. +func TestClientRolesGetTimeoutDefaults(t *testing.T) { + for _, want := range []string{ + "ARRAY['lock_timeout=60s', 'idle_in_transaction_session_timeout=10min']", + "WHERE d.setrole = r.oid AND d.setdatabase = 0", + "pg_has_role(oid, 'db_client', 'MEMBER')", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } + if strings.Contains(Schema, "statement_timeout=0") { + t.Error("client logins should not have their statement timeout touched") + } +} diff --git a/internal/ledger/datachanges.sql b/internal/ledger/datachanges.sql index 8b6e6985..f3609543 100644 --- a/internal/ledger/datachanges.sql +++ b/internal/ledger/datachanges.sql @@ -66,9 +66,7 @@ BEGIN IF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) VALUES (%L,%L,%L,%L,%L,'TRUNCATE','table',%L,%L,'BLOCKED','policy')$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, obj, current_query())); @@ -125,9 +123,7 @@ BEGIN IF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) VALUES (%L,%L,%L,%L,%L,%L,'table',%L,%L,'BLOCKED','unqualified')$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, TG_OP, obj, q)); @@ -308,9 +304,7 @@ BEGIN ELSIF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) VALUES (%L,%L,%L,%L,%L,%L,'table',%L,%L,'BLOCKED',%L)$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, op, obj, current_query(), risk)); diff --git a/internal/ledger/ledger.sql b/internal/ledger/ledger.sql index 3ec4bec1..dbcd5727 100644 --- a/internal/ledger/ledger.sql +++ b/internal/ledger/ledger.sql @@ -568,6 +568,33 @@ LANGUAGE sql STABLE AS $$ AND ((classid::bigint << 32) | objid::bigint) = hashtext('bb.schema_ledger')::bigint); $$; +-- The second connection a blocked attempt is recorded through. A wait inside it +-- is one Postgres cannot see as part of any deadlock (the client's backend is +-- waiting on a socket, not a lock), so nothing would ever break one: another +-- transaction sitting on the Blackbox's append lock used to hang the refused +-- client for as long as it stayed open. The timeouts are for that connection +-- alone, and short: a refusal should come back at once, and it comes back +-- whether or not the record could be written. +CREATE OR REPLACE FUNCTION bb._blocked_conn() RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user + || $o$ options='-c lock_timeout=2s -c statement_timeout=5s'$o$; +$$; + +-- Write a blocked attempt through that connection. Failing to record it is a +-- WARNING and never the reason the statement stops: every caller raises its +-- own refusal next, and a refusal must not turn into a hang or another error. +-- Not SECURITY DEFINER, and a client can't reach dblink: it is only of use from +-- the triggers, which run as the owner. +CREATE OR REPLACE FUNCTION bb._record_blocked(stmt text) RETURNS void +LANGUAGE plpgsql SET search_path = pg_catalog, public, bb AS $$ +BEGIN + PERFORM dblink_exec(bb._blocked_conn(), stmt); +EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox: %', SQLERRM; +END; +$$; + -- ddl_command_start: enforce guardrails BEFORE the command runs. -- SECURITY DEFINER: the ledger's triggers record history with the owner's -- privileges, so a non-superuser client can trigger them (by running DDL) without @@ -590,9 +617,7 @@ BEGIN IF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,statement,status,risk) VALUES (%L,%L,%L,%L,%L,%L,%L,'BLOCKED','policy')$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, TG_TAG, current_query())); @@ -793,6 +818,33 @@ BEGIN END $$; ALTER ROLE db_agent SET role = db_client; +-- Defaults for client logins, for what the append lock does to everyone else. A +-- transaction that has recorded something holds that lock until it ends, and +-- every other session's DDL (and every agent write) queues behind it, so one +-- left open in a client stopped the database's history without an error. +-- idle_in_transaction_session_timeout = 10min the value the hardening check +-- recommends (conn.idle); it ends only a session that sat idle mid-transaction +-- lock_timeout = 60s a queued client gets an error, not an endless wait +-- No statement_timeout: long queries and migrations are legitimate work. These +-- are defaults a session can SET for itself. Set once for each role, so a value +-- an administrator chose stays; a role made since the last start has it from the +-- next one. The engine's own superuser logins are left without them. +DO $$ +DECLARE r record; s text; +BEGIN + FOR r IN SELECT oid, rolname FROM pg_roles + WHERE rolcanlogin AND NOT rolsuper + AND (rolname IN ('db_client', 'db_agent') OR pg_has_role(oid, 'db_client', 'MEMBER')) LOOP + FOREACH s IN ARRAY ARRAY['lock_timeout=60s', 'idle_in_transaction_session_timeout=10min'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_db_role_setting d, unnest(d.setconfig) c + WHERE d.setrole = r.oid AND d.setdatabase = 0 + AND split_part(c, '=', 1) = split_part(s, '=', 1)) THEN + EXECUTE format('ALTER ROLE %I SET %I = %L', r.rolname, split_part(s, '=', 1), split_part(s, '=', 2)); + END IF; + END LOOP; + END LOOP; +END $$; + -- Full access to application data (it owns what it creates; these cover what the -- superuser created, e.g. imported tables, now and in future). GRANT USAGE, CREATE ON SCHEMA public TO db_client; From d2eaa03e42ebb6b7a957b05aa72f124f943b3678 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 15:01:51 +0530 Subject: [PATCH 18/22] fix(perf): read the audit log and its evidence a day at a time ReadAudit prepended each day to everything read so far and re-joined the lot, so a rare match walking many days did quadratic work and held every record. It now chooses the records to show in one pass, holding at most Limit, and reads those days again only for the keys of the statements chosen. The answers are the same as before, including a transaction filed days away from its statement in either direction, which is why this is a second pass and not a fixed window. AuditRecordsBetween is streamed, and evidence counts its records without holding them unless the records were asked for. It stops after the period's end plus a week, and goes on while a day still holds records of the period; the one case that can miss is written into the evidence's limits. Co-Authored-By: Claude Opus 5.5 --- enterprise/auditevidence/evidence.go | 20 +- enterprise/auditevidence/evidence_test.go | 6 +- internal/branch/auditevidence.go | 93 +++++- internal/branch/auditread_bounds_test.go | 387 ++++++++++++++++++++++ internal/branch/auditstore.go | 176 +++++++--- 5 files changed, 612 insertions(+), 70 deletions(-) create mode 100644 internal/branch/auditread_bounds_test.go diff --git a/enterprise/auditevidence/evidence.go b/enterprise/auditevidence/evidence.go index aa6145ae..204f8c3e 100644 --- a/enterprise/auditevidence/evidence.go +++ b/enterprise/auditevidence/evidence.go @@ -34,7 +34,8 @@ type inputs struct { keyID string forward *branch.ForwardStatus findings []branch.AuditFinding - records []branch.AuditRecord + activity branch.EvidenceActivity + records []branch.AuditRecord // only when they were asked for anchorSet []ledger.Anchor publicKey string } @@ -85,7 +86,7 @@ func gather(o branch.EvidenceOptions) (inputs, error) { } } in.findings = branch.AuditFindings(marks) - if in.records, err = branch.AuditRecordsBetween(o.Since, o.Until); err != nil { + if in.activity, in.records, err = branch.AuditActivityBetween(o.Since, o.Until, o.Records); err != nil { return in, err } if o.Records { @@ -108,7 +109,7 @@ func assemble(o branch.EvidenceOptions, in inputs) branch.AuditEvidence { }, Integrity: branch.EvidenceIntegrity{ChainReport: in.chain, AnchorsKept: in.anchors, KeyID: in.keyID}, Delivery: branch.EvidenceDelivery{Forward: in.forward, Findings: in.findings}, - Activity: branch.EvidenceActivity{ByKind: map[string]int{}, ByBranch: map[string]int{}, Gaps: []branch.AuditRecord{}}, + Activity: in.activity, Changes: []auth.SecurityEvent{}, Limits: branch.EvidenceLimits, } @@ -121,19 +122,6 @@ func assemble(o branch.EvidenceOptions, in inputs) branch.AuditEvidence { if ev.Delivery.Findings == nil { ev.Delivery.Findings = []branch.AuditFinding{} } - for _, r := range in.records { - ev.Activity.Records++ - ev.Activity.ByKind[r.Kind]++ - ev.Activity.ByBranch[r.Branch]++ - switch r.Kind { - case "outcome": - ev.Activity.Errors++ - case "heartbeat": - ev.Activity.Heartbeats++ - case "gap": - ev.Activity.Gaps = append(ev.Activity.Gaps, r) - } - } for _, e := range o.Events { if !strings.HasPrefix(e.Kind, "audit-log.") { continue diff --git a/enterprise/auditevidence/evidence_test.go b/enterprise/auditevidence/evidence_test.go index e0a85110..68e380fc 100644 --- a/enterprise/auditevidence/evidence_test.go +++ b/enterprise/auditevidence/evidence_test.go @@ -23,7 +23,11 @@ func TestAssemble(t *testing.T) { {Seq: 1, Kind: "statement", Branch: "main"}, {Seq: 2, Kind: "outcome", Branch: "main"}, {Seq: 3, Kind: "heartbeat", Branch: "dev"}, {Seq: 4, Kind: "gap", Branch: "dev", Missing: 2}, }} - o := branch.EvidenceOptions{Since: since, Until: until, By: "ada@example.com", Events: []auth.SecurityEvent{ + in.activity = branch.NewAuditActivity() + for _, r := range in.records { + branch.TallyAuditActivity(&in.activity, r) + } + o :=branch.EvidenceOptions{Since: since, Until: until, By: "ada@example.com", Events: []auth.SecurityEvent{ {Kind: auth.EvAuditLogOn, At: "2026-11-02T10:00:00Z", Actor: "ada@example.com", IP: "10.0.0.9"}, {Kind: auth.EvAuditMarked, At: "2026-10-30T10:00:00Z", Subject: "main", Detail: "auditing writes of public.t"}, // before the period {Kind: "user.login", At: "2026-11-02T11:00:00Z"}, // not the audit log's diff --git a/internal/branch/auditevidence.go b/internal/branch/auditevidence.go index e1764b45..577c10f6 100644 --- a/internal/branch/auditevidence.go +++ b/internal/branch/auditevidence.go @@ -143,6 +143,7 @@ var ( "SECURITY DEFINER functions are found by a word match on their source: it can name too many, and misses a table name built at run time.", "A record says who and what, and never what a statement held; in forensic mode its text is kept apart from the chain and expires.", "The anchors prove the records against an edit made after they were anchored; records collected since the last anchor rest on the chain alone.", + "A record of the period collected more than a week after it ended, and after a whole later day of newer records had been collected, is not counted here. The chain was still verified over it.", } ) @@ -182,32 +183,104 @@ func EvidenceSummary(ev AuditEvidence) string { return s } +// auditCollectedLateDays is how many days past a period's end its records are +// still looked for, however little those days hold of it. A day file holds what +// was collected that day, and that can be days after a record happened. +const auditCollectedLateDays = 7 + // AuditRecordsBetween reads the records collected in [since, until), in // order, without statement text. func AuditRecordsBetween(since, until time.Time) ([]AuditRecord, error) { var out []AuditRecord + err := ScanAuditRecordsBetween(since, until, func(r AuditRecord) { out = append(out, r) }) + if err != nil { + return nil, err + } + return out, nil +} + +// ScanAuditRecordsBetween hands the records in [since, until) to fn in order, +// without statement text, holding one day at a time at most and none of it +// once fn has been called. +// +// It stops once the period's end is well behind: past auditCollectedLateDays, +// at the first day file with nothing from before until in it. A week is a wide +// margin for a collector that was stopped, and the rule extends itself past +// that when a collector has been down for longer, because the first day it +// runs again still holds records of the period. What it cannot find is a +// record that arrives after a whole later day of newer ones; EvidenceLimits +// says so. +func ScanAuditRecordsBetween(since, until time.Time, fn func(AuditRecord)) error { + first := since.UTC().AddDate(0, 0, -1).Format("2006-01-02") + carry := until.UTC().AddDate(0, 0, auditCollectedLateDays).Format("2006-01-02") for _, day := range storedDays() { // A day file holds what was collected that day, so one before the // period's first day holds nothing in it. - if day < since.UTC().AddDate(0, 0, -1).Format("2006-01-02") { + if day < first { continue } - recs, err := readDay(day) - if err != nil { - return nil, fmt.Errorf("the records of %s cannot all be read (%v)", day, err) - } - for _, r := range recs { + before := false + err := scanDay(day, func(r AuditRecord) { // Parsed, not compared as text: RFC 3339 drops trailing zeros, // so "…01.5Z" sorts after "…01.55Z". at, err := time.Parse(time.RFC3339Nano, r.Time) - if err != nil || at.Before(since) || !at.Before(until) { - continue + if err != nil { + return + } + if at.Before(until) { + before = true + if !at.Before(since) { + r.Text = "" + fn(r) + } } - r.Text = "" + }) + if err != nil { + return fmt.Errorf("the records of %s cannot all be read (%v)", day, err) + } + if day > carry && !before { + break + } + } + return nil +} + +// TallyAuditActivity adds one record to a period's counts. +func TallyAuditActivity(a *EvidenceActivity, r AuditRecord) { + a.Records++ + a.ByKind[r.Kind]++ + a.ByBranch[r.Branch]++ + switch r.Kind { + case "outcome": + a.Errors++ + case "heartbeat": + a.Heartbeats++ + case "gap": + a.Gaps = append(a.Gaps, r) + } +} + +// NewAuditActivity is an empty EvidenceActivity, every part present. +func NewAuditActivity() EvidenceActivity { + return EvidenceActivity{ByKind: map[string]int{}, ByBranch: map[string]int{}, Gaps: []AuditRecord{}} +} + +// AuditActivityBetween counts the period's records in one streaming pass, and +// returns the records themselves only when asked for them: a period's counts +// cost what the counts hold, not what the records do. +func AuditActivityBetween(since, until time.Time, records bool) (EvidenceActivity, []AuditRecord, error) { + a := NewAuditActivity() + var out []AuditRecord + err := ScanAuditRecordsBetween(since, until, func(r AuditRecord) { + TallyAuditActivity(&a, r) + if records { out = append(out, r) } + }) + if err != nil { + return EvidenceActivity{}, nil, err } - return out, nil + return a, out, nil } // AuditBranchCoverage reads a running branch's marks, the definer functions diff --git a/internal/branch/auditread_bounds_test.go b/internal/branch/auditread_bounds_test.go new file mode 100644 index 00000000..3d0ffe62 --- /dev/null +++ b/internal/branch/auditread_bounds_test.go @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "fmt" + "reflect" + "testing" + "time" +) + +// oldReadAudit is ReadAudit before it stopped re-joining everything per day, +// kept to prove the new one returns the same answers. +func oldReadAudit(q AuditQuery) ([]AuditEntry, error) { + if q.Limit <= 0 { + q.Limit = 100 + } + days := storedDays() + var all []AuditRecord + var out []AuditEntry + for i := len(days) - 1; i >= 0 && len(out) < q.Limit; i-- { + if !q.Since.IsZero() && days[i] < q.Since.UTC().Format("2006-01-02") { + break + } + recs, err := readDay(days[i]) + if err != nil { + return nil, err + } + all = append(recs, all...) + out = joinAudit(all, q) + } + if len(out) > q.Limit { + out = out[:q.Limit] + } + attachTexts(out) + return out, nil +} + +// oldAuditRecordsBetween is AuditRecordsBetween before it stopped at until. +func oldAuditRecordsBetween(since, until time.Time) ([]AuditRecord, error) { + var out []AuditRecord + for _, day := range storedDays() { + if day < since.UTC().AddDate(0, 0, -1).Format("2006-01-02") { + continue + } + recs, err := readDay(day) + if err != nil { + return nil, fmt.Errorf("the records of %s cannot all be read (%v)", day, err) + } + for _, r := range recs { + at, err := time.Parse(time.RFC3339Nano, r.Time) + if err != nil || at.Before(since) || !at.Before(until) { + continue + } + r.Text = "" + out = append(out, r) + } + } + return out, nil +} + +func stmtLine(at, session, vxid, user, table string) string { + return fmt.Sprintf(`{"timestamp":"%s UTC","user":"%s","dbname":"appdb","pid":7,"session_id":"%s","line_num":1,"ps":"INSERT","vxid":"%s","error_severity":"LOG","message":"AUDIT: OBJECT,1,1,WRITE,INSERT,TABLE,%s,,"}`, + at, user, session, vxid, table) +} + +func txnLine(at, session, vxid, outcome string) string { + return fmt.Sprintf(`{"timestamp":"%s UTC","user":"alice@x.in","session_id":"%s","vxid":"4/0","error_severity":"LOG","message":"AUDIT: TXN,%s,%s,asserted,shop,\"o'brien, p\""}`, + at, session, vxid, outcome) +} + +func errorLine(at, session, vxid, state string) string { + return fmt.Sprintf(`{"timestamp":"%s UTC","user":"alice@x.in","session_id":"%s","vxid":"%s","error_severity":"ERROR","state_code":"%s","message":"AUDIT: OUTCOME,ERROR,%s"}`, + at, session, vxid, state, state) +} + +// seedAuditFixture is a store with what makes the join and the day boundaries +// interesting: transactions spanning days, a statement filed after the +// transaction that closed it, one key finished twice, errors, and several +// logins and tables. +func seedAuditFixture(t *testing.T) { + t.Helper() + storeIn(t) + settingsIn(t) + // A long transaction: the statement on the 1st, its commit on the 4th. + seed(t, "2026-11-01", + stmtLine("2026-11-01 09:00:00.000", "s1", "1/1", "alice@x.in", "public.orders"), + stmtLine("2026-11-01 09:00:01.000", "s2", "2/1", "bob", "public.users"), + txnLine("2026-11-01 09:00:02.000", "s2", "2/1", "ROLLBACK"), + errorLine("2026-11-01 09:00:03.000", "s2", "2/2", "23505"), + ) + // A transaction filed a day before the statement it closes, as a late raw + // file does; and the same key finished twice, where the last one wins. + seed(t, "2026-11-02", + txnLine("2026-11-02 23:59:58.000", "s3", "3/1", "COMMIT"), + stmtLine("2026-11-02 12:00:00.000", "s4", "4/1", "alice@x.in", "public.orders"), + txnLine("2026-11-02 12:00:01.000", "s4", "4/1", "ROLLBACK"), + stmtLine("2026-11-02 12:00:02.000", "s5", "5/1", "bob", "public.users"), + ) + seed(t, "2026-11-03", + stmtLine("2026-11-02 23:59:59.000", "s3", "3/1", "alice@x.in", "public.orders"), // filed after its transaction + stmtLine("2026-11-03 10:00:00.000", "s5", "5/2", "alice@x.in", "public.users"), + errorLine("2026-11-03 10:00:01.000", "s5", "5/2", "42501"), + txnLine("2026-11-03 10:00:02.000", "s4", "4/1", "COMMIT"), // the same key again + ) + seed(t, "2026-11-04", + txnLine("2026-11-04 08:00:00.000", "s1", "1/1", "COMMIT"), // closes the 1st's statement + stmtLine("2026-11-04 08:00:01.000", "s6", "6/1", "alice@x.in", "public.orders"), + txnLine("2026-11-04 08:00:02.000", "s6", "6/1", "COMMIT"), + errorLine("2026-11-04 08:00:03.000", "s7", "7/1", "22P02"), + stmtLine("2026-11-04 08:00:04.000", "s7", "7/2", "bob", "public.users"), + ) +} + +// The new read against the old one, over filters and limits falling on and +// around every day boundary. +func TestReadAuditMatchesTheOldRead(t *testing.T) { + seedAuditFixture(t) + since := time.Date(2026, 11, 2, 12, 0, 1, 0, time.UTC) + queries := []AuditQuery{ + {}, {Limit: 1}, {Limit: 2}, {Limit: 3}, {Limit: 4}, {Limit: 5}, {Limit: 6}, {Limit: 7}, {Limit: 8}, + {Limit: 9}, {Limit: 10}, {Limit: 1000}, + {Login: "alice@x.in"}, {Login: "alice@x.in", Limit: 2}, {Login: "alice@x.in", Limit: 3}, + {Login: "nobody"}, + {Table: "orders"}, {Table: "orders", Limit: 1}, {Table: "public.users", Limit: 2}, + {Branch: "main", Limit: 3}, {Branch: "other"}, + {Since: since}, {Since: since, Limit: 2}, {Since: since, Table: "users"}, + {Kinds: []string{"outcome"}}, {Kinds: []string{"outcome"}, Limit: 1}, + {Kinds: []string{"transaction"}, Limit: 3}, {Kinds: []string{"statement", "outcome"}, Limit: 5}, + } + for _, q := range queries { + old, oldErr := oldReadAudit(q) + got, err := ReadAudit(q) + if err != nil || oldErr != nil { + t.Fatalf("%+v: %v / %v", q, err, oldErr) + } + if !reflect.DeepEqual(got, old) { + t.Errorf("%+v:\n got %+v\nwant %+v", q, got, old) + } + } + // The cases the fixture exists for, stated, so they cannot pass vacuously. + all, err := ReadAudit(AuditQuery{Limit: 1000}) + if err != nil { + t.Fatal(err) + } + find := func(session, vxid string) AuditEntry { + for _, e := range all { + if e.Session == session && e.VXID == vxid && e.Kind == "statement" { + return e + } + } + t.Fatalf("no statement %s %s", session, vxid) + return AuditEntry{} + } + if e := find("s1", "1/1"); e.TxnOutcome != "COMMIT" { + t.Errorf("a transaction closed three days later: %+v", e) + } + if e := find("s3", "3/1"); e.TxnOutcome != "COMMIT" { + t.Errorf("a transaction filed before its statement: %+v", e) + } + if e := find("s4", "4/1"); e.TxnOutcome != "COMMIT" { + t.Errorf("the last record of a key did not win: %+v", e) + } + if e := find("s5", "5/2"); e.Error != "42501" { + t.Errorf("an error on a statement: %+v", e) + } +} + +// The records and the counts against the old read, including a period whose +// days are not all before its end. +func TestEvidenceReadMatchesTheOldRead(t *testing.T) { + seedAuditFixture(t) + periods := [][2]time.Time{ + {time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 12, 1, 0, 0, 0, 0, time.UTC)}, + {time.Date(2026, 11, 2, 12, 0, 1, 0, time.UTC), time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}, + {time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC), time.Date(2026, 11, 3, 10, 0, 1, 0, time.UTC)}, + {time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 11, 1, 9, 0, 2, 0, time.UTC)}, + {time.Date(2027, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2027, 1, 2, 0, 0, 0, 0, time.UTC)}, + } + for _, p := range periods { + old, err := oldAuditRecordsBetween(p[0], p[1]) + if err != nil { + t.Fatal(err) + } + got, err := AuditRecordsBetween(p[0], p[1]) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, old) { + t.Errorf("%v: records differ:\n got %+v\nwant %+v", p, got, old) + } + want := NewAuditActivity() + for _, r := range old { + TallyAuditActivity(&want, r) + } + a, none, err := AuditActivityBetween(p[0], p[1], false) + if err != nil || none != nil { + t.Fatalf("%v: records came back unasked: %d, %v", p, len(none), err) + } + if !reflect.DeepEqual(a, want) { + t.Errorf("%v: counts differ:\n got %+v\nwant %+v", p, a, want) + } + _, kept, err := AuditActivityBetween(p[0], p[1], true) + if err != nil || len(kept) != len(old) { + t.Fatalf("%v: asked for the records: %d of %d, %v", p, len(kept), len(old), err) + } + } +} + +// countScans wraps scanDay, counting the days opened and the records handed +// over, which is the work a read does. +func countScans(t *testing.T) (days, records *int) { + t.Helper() + old := scanDay + days, records = new(int), new(int) + scanDay = func(day string, fn func(AuditRecord)) error { + *days++ + return old(day, func(r AuditRecord) { *records++; fn(r) }) + } + t.Cleanup(func() { scanDay = old }) + return days, records +} + +// seedDays is a store of n days, each of perDay statements, with one +// statement a filter can find on the very first day only. +func seedDays(t *testing.T, n, perDay int) { + t.Helper() + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for d := 0; d < n; d++ { + day := start.AddDate(0, 0, d) + lines := make([]string, 0, perDay) + for i := 0; i < perDay; i++ { + at := day.Add(time.Duration(i) * time.Second).Format("2006-01-02 15:04:05.000") + table := "public.common" + if d == 0 && i == 0 { + table = "public.rare" + } + lines = append(lines, stmtLine(at, "s", fmt.Sprintf("%d/%d", d, i), "alice@x.in", table)) + } + seed(t, day.Format("2006-01-02"), lines...) + } +} + +// Quadratic growth shows up as records handled per record stored rising with +// the number of days. A rare match walks every day, the worst case, and costs +// two passes over them at most; the old read handled each day once per day +// that followed it, and the fixture asserts that really was growth. +func TestReadAuditWorkIsLinearInTheDaysRead(t *testing.T) { + storeIn(t) + settingsIn(t) + const n, perDay = 40, 10 + seedDays(t, n, perDay) + days, records := countScans(t) + got, err := ReadAudit(AuditQuery{Table: "rare", Limit: 5}) + if err != nil || len(got) != 1 { + t.Fatalf("%d, %v", len(got), err) + } + if want := 2 * n * perDay; *records > want { + t.Errorf("handled %d records of %d stored, want at most %d", *records, n*perDay, want) + } + if *days > 2*n { + t.Errorf("opened %d day files of %d", *days, n) + } + // What the old read handled, by construction: every day re-joined once per + // day walked after it. + if old := perDay * n * (n + 1) / 2; old <= *records { + t.Errorf("the fixture does not separate the two: old %d, new %d", old, *records) + } + + // Most of the time nothing goes back further than it must. + *days, *records = 0, 0 + if _, err := ReadAudit(AuditQuery{Limit: 3}); err != nil { + t.Fatal(err) + } + if *days > 2 { + t.Errorf("a limit met by the newest day opened %d day files", *days) + } +} + +// Held memory is bounded by Limit, even for matches that fill a whole day. +func TestReadAuditHoldsAtMostLimit(t *testing.T) { + storeIn(t) + settingsIn(t) + seedDays(t, 3, 500) + got, err := ReadAudit(AuditQuery{Limit: 7}) + if err != nil { + t.Fatal(err) + } + if len(got) != 7 || got[0].VXID != "2/499" || got[6].VXID != "2/493" { + t.Fatalf("the newest seven: %d, %+v", len(got), got) + } +} + +// Stops at until, plus the days a record can be collected late. Days after +// that are not opened at all, but a period's record collected late is found +// however late, as long as the day it arrives in still holds such a record. +func TestEvidenceReadStopsAfterThePeriod(t *testing.T) { + storeIn(t) + settingsIn(t) + seedDays(t, 40, 3) + since := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + until := time.Date(2026, 1, 4, 0, 0, 0, 0, time.UTC) + days, _ := countScans(t) + recs, err := AuditRecordsBetween(since, until) + if err != nil { + t.Fatal(err) + } + if len(recs) != 6 { + t.Fatalf("records: %d", len(recs)) + } + // from the day before since, through until and its carry-over, and the one + // day that shows the period is behind us. + if want := 2 + 2 + auditCollectedLateDays + 1; *days > want { + t.Errorf("opened %d day files, want at most %d", *days, want) + } +} + +func TestEvidenceReadFindsRecordsCollectedLate(t *testing.T) { + storeIn(t) + settingsIn(t) + // Nothing was collected for three weeks after the period's end, and then + // the backlog arrived. + seed(t, "2026-03-01", stmtLine("2026-03-01 10:00:00.000", "s", "1/1", "alice@x.in", "public.t")) + seed(t, "2026-03-22", stmtLine("2026-03-01 23:59:59.000", "s", "1/2", "alice@x.in", "public.t"), + stmtLine("2026-03-21 12:00:00.000", "s", "1/3", "alice@x.in", "public.t")) + since := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + until := time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC) + got, err := AuditRecordsBetween(since, until) + if err != nil { + t.Fatal(err) + } + old, _ := oldAuditRecordsBetween(since, until) + if len(got) != 2 || !reflect.DeepEqual(got, old) { + t.Fatalf("late records:\n got %+v\nwant %+v", got, old) + } +} + +// BenchmarkReadAudit compares the old read with the new on a rare match that +// walks every day: the old grows with the square of the days, the new with +// the days. +func BenchmarkReadAudit(b *testing.B) { + for _, n := range []int{10, 40, 160} { + dir := b.TempDir() + old := AuditStoreDir + AuditStoreDir = func() string { return dir } + defer func() { AuditStoreDir = old }() + seedBenchDays(b, n, 200) + q := AuditQuery{Table: "rare", Limit: 5} + b.Run(fmt.Sprintf("new/days=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := ReadAudit(q); err != nil { + b.Fatal(err) + } + } + }) + b.Run(fmt.Sprintf("old/days=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := oldReadAudit(q); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func seedBenchDays(b *testing.B, n, perDay int) { + b.Helper() + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for d := 0; d < n; d++ { + day := start.AddDate(0, 0, d) + recs := make([]AuditRecord, 0, perDay) + for i := 0; i < perDay; i++ { + r, ok := recordFromLine("main", []byte(stmtLine(day.Add(time.Duration(i)*time.Second).Format("2006-01-02 15:04:05.000"), + "s", fmt.Sprintf("%d/%d", d, i), "alice@x.in", "public.common"))) + if !ok { + b.Fatal("not a record") + } + if d == 0 && i == 0 { + r.Object = "public.rare" + } + recs = append(recs, r) + } + if _, err := appendDay(day.Format("2006-01-02"), recs); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/branch/auditstore.go b/internal/branch/auditstore.go index 99bcdabf..fee0a7ec 100644 --- a/internal/branch/auditstore.go +++ b/internal/branch/auditstore.go @@ -207,31 +207,40 @@ func appendDay(day string, recs []AuditRecord) (int64, error) { return st.Size(), nil } -// readDay reads a day's records. +// readDay reads a day's records. On an error it returns those it got to, which +// is how verify says how far a damaged file still reads. func readDay(day string) ([]AuditRecord, error) { + var out []AuditRecord + err := scanDay(day, func(r AuditRecord) { out = append(out, r) }) + return out, err +} + +// scanDay hands a day's records to fn in file order, one at a time, so a reader +// that does not need them all does not hold them. A variable so a test can +// count what a read costs. +var scanDay = func(day string, fn func(AuditRecord)) error { f, err := os.Open(dayPath(day)) if err != nil { - return nil, err + return err } defer f.Close() zr, err := gzip.NewReader(f) if err != nil { if err == io.EOF { - return nil, nil + return nil } - return nil, err + return err } defer zr.Close() - var out []AuditRecord dec := json.NewDecoder(zr) for { var r AuditRecord if err := dec.Decode(&r); err == io.EOF { - return out, nil + return nil } else if err != nil { - return out, fmt.Errorf("%s: %w", day, err) + return fmt.Errorf("%s: %w", day, err) } - out = append(out, r) + fn(r) } } @@ -524,26 +533,76 @@ type AuditEntry struct { // ReadAudit reads the newest records that match, newest first: statements // joined to their transaction's outcome and whom it was for, and the // outcome records of errors outside any audited statement. +// +// Two passes over the days, each a day at a time, so the work grows with the +// records read and what is held with Limit, however far back a rare match +// makes it look. The first walks back from the newest day choosing the +// records to show and stops once it has Limit of them. The second reads those +// same days again for the transactions and errors of the statements chosen. +// +// The second pass, and not a window carried across day boundaries, because a +// statement's transaction can be filed on any day the walk read, in either +// direction: a transaction can run for days, and a late raw file can file a +// statement after the transaction that closed it. A window of any fixed width +// would give a different answer for those. Knowing the statements first means +// only their keys are looked for, at most Limit of them. func ReadAudit(q AuditQuery) ([]AuditEntry, error) { if q.Limit <= 0 { q.Limit = 100 } + match := q.matcher() days := storedDays() - var all []AuditRecord - var out []AuditEntry - for i := len(days) - 1; i >= 0 && len(out) < q.Limit; i-- { + var picked []AuditRecord // newest first, at most Limit + oldest := len(days) + for i := len(days) - 1; i >= 0 && len(picked) < q.Limit; i-- { if !q.Since.IsZero() && days[i] < q.Since.UTC().Format("2006-01-02") { break } - recs, err := readDay(days[i]) + // A day is walked backwards, so only its last matches can be shown. + need := q.Limit - len(picked) + var day []AuditRecord + err := scanDay(days[i], func(r AuditRecord) { + if !match(r) { + return + } + if len(day) == 2*need { + day = day[:copy(day, day[need:])] + } + day = append(day, r) + }) if err != nil { return nil, err } - all = append(recs, all...) - out = joinAudit(all, q) + if len(day) > need { + day = day[len(day)-need:] + } + for j := len(day) - 1; j >= 0; j-- { + picked = append(picked, day[j]) + } + oldest = i + } + join := newAuditJoin() + statements := map[auditKey]bool{} + for _, r := range picked { + if r.Kind == "statement" { + statements[keyOf(r)] = true + } + } + if len(statements) > 0 { + for _, day := range days[oldest:] { // oldest first: the last record of a key wins + err := scanDay(day, func(r AuditRecord) { + if statements[keyOf(r)] { + join.add(r) + } + }) + if err != nil { + return nil, err + } + } } - if len(out) > q.Limit { - out = out[:q.Limit] + var out []AuditEntry + for _, r := range picked { + out = append(out, join.entry(r)) } attachTexts(out) return out, nil @@ -551,56 +610,87 @@ func ReadAudit(q AuditQuery) ([]AuditEntry, error) { // joinAudit is ReadAudit's work on records in hand, pure so it is tested. func joinAudit(recs []AuditRecord, q AuditQuery) []AuditEntry { - type key struct{ session, vxid string } - txn := map[key]AuditRecord{} - errs := map[key]string{} + join := newAuditJoin() for _, r := range recs { - k := key{r.Session, r.VXID} - switch { - case r.Kind == "transaction" && r.Outcome != "SUBROLLBACK": - txn[k] = r - case r.Kind == "outcome" && r.SQLState != "": - errs[k] = r.SQLState + join.add(r) + } + match := q.matcher() + var out []AuditEntry + for i := len(recs) - 1; i >= 0; i-- { + if match(recs[i]) { + out = append(out, join.entry(recs[i])) } } + return out +} + +// auditKey is what joins a statement to its transaction's records. +type auditKey struct{ session, vxid string } + +func keyOf(r AuditRecord) auditKey { return auditKey{r.Session, r.VXID} } + +// auditJoin is how each transaction ended and which error it hit, by key. +type auditJoin struct { + txn map[auditKey]AuditRecord + errs map[auditKey]string +} + +func newAuditJoin() auditJoin { + return auditJoin{txn: map[auditKey]AuditRecord{}, errs: map[auditKey]string{}} +} + +// add takes in a record, if it says how a transaction went. +func (j auditJoin) add(r AuditRecord) { + switch { + case r.Kind == "transaction" && r.Outcome != "SUBROLLBACK": + j.txn[keyOf(r)] = r + case r.Kind == "outcome" && r.SQLState != "": + j.errs[keyOf(r)] = r.SQLState + } +} + +// entry is a record as people read it, joined to what was added. +func (j auditJoin) entry(r AuditRecord) AuditEntry { + e := AuditEntry{AuditRecord: r} + if r.Kind == "statement" { + if t, ok := j.txn[keyOf(r)]; ok { + e.TxnOutcome, e.TxnFor = t.Outcome, t.For + } + e.Error = j.errs[keyOf(r)] + } + return e +} + +// matcher is the query's filter over one record. +func (q AuditQuery) matcher() func(AuditRecord) bool { want := map[string]bool{} for _, k := range q.Kinds { want[k] = true } - var out []AuditEntry - for i := len(recs) - 1; i >= 0; i-- { - r := recs[i] + return func(r AuditRecord) bool { if len(want) > 0 && !want[r.Kind] { - continue + return false } if len(want) == 0 && (r.Kind == "transaction" || r.Kind == "heartbeat") { - continue // transactions are shown on the statements they close; heartbeats are the engine's + return false // transactions are shown on the statements they close; heartbeats are the engine's } if q.Branch != "" && r.Branch != q.Branch { - continue + return false } if q.Login != "" && r.Login != q.Login { - continue + return false } if q.Table != "" && !tableMatches(r.Object, q.Table) { - continue + return false } if !q.Since.IsZero() { // Parsed: as text, "…01.5Z" sorts after "…01.55Z". if at, err := time.Parse(time.RFC3339Nano, r.Time); err == nil && at.Before(q.Since) { - continue + return false } } - e := AuditEntry{AuditRecord: r} - if t, ok := txn[key{r.Session, r.VXID}]; ok && r.Kind == "statement" { - e.TxnOutcome, e.TxnFor = t.Outcome, t.For - } - if r.Kind == "statement" { - e.Error = errs[key{r.Session, r.VXID}] - } - out = append(out, e) + return true } - return out } // attachTexts gives each forensic record its statement text, while it is From a30afb0829d45d2c03375f412938f3f39b39294f Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 15:03:46 +0530 Subject: [PATCH 19/22] style: gofmt the evidence test Co-Authored-By: Claude Opus 5.5 --- enterprise/auditevidence/evidence_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/auditevidence/evidence_test.go b/enterprise/auditevidence/evidence_test.go index 68e380fc..06f4623a 100644 --- a/enterprise/auditevidence/evidence_test.go +++ b/enterprise/auditevidence/evidence_test.go @@ -27,7 +27,7 @@ func TestAssemble(t *testing.T) { for _, r := range in.records { branch.TallyAuditActivity(&in.activity, r) } - o :=branch.EvidenceOptions{Since: since, Until: until, By: "ada@example.com", Events: []auth.SecurityEvent{ + o := branch.EvidenceOptions{Since: since, Until: until, By: "ada@example.com", Events: []auth.SecurityEvent{ {Kind: auth.EvAuditLogOn, At: "2026-11-02T10:00:00Z", Actor: "ada@example.com", IP: "10.0.0.9"}, {Kind: auth.EvAuditMarked, At: "2026-10-30T10:00:00Z", Subject: "main", Detail: "auditing writes of public.t"}, // before the period {Kind: "user.login", At: "2026-11-02T11:00:00Z"}, // not the audit log's From b38f2fb2c191da0c9b0624bbb796ed0378ca22dc Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 15:03:46 +0530 Subject: [PATCH 20/22] fix(perf): advance a quiet realtime slot from the server's position The position moved only on a published commit, so a table nobody wrote to held the slot still while the rest of the database wrote, until max_slot_wal_keep_size invalidated it. With no transaction open and every subscriber's buffer empty, keepalives and XLogData's ServerWALEnd now acknowledge max(acked, position). Inside a transaction, with events still buffered, or with nobody attached, nothing moves. Moving the slot past the last commit a client saw would have made every reconnect on a quiet table a refetch, because the resume check treats anything before the slot's confirmed position as gone. quiet.go keeps, per slot, the span acknowledged with nothing published so such a position can still resume. Without a span, after a restart, it stays a refetch: never a silent gap. Also guards empty CopyData, which indexed Data[0] and would have panicked. Co-Authored-By: Claude Opus 5.5 --- enterprise/realtime/decode.go | 116 +++++++---- enterprise/realtime/hub.go | 20 ++ enterprise/realtime/quiet.go | 104 ++++++++++ enterprise/realtime/quiet_test.go | 275 +++++++++++++++++++++++++++ enterprise/realtime/resume.go | 16 +- internal/controlplane/realtime_on.go | 5 + 6 files changed, 501 insertions(+), 35 deletions(-) create mode 100644 enterprise/realtime/quiet.go create mode 100644 enterprise/realtime/quiet_test.go diff --git a/enterprise/realtime/decode.go b/enterprise/realtime/decode.go index 3ab0ed4f..865df015 100644 --- a/enterprise/realtime/decode.go +++ b/enterprise/realtime/decode.go @@ -48,9 +48,10 @@ type Decoder struct { // streaming sends only complete transactions, in commit order, so there is // exactly one of these at a time and no interleaving to untangle. tx txState - // acked is the last position every subscriber has been given. Only this is - // reported to Postgres, so WAL behind it stays reclaimable and WAL ahead of - // it stays available for a subscriber that reconnects. + // acked is the last position every subscriber has been given, or the + // server's own position once nothing is owed to anyone (advanceIfQuiet). + // Only this is reported to Postgres, so WAL behind it stays reclaimable and + // WAL ahead of it stays available for a subscriber that reconnects. acked pglogrepl.LSN } @@ -134,6 +135,7 @@ func (d *Decoder) Run(ctx context.Context, since pglogrepl.LSN) error { } } d.acked = since + noteQuietStart(d.slot, since, d.hub.Floor()) if err := pglogrepl.StartReplication(ctx, d.conn, d.slot, since, pglogrepl.StartReplicationOptions{PluginArgs: []string{ "proto_version '1'", "publication_names '" + d.publication + "'", @@ -166,41 +168,81 @@ func (d *Decoder) Run(ctx context.Context, since pglogrepl.LSN) error { if !ok { continue // a notice or similar; nothing to decode } - switch cd.Data[0] { - case pglogrepl.PrimaryKeepaliveMessageByteID: - ka, err := pglogrepl.ParsePrimaryKeepaliveMessage(cd.Data[1:]) - if err != nil { - return err - } - if ka.ReplyRequested { - next = time.Now() // answer on the next turn of the loop - } - case pglogrepl.XLogDataByteID: - xld, err := pglogrepl.ParseXLogData(cd.Data[1:]) - if err != nil { - return err - } - if err := d.handle(xld); err != nil { - return err - } - // A subscriber overflowed inside the transaction that has just - // finished, and is still attached for exactly this moment: report - // the position before it goes, because the last subscriber leaving - // is what stops this decoder, and a transaction nobody could be - // given has to be behind us when that happens. Otherwise the next - // subscriber starts in front of it and overflows in the same place. - if d.hub.Releasable() { - if err := pglogrepl.SendStandbyStatusUpdate(ctx, d.conn, - pglogrepl.StandbyStatusUpdate{WALWritePosition: d.acked}); err != nil { - return fmt.Errorf("reporting the position of a transaction that could not be delivered: %w", err) - } - next = time.Now().Add(StandbyInterval) - d.hub.ReleasePending() + reply, err := d.handleCopyData(cd.Data) + if err != nil { + return err + } + if reply { + next = time.Now() // answer on the next turn of the loop + } + // A subscriber overflowed inside the transaction that has just + // finished, and is still attached for exactly this moment: report + // the position before it goes, because the last subscriber leaving + // is what stops this decoder, and a transaction nobody could be + // given has to be behind us when that happens. Otherwise the next + // subscriber starts in front of it and overflows in the same place. + if d.hub.Releasable() { + if err := pglogrepl.SendStandbyStatusUpdate(ctx, d.conn, + pglogrepl.StandbyStatusUpdate{WALWritePosition: d.acked}); err != nil { + return fmt.Errorf("reporting the position of a transaction that could not be delivered: %w", err) } + next = time.Now().Add(StandbyInterval) + d.hub.ReleasePending() } } } +// handleCopyData takes in one message from the replication stream and reports +// whether Postgres asked for an answer straight away. +func (d *Decoder) handleCopyData(data []byte) (bool, error) { + if len(data) == 0 { + return false, nil // nothing a walsender sends, and nothing to index + } + switch data[0] { + case pglogrepl.PrimaryKeepaliveMessageByteID: + ka, err := pglogrepl.ParsePrimaryKeepaliveMessage(data[1:]) + if err != nil { + return false, err + } + d.advanceIfQuiet(ka.ServerWALEnd) + return ka.ReplyRequested, nil + case pglogrepl.XLogDataByteID: + xld, err := pglogrepl.ParseXLogData(data[1:]) + if err != nil { + return false, err + } + if err := d.handle(xld); err != nil { + return false, err + } + d.advanceIfQuiet(xld.ServerWALEnd) + } + return false, nil +} + +// advanceIfQuiet acknowledges a position the server has reported, when that +// is safe to do. +// +// A table nobody writes to produces no commits here, so without this the +// position stands still while the rest of the database keeps writing, the WAL +// behind it cannot be reclaimed, and max_slot_wal_keep_size eventually +// invalidates the slot. Nobody could be told what happened: there was nothing +// to deliver. +// +// The server only sends a position once everything committing before it has +// been written to this connection, and a message arrives after the ones sent +// before it. So once no transaction is open here, and everything handed to +// subscribers has left their buffers, nothing before that position is still +// owed to anyone. Transactions not yet committed at that position are not +// affected: they are sent when they commit, and Postgres keeps their WAL for +// that whatever is acknowledged here. +func (d *Decoder) advanceIfQuiet(end pglogrepl.LSN) { + if d.tx.commitLSN != "" || end <= d.acked || !d.hub.Drained() { + return + } + d.acked = end + noteQuietAcked(d.slot, end) +} + // Close ends the replication connection. The slot stays: it is what a // reconnecting subscriber resumes from, and sweeping it is a decision made // elsewhere, where "is anyone still interested" can be answered. @@ -284,10 +326,16 @@ func (d *Decoder) handleMessage(m pglogrepl.Message, lsn string) error { At: msg.CommitTime.UTC().Format(time.RFC3339Nano), Changes: d.tx.changes}) } - d.tx = txState{} // Everything in this transaction has been handed to the hub, so the // position is now safe to acknowledge. + published := d.tx.begun + d.tx = txState{} d.acked = msg.CommitLSN + if published { + noteQuietPublished(d.slot, msg.CommitLSN) + } else { + noteQuietAcked(d.slot, msg.CommitLSN) + } } return nil } diff --git a/enterprise/realtime/hub.go b/enterprise/realtime/hub.go index 54b8c926..943042da 100644 --- a/enterprise/realtime/hub.go +++ b/enterprise/realtime/hub.go @@ -230,6 +230,26 @@ func (h *Hub) Stats() (subscribers int, delivered int64, peak int) { return len(h.subs), h.delivered, h.peak } +// Drained reports whether somebody is attached and has been given everything +// published so far: no buffer holds an event, and nobody is waiting to be told +// their stream is over. +// +// False with no subscribers at all. Nobody has been delivered anything then, +// and the decoder is either about to stop or about to be subscribed to. +func (h *Hub) Drained() bool { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.subs) == 0 { + return false + } + for _, s := range h.subs { + if len(s.ch) > 0 || s.isPending() { + return false + } + } + return true +} + // Publish sends to every subscriber without blocking on any of them. func (h *Hub) Publish(ev any) { done := false // a transaction finished in this call diff --git a/enterprise/realtime/quiet.go b/enterprise/realtime/quiet.go new file mode 100644 index 00000000..fe1e6efb --- /dev/null +++ b/enterprise/realtime/quiet.go @@ -0,0 +1,104 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package realtime + +import ( + "sync" + + "github.com/jackc/pglogrepl" +) + +// What a quiet stretch of a slot's position leaves behind for a resume. +// +// A slot acknowledged on keepalives moves past positions where nothing was +// published, which is what stops a quiet table's WAL from piling up. But the +// resume check reads the slot's confirmed position as "everything before this +// was somebody else's": a client reconnecting with the commit position it +// last saw is judged against it, and the slot having moved on would tell it to +// refetch, for a gap that never held a single change. A client on a quiet table +// reconnects at least hourly, so that would be a refetch an hour for nothing. +// +// So this keeps, per slot, the span the decoder has acknowledged without +// publishing anything: from the last position a change was published at +// (from) to the furthest position acknowledged (to). A position inside that +// span has missed nothing, and QuietSpan hands it to the resume check. +// +// In memory, per process, and only ever evidence that a position is safe: with +// nothing recorded (a restarted control plane, a slot read by something else) +// the check goes back to refusing whatever is before the slot's position, which +// costs a client one refetch and never a silent gap. A span from a slot that +// has since been dropped or moved by something else is refused by QuietSpan, +// because the slot's position is then past anything this process acknowledged. +type quietSpan struct{ from, to pglogrepl.LSN } + +var quiet = struct { + mu sync.Mutex + m map[string]quietSpan +}{m: map[string]quietSpan{}} + +// noteQuietStart records a decoder starting on a slot. The first decoder on a +// slot since this process began starts the span at the position it was asked +// for, or else at the hub's floor; a later one continues the span. +func noteQuietStart(slot string, since, floor pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s, ok := quiet.m[slot] + if !ok { + s.from = since + if s.from == 0 { + s.from = floor + } + } + if since > s.to { + s.to = since + } + quiet.m[slot] = s +} + +// noteQuietAcked records a position acknowledged with nothing published. +func noteQuietAcked(slot string, lsn pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s := quiet.m[slot] + if lsn > s.to { + s.to = lsn + } + quiet.m[slot] = s +} + +// noteQuietPublished records a position a change was published at: everything +// before it has been somebody's to receive, so the span starts again here. +func noteQuietPublished(slot string, lsn pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s := quiet.m[slot] + s.from = lsn + if lsn > s.to { + s.to = lsn + } + quiet.m[slot] = s +} + +// QuietSpan is a slot's quiet span for a resume check, given the slot's +// confirmed position as read just now. Both are zero when there is nothing +// trustworthy to go on. A confirmed position past what this process ever +// acknowledged means the slot is not the one the span describes -- dropped and +// made again, or moved by something else -- so the span is forgotten. +func QuietSpan(slot string, confirmed pglogrepl.LSN, haveSlot bool) (from, to pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s, ok := quiet.m[slot] + if !ok { + return 0, 0 + } + if !haveSlot || confirmed > s.to { + delete(quiet.m, slot) + return 0, 0 + } + if s.from == 0 { + return 0, 0 + } + return s.from, s.to +} diff --git a/enterprise/realtime/quiet_test.go b/enterprise/realtime/quiet_test.go new file mode 100644 index 00000000..f41744f9 --- /dev/null +++ b/enterprise/realtime/quiet_test.go @@ -0,0 +1,275 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package realtime + +import ( + "encoding/binary" + "fmt" + "testing" + + "github.com/jackc/pglogrepl" +) + +// A decoder on a test's own slot, with one subscriber that has read everything +// so far unless the test leaves its events in the buffer. +func quietDecoder(t *testing.T, slot string) (*Decoder, *Subscriber) { + t.Helper() + quiet.mu.Lock() + delete(quiet.m, slot) + quiet.mu.Unlock() + h := NewHub() + s := h.Subscribe() + t.Cleanup(s.Close) + d := &Decoder{hub: h, slot: slot, relations: map[uint32]Relation{}} + d.relations[1] = Relation{Schema: "public", Name: "t", ReplicaIdentity: 'd', + Columns: []Column{{Name: "id", TypeOID: 20, Key: true}}} + return d, s +} + +func keepaliveAt(end pglogrepl.LSN, reply bool) []byte { + b := make([]byte, 18) + b[0] = pglogrepl.PrimaryKeepaliveMessageByteID + binary.BigEndian.PutUint64(b[1:], uint64(end)) + if reply { + b[17] = 1 + } + return b +} + +// xlogAt wraps pgoutput's own bytes for a message as a walsender sends them. +func xlogAt(start, end pglogrepl.LSN, wal []byte) []byte { + b := make([]byte, 25, 25+len(wal)) + b[0] = pglogrepl.XLogDataByteID + binary.BigEndian.PutUint64(b[1:], uint64(start)) + binary.BigEndian.PutUint64(b[9:], uint64(end)) + return append(b, wal...) +} + +func beginWAL(final pglogrepl.LSN, xid uint32) []byte { + b := make([]byte, 21) + b[0] = 'B' + binary.BigEndian.PutUint64(b[1:], uint64(final)) + binary.BigEndian.PutUint32(b[17:], xid) + return b +} + +func commitWAL(commit, end pglogrepl.LSN) []byte { + b := make([]byte, 26) + b[0] = 'C' + binary.BigEndian.PutUint64(b[2:], uint64(commit)) + binary.BigEndian.PutUint64(b[10:], uint64(end)) + return b +} + +func insertWAL(rel uint32, value string) []byte { + b := []byte{'I', 0, 0, 0, 0, 'N', 0, 1, 't', 0, 0, 0, byte(len(value))} + binary.BigEndian.PutUint32(b[1:], rel) + return append(b, value...) +} + +func drain(s *Subscriber) (n int) { + for { + select { + case <-s.Events(): + n++ + default: + return n + } + } +} + +func mustCopy(t *testing.T, d *Decoder, data []byte) bool { + t.Helper() + reply, err := d.handleCopyData(data) + if err != nil { + t.Fatal(err) + } + return reply +} + +func TestAKeepaliveAdvancesAQuietSlot(t *testing.T) { + d, _ := quietDecoder(t, "ka") + if mustCopy(t, d, keepaliveAt(1000, false)) { + t.Error("a keepalive not asking for a reply got one") + } + if d.acked != 1000 { + t.Fatalf("acked = %v, want the keepalive's position", d.acked) + } + if !mustCopy(t, d, keepaliveAt(1500, true)) { + t.Error("a keepalive asking for a reply did not get one") + } + // Never backwards: a server can report what it has already reported. + mustCopy(t, d, keepaliveAt(1200, false)) + if d.acked != 1500 { + t.Errorf("acked = %v after an older keepalive, want 1500", d.acked) + } + // The XLogData header carries a position too, and a message arriving when + // nothing is open advances on it just the same. + mustCopy(t, d, xlogAt(1500, 1800, []byte("O\x00\x00\x00\x00\x00\x00\x00\x00x\x00"))) + if d.acked != 1800 { + t.Errorf("acked = %v after XLogData's position, want 1800", d.acked) + } +} + +// Empty CopyData has no type byte: indexing it would panic in the decoder that +// every subscriber on the branch is reading from. +func TestEmptyCopyDataIsHarmless(t *testing.T) { + d, _ := quietDecoder(t, "empty") + if reply, err := d.handleCopyData(nil); reply || err != nil { + t.Fatalf("empty CopyData: %v %v", reply, err) + } + if reply, err := d.handleCopyData([]byte{}); reply || err != nil { + t.Fatalf("empty CopyData: %v %v", reply, err) + } + if d.acked != 0 { + t.Errorf("acked moved on nothing: %v", d.acked) + } + // Short messages are refused by their parsers, not read past their end. + if _, err := d.handleCopyData([]byte{pglogrepl.PrimaryKeepaliveMessageByteID}); err == nil { + t.Error("a truncated keepalive was accepted") + } +} + +// The one that must not happen: a position past a change someone has not been +// given. Inside a transaction a keepalive may be ahead of the commit. +func TestAKeepaliveDoesNotAcknowledgeInsideATransaction(t *testing.T) { + d, s := quietDecoder(t, "intx") + mustCopy(t, d, keepaliveAt(1000, false)) + mustCopy(t, d, xlogAt(1000, 1100, beginWAL(2000, 7))) + mustCopy(t, d, keepaliveAt(1900, false)) + if d.acked != 1000 { + t.Fatalf("a keepalive inside a transaction moved acked to %v", d.acked) + } + mustCopy(t, d, xlogAt(1100, 1200, insertWAL(1, "1"))) + drain(s) + mustCopy(t, d, keepaliveAt(1950, false)) + if d.acked != 1000 { + t.Fatalf("a keepalive after a row moved acked to %v", d.acked) + } + mustCopy(t, d, xlogAt(1200, 2000, commitWAL(2000, 2010))) + if d.acked != 2000 { + t.Fatalf("a commit acknowledged %v, want the commit's position", d.acked) + } + drain(s) + mustCopy(t, d, keepaliveAt(3000, false)) + if d.acked != 3000 { + t.Errorf("acked = %v once quiet again, want 3000", d.acked) + } +} + +func TestAKeepaliveWaitsForSubscribersToBeGivenEverything(t *testing.T) { + d, s := quietDecoder(t, "buffered") + d.hub.Publish(Commit{Type: "commit"}) // an event still in the buffer + mustCopy(t, d, keepaliveAt(1000, false)) + if d.acked != 0 { + t.Fatalf("acked = %v with an event still buffered", d.acked) + } + drain(s) + mustCopy(t, d, keepaliveAt(1000, false)) + if d.acked != 1000 { + t.Errorf("acked = %v once drained, want 1000", d.acked) + } +} + +// With nobody attached, nobody has been given anything. +func TestAKeepaliveDoesNotAdvanceWithNoSubscribers(t *testing.T) { + d, s := quietDecoder(t, "nosubs") + s.Close() + mustCopy(t, d, keepaliveAt(1000, false)) + if d.acked != 0 { + t.Errorf("acked = %v with nobody attached", d.acked) + } +} + +// The resume check against the positions acknowledged on keepalives, which is +// what the keepalive advance puts at risk: a client that saw the last commit, +// on a table nobody has written to since, must still be able to come back. +func TestAResumeSurvivesAQuietSlotMovingOn(t *testing.T) { + d, s := quietDecoder(t, "resume") + noteQuietStart(d.slot, 1000, 1000) + d.acked = 1000 + mustCopy(t, d, xlogAt(1000, 1100, beginWAL(2000, 7))) + mustCopy(t, d, xlogAt(1100, 1200, insertWAL(1, "1"))) + mustCopy(t, d, xlogAt(1200, 2000, commitWAL(2000, 2010))) + drain(s) + mustCopy(t, d, keepaliveAt(9000, false)) + if d.acked != 9000 { + t.Fatalf("acked = %v", d.acked) + } + + window := func(confirmed pglogrepl.LSN) Window { + w := Window{HaveSlot: true, Confirmed: confirmed, WALEnd: 10000} + w.QuietFrom, w.QuietTo = QuietSpan(d.slot, confirmed, true) + return w + } + w := window(9000) + if got := JudgeResume(2000, w); got != ResumeFrom { + t.Errorf("a client at the last commit, after only keepalives: %v, want ResumeFrom", got) + } + if got := JudgeResume(5000, w); got != ResumeFrom { + t.Errorf("a position inside the quiet span: %v, want ResumeFrom", got) + } + if got := JudgeResume(1500, w); got != ResumeExpired { + t.Errorf("a client from before the last commit has missed a change: %v, want ResumeExpired", got) + } + + // A slot gone past what this process acknowledged is a different slot: + // the span is no evidence about it, and is forgotten. + w = window(12000) + if w.QuietFrom != 0 || w.QuietTo != 0 { + t.Fatalf("a span survived a slot that moved past it: %+v", w) + } + if got := JudgeResume(2000, Window{HaveSlot: true, Confirmed: 12000, WALEnd: 20000}); got != ResumeExpired { + t.Errorf("no evidence must mean a refetch: %v", got) + } +} + +func TestAQuietSpanIsForgottenWithItsSlot(t *testing.T) { + const slot = "dropped" + noteQuietPublished(slot, 100) + noteQuietAcked(slot, 500) + if from, to := QuietSpan(slot, 500, true); from != 100 || to != 500 { + t.Fatalf("span: %v %v", from, to) + } + if from, _ := QuietSpan(slot, 0, false); from != 0 { + t.Fatal("a span survived its slot") + } + if from, _ := QuietSpan(slot, 500, true); from != 0 { + t.Fatal("a forgotten span came back") + } + if from, to := QuietSpan("unknown-slot", 500, true); from != 0 || to != 0 { + t.Fatalf("a span for a slot never seen: %v %v", from, to) + } + // A fan-out that could not say where it started has no span to give. + noteQuietStart("no-floor", 0, 0) + if from, _ := QuietSpan("no-floor", 0, true); from != 0 { + t.Errorf("a span from nowhere: %v", from) + } + // A later decoder continues a span rather than restarting it. + noteQuietPublished("continued", 800) + noteQuietStart("continued", 900, 900) + if from, to := QuietSpan("continued", 900, true); from != 800 || to != 900 { + t.Errorf("continued span: %v %v", from, to) + } +} + +func TestJudgeResumeQuietNeedsTheSpanToCoverTheSlot(t *testing.T) { + cases := []struct { + w Window + want ResumeVerdict + }{ + {Window{HaveSlot: true, Confirmed: 5000, QuietFrom: 1000, QuietTo: 5000}, ResumeFrom}, + {Window{HaveSlot: true, Confirmed: 5000, QuietFrom: 1000, QuietTo: 6000}, ResumeFrom}, // the slot lags what was reported + {Window{HaveSlot: true, Confirmed: 7000, QuietFrom: 1000, QuietTo: 6000}, ResumeExpired}, + {Window{HaveSlot: true, Confirmed: 5000, QuietFrom: 3000, QuietTo: 6000}, ResumeExpired}, + {Window{HaveSlot: true, Confirmed: 5000}, ResumeExpired}, + {Window{HaveSlot: false, Confirmed: 0, QuietFrom: 1000, QuietTo: 6000}, ResumeExpired}, + } + for i, c := range cases { + if got := JudgeResume(2000, c.w); got != c.want { + t.Errorf("case %d (%s): %v, want %v", i, fmt.Sprint(c.w), got, c.want) + } + } +} diff --git a/enterprise/realtime/resume.go b/enterprise/realtime/resume.go index 3dd55275..70bbf33d 100644 --- a/enterprise/realtime/resume.go +++ b/enterprise/realtime/resume.go @@ -72,6 +72,15 @@ type Window struct { // changes are published to the subscribers already attached. Meaningless // unless Live. Floor pglogrepl.LSN + // QuietFrom and QuietTo are the span the slot has been acknowledged across + // with nothing published (QuietSpan), or zero when that is not known. + QuietFrom, QuietTo pglogrepl.LSN +} + +// quiet reports whether a position lies in the span acknowledged with nothing +// published, and that span reaches as far as the slot has. +func (w Window) quiet(since pglogrepl.LSN) bool { + return w.QuietFrom != 0 && since >= w.QuietFrom && w.Confirmed <= w.QuietTo } // JudgeResume decides what to do with a subscriber's ?since=. @@ -109,7 +118,12 @@ func JudgeResume(since pglogrepl.LSN, w Window) ResumeVerdict { // Postgres will not go back further than the slot has been acknowledged to, // and it does not say so. Asking for an older position there is how the // stream silently became "from the slot" instead. - if !w.HaveSlot || since < w.Confirmed { + // + // Older than the slot is still fine when the slot only moved past it on + // keepalives: nothing was published in between, so the position has missed + // nothing, and starting from the confirmed position is starting in the + // right place. Quiet is what says so; see quiet.go. + if !w.HaveSlot || (since < w.Confirmed && !w.quiet(since)) { return ResumeExpired } return ResumeFrom diff --git a/internal/controlplane/realtime_on.go b/internal/controlplane/realtime_on.go index 923ccdd2..c5c1b810 100644 --- a/internal/controlplane/realtime_on.go +++ b/internal/controlplane/realtime_on.go @@ -235,6 +235,8 @@ func resumeFor(name string, since pglogrepl.LSN) (*realtime.Hub, realtime.Resume floor := w.Confirmed if !w.HaveSlot { floor = w.WALEnd + } else if w.QuietFrom != 0 && w.QuietFrom < floor { + floor = w.QuietFrom // nothing was published between there and the slot } h, _ := startHub(name, 0, floor) return h, realtime.ResumeLive, nil @@ -306,6 +308,9 @@ func resumeWindow(name string) (realtime.Window, error) { if lsn, err := pglogrepl.ParseLSN(rw.WALEnd); err == nil { w.WALEnd = lsn } + // What the slot moved past on keepalives, with nothing published: a position + // in there has missed nothing, however far the slot has gone since. + w.QuietFrom, w.QuietTo = realtime.QuietSpan(branch.SlotName(name, "stream"), w.Confirmed, w.HaveSlot) return w, nil } From 6a6cf4ac9c388c2a9ad69bf9f18cccc8d76c9169 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 15:03:46 +0530 Subject: [PATCH 21/22] fix(security): take EXECUTE on the Blackbox's helpers away from clients SECURITY DEFINER functions in schema bb ran for anyone who could call one, and ledger.sql granted the whole schema to db_client: bb._record_impact wrote chained entries from caller-supplied actors. Revoke by default: every SECURITY DEFINER function not on the list of client-callable ones loses EXECUTE from PUBLIC and db_client, on every install, and an event trigger seals any made later. The trigger runs ALWAYS because most of the later schema files run in replica mode. Co-Authored-By: Claude Opus 5.5 --- internal/ledger/datachanges.sql | 9 ++-- internal/ledger/identity.sql | 8 +-- internal/ledger/ledger.go | 4 +- internal/ledger/ledger.sql | 85 +++++++++++++++++++++++++++-- internal/ledger/sealing_test.go | 89 +++++++++++++++++++++++++++++++ internal/ledger/statement_test.go | 9 ++-- 6 files changed, 183 insertions(+), 21 deletions(-) create mode 100644 internal/ledger/sealing_test.go diff --git a/internal/ledger/datachanges.sql b/internal/ledger/datachanges.sql index f3609543..e482e010 100644 --- a/internal/ledger/datachanges.sql +++ b/internal/ledger/datachanges.sql @@ -515,11 +515,10 @@ BEGIN END; $$; --- No REVOKE here: ledger.sql grants every bb function to db_client on each --- start, so a REVOKE would have to run on each start too, and a privilege --- change after the event triggers are on is recorded as an unattributed --- change (integration-v2 §9). Calling bb._attach_data_triggers only adds --- these protective triggers, which is harmless. +-- No REVOKE here: a privilege change after the event triggers are on is +-- recorded as an unattributed change (integration-v2 §9). None is needed: +-- bb.seal_new_functions (ledger.sql) takes EXECUTE from clients on each +-- SECURITY DEFINER function here as it is made. -- Every existing table, now. (Creating these triggers is DDL; ledger.sql's -- recorder skips the Blackbox's own triggers, so a start records nothing.) diff --git a/internal/ledger/identity.sql b/internal/ledger/identity.sql index 807f84a8..deb2a151 100644 --- a/internal/ledger/identity.sql +++ b/internal/ledger/identity.sql @@ -30,10 +30,10 @@ SET session_replication_role = replica; -- ── The private half ──────────────────────────────────────────────────────── --- ledger.sql grants db_client EXECUTE on every function in schema bb, on every --- start. The MAC and the secret must never be callable by a client (a client --- that could compute a MAC could forge an actor), so they live in a schema no --- one but its owner can use. +-- The MAC and the secret must never be callable by a client (a client that +-- could compute a MAC could forge an actor), so they live in a schema no one +-- but its owner can use, rather than relying on bb's list of functions a +-- client may call (ledger.sql, bb._client_callable). CREATE SCHEMA IF NOT EXISTS bb_f1; REVOKE ALL ON SCHEMA bb_f1 FROM PUBLIC; diff --git a/internal/ledger/ledger.go b/internal/ledger/ledger.go index eb49271e..389123e8 100644 --- a/internal/ledger/ledger.go +++ b/internal/ledger/ledger.go @@ -18,8 +18,8 @@ const SchemaName = "bb" // PrivateSchemaName holds what no client may reach: application identity's // install secret and the function that makes its MACs (identity.sql). A schema -// of its own because ledger.sql grants db_client EXECUTE on everything in -// SchemaName. Like SchemaName it is the engine's, never a user's: everything +// of its own so that the secret never depends on what SchemaName exposes to +// db_client. Like SchemaName it is the engine's, never a user's: everything // that lists, counts, guards or tracks a user's tables leaves both out. const PrivateSchemaName = "bb_f1" diff --git a/internal/ledger/ledger.sql b/internal/ledger/ledger.sql index dbcd5727..d6afcb73 100644 --- a/internal/ledger/ledger.sql +++ b/internal/ledger/ledger.sql @@ -17,6 +17,7 @@ DROP EVENT TRIGGER IF EXISTS bb_guard_start; DROP EVENT TRIGGER IF EXISTS bb_log_end; DROP EVENT TRIGGER IF EXISTS bb_log_drop; +DROP EVENT TRIGGER IF EXISTS bb_seal_functions; CREATE EXTENSION IF NOT EXISTS dblink; CREATE SCHEMA IF NOT EXISTS bb; @@ -858,11 +859,79 @@ ALTER DEFAULT PRIVILEGES FOR ROLE dbadmin IN SCHEMA public GRANT ALL ON SEQUENCE -- INSERT/UPDATE/DELETE on the table at all — it cannot forge or rewrite entries. GRANT USAGE ON SCHEMA bb TO db_client; GRANT SELECT ON bb.schema_ledger, bb.policy, bb.policy_history TO db_client; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA bb TO db_client; --- …except the one that advances a run's statement position: a client calling it --- could line a blocked statement up with a harmless one's text. -REVOKE EXECUTE ON FUNCTION bb._statement_texts(text, text, text) FROM PUBLIC, db_client; --- …and dblink, wherever it lives. Only the Blackbox's SECURITY DEFINER triggers + +-- Functions. A SECURITY DEFINER function runs as the Blackbox's owner, so being +-- able to call one is being able to do what it does: bb._record_impact takes its +-- actor as arguments and writes a chained entry, bb._statement_texts moves a +-- run's statement position so a blocked statement matches a harmless one's text. +-- Postgres gives every new function to PUBLIC, and everything in bb used to be +-- granted to db_client on top, so all of them were callable. +-- +-- So: no client may call a SECURITY DEFINER function here unless it is listed. +-- The list is every function a client, the gateway or the console calls itself, +-- SECURITY DEFINER or not (bb.act_for is the only one that is today); nothing +-- else a client reaches needs its own grant. Trigger and event-trigger functions +-- are not on it: Postgres checks EXECUTE only when a trigger is made, and they +-- are made here, by the owner, so they fire for any client. So are the nested +-- calls among these functions: the inner call is checked against the owner. +-- SECURITY INVOKER helpers keep what they have; they can do no more than their caller. +CREATE OR REPLACE FUNCTION bb._client_callable() RETURNS name[] +LANGUAGE sql IMMUTABLE AS $$ + SELECT ARRAY[ + 'act_for', 'app_actor', 'app_actor_label', 'app_identity_mode', -- application identity + '_ledger_hash', '_ext_hash', -- verifying the record + 'track_table', 'untrack_table', 'tracked_tables', 'track_row', -- row tracking, by a table's owner + 'policy_check', 'blast_radius', -- previews in the console and MCP + 'ledger_v2_version', 'blackbox_provenance_version', + 'blackbox_policy_version', 'blackbox_impact_version' + ]::name[]; +$$; + +-- Take EXECUTE from clients on one function, unless it is listed above. +-- session_replication_role: the REVOKE is the Blackbox's own plumbing, not a +-- change anybody made. +CREATE OR REPLACE FUNCTION bb._seal_function(f oid) RETURNS void +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog SET session_replication_role = replica AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_proc p WHERE p.oid = f AND p.prosecdef AND NOT p.proname = ANY (bb._client_callable())) THEN + EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC, db_client', f::regprocedure); + END IF; +END; +$$; + +-- A function added later, by any of the schema files that run after this one +-- or by a future release, is sealed as it is made. The files that come after +-- are run with session_replication_role = replica or must not REVOKE at all +-- (see datachanges.sql), so the event trigger below is what covers a new +-- install, where those functions do not exist yet at this point in the file. +CREATE OR REPLACE FUNCTION bb.seal_new_functions() RETURNS event_trigger +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $$ +DECLARE r record; +BEGIN + FOR r IN SELECT objid FROM pg_event_trigger_ddl_commands() + WHERE object_type IN ('function', 'procedure') AND schema_name = 'bb' LOOP + PERFORM bb._seal_function(r.objid); + END LOOP; +END; +$$; + +-- And every function already here, on every start: this is what closes an +-- existing install. A CREATE OR REPLACE keeps a function's privileges, so what +-- is sealed stays sealed. +DO $seal$ +DECLARE p record; +BEGIN + FOR p IN SELECT f.oid, f.proname FROM pg_proc f JOIN pg_namespace n ON n.oid = f.pronamespace + WHERE n.nspname = 'bb' AND f.prokind IN ('f', 'p') LOOP + IF p.proname = ANY (bb._client_callable()) THEN + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO db_client', p.oid::regprocedure); + ELSE + PERFORM bb._seal_function(p.oid); + END IF; + END LOOP; +END $seal$; + +-- dblink, too, wherever it lives. Only the Blackbox's SECURITY DEFINER triggers -- call it, and they run as the owner. A client holding it could open -- connections from inside the database to anywhere the container can reach, -- with any credentials it has or has learned (audit GW-1). @@ -892,3 +961,9 @@ CREATE EVENT TRIGGER bb_log_end ON ddl_command_end EXECUTE FUNCTION bb.log_ddl_end(); CREATE EVENT TRIGGER bb_log_drop ON sql_drop EXECUTE FUNCTION bb.log_ddl_drop(); +-- ALWAYS: most of the schema files that follow run with session_replication_role +-- = replica, and a function they make must be sealed all the same. +CREATE EVENT TRIGGER bb_seal_functions ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION', 'ALTER FUNCTION', 'CREATE PROCEDURE', 'ALTER PROCEDURE', 'ALTER ROUTINE') + EXECUTE FUNCTION bb.seal_new_functions(); +ALTER EVENT TRIGGER bb_seal_functions ENABLE ALWAYS; diff --git a/internal/ledger/sealing_test.go b/internal/ledger/sealing_test.go new file mode 100644 index 00000000..255ef75f --- /dev/null +++ b/internal/ledger/sealing_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "regexp" + "strings" + "testing" +) + +// allowedCallable reads ledger.sql's list of what a client may call. +func allowedCallable(t *testing.T) map[string]bool { + t.Helper() + i := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb._client_callable()") + if i < 0 { + t.Fatal("bb._client_callable is gone") + } + body := Schema[i:] + body = body[:strings.Index(body, "$$;")] + body = body[strings.Index(body, "SELECT ARRAY["):] + allowed := map[string]bool{} + for _, m := range regexp.MustCompile(`'(_?[a-z0-9_]+)'`).FindAllStringSubmatch(body, -1) { + allowed[m[1]] = true + } + return allowed +} + +// The ledger's entries are forged through what a client can call, so nothing a +// client can call may be a helper that writes one. Nothing grants the whole +// schema any more, and the seal is applied both to what exists and to what is +// made later (which, for most schema files, is made under a replica session). +func TestClientsCannotCallTheHelpers(t *testing.T) { + if strings.Contains(Schema, "ON ALL FUNCTIONS IN SCHEMA bb") { + t.Error("ledger.sql grants the whole of schema bb again") + } + for _, want := range []string{ + "format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC, db_client', f::regprocedure)", + "AND p.prosecdef AND NOT p.proname = ANY (bb._client_callable())", + "WHEN TAG IN ('CREATE FUNCTION', 'ALTER FUNCTION', 'CREATE PROCEDURE', 'ALTER PROCEDURE', 'ALTER ROUTINE')", + "ALTER EVENT TRIGGER bb_seal_functions ENABLE ALWAYS;", + "SET search_path = pg_catalog SET session_replication_role = replica", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } + // On every start, before capture is switched on, like the other privileges. + sweep := strings.Index(Schema, "DO $seal$") + on := strings.Index(Schema, "CREATE EVENT TRIGGER bb_guard_start") + if sweep < 0 || on < 0 || sweep > on { + t.Errorf("the sweep over existing functions must run before capture is on (sweep %d, capture %d)", sweep, on) + } + + allowed := allowedCallable(t) + for _, forbidden := range []string{"_record_impact", "_attach_data_triggers", "_statement_texts", + "guard_ddl_start", "log_ddl_end", "guard_truncate", "capture_ext", "_capture_disabled"} { + if allowed[forbidden] { + t.Errorf("bb.%s is client-callable: a client could use the owner's rights through it", forbidden) + } + } +} + +// Everything the client-side code calls as a client is on the list. What the +// engine calls as its superuser needs no grant, and neither does a function +// the docs name only for operators. +func TestClientCalledFunctionsAreOnTheList(t *testing.T) { + allowed := allowedCallable(t) + for _, name := range []string{"act_for", "app_actor", "app_actor_label", "app_identity_mode", + "_ledger_hash", "_ext_hash", "policy_check", "blast_radius"} { + if !allowed[name] { + t.Errorf("bb.%s is called by clients and is not on the list", name) + } + } +} + +// Every SECURITY DEFINER function the free schemas define is either on the +// list, named to be sealed, or simply not; what must hold is that the list +// does not grow by accident to include a definer that writes the ledger. +func TestOnlyActForIsAClientCallableDefiner(t *testing.T) { + allowed := allowedCallable(t) + def := regexp.MustCompile(`(?is)FUNCTION bb\.([a-z0-9_]+)\([^)]*\)[^;$]*?SECURITY DEFINER`) + for _, sql := range []string{Schema, SchemaData, SchemaExt, SchemaTracking, SchemaIdentity} { + for _, m := range def.FindAllStringSubmatch(sql, -1) { + if allowed[m[1]] && m[1] != "act_for" { + t.Errorf("bb.%s is SECURITY DEFINER and client-callable: review it, then allow it here", m[1]) + } + } + } +} diff --git a/internal/ledger/statement_test.go b/internal/ledger/statement_test.go index 67dc4e33..f8205e53 100644 --- a/internal/ledger/statement_test.go +++ b/internal/ledger/statement_test.go @@ -32,11 +32,10 @@ func TestSchemaStatementMatching(t *testing.T) { } // A client that could call the position function could line a blocked statement - // up with a harmless one's text. The revoke must come after the blanket grant. - grant := strings.Index(ledgerSQL, "GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA bb TO db_client;") - revoke := strings.Index(ledgerSQL, "REVOKE EXECUTE ON FUNCTION bb._statement_texts(text, text, text) FROM PUBLIC, db_client;") - if grant < 0 || revoke < 0 || revoke < grant { - t.Errorf("_statement_texts must be revoked from clients after the blanket grant (grant at %d, revoke at %d)", grant, revoke) + // up with a harmless one's text. It is SECURITY DEFINER and not on the list of + // what clients may call, which is what seals it (sealing_test.go). + if strings.Contains(ledgerSQL, "'_statement_texts'") { + t.Error("_statement_texts must not be client-callable") } if regexp.MustCompile(`(?i)grant[^;]*bb\.statement_cursor[^;]*to`).MatchString(ledgerSQL) { t.Error("clients must not be granted anything on bb.statement_cursor") From a7f45c295309c32e2e023ba7502b46ccf3199969 Mon Sep 17 00:00:00 2001 From: SauravYadav12 Date: Tue, 10 Nov 2026 15:06:34 +0530 Subject: [PATCH 22/22] fix(security): refuse a replaced Blackbox trigger ALTER and DROP on the data guards were refused, but CREATE OR REPLACE TRIGGER on a bb_ name was not, and the recorder skipped it by name, so anyone holding TRIGGER on a table could silently swap a guard for a no-op. A trigger with a bb_ name on a user's table must now match what the Blackbox installs (tgfoid, tgtype, tgattr, tgqual), or it is refused unless the override is in force, and the attempt is recorded BLOCKED. The engine's own install matches and stays unrecorded. A replacement under the override is recorded as a change. Co-Authored-By: Claude Opus 5.5 --- internal/ledger/blockedconn_test.go | 4 +- internal/ledger/datachanges.sql | 31 +++++++++++++-- internal/ledger/guardtrigger_test.go | 51 ++++++++++++++++++++++++ internal/ledger/ledger.sql | 59 +++++++++++++++++++++++++++- internal/ledger/sealing_test.go | 5 +-- 5 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 internal/ledger/guardtrigger_test.go diff --git a/internal/ledger/blockedconn_test.go b/internal/ledger/blockedconn_test.go index 24c4a413..6b26c5cd 100644 --- a/internal/ledger/blockedconn_test.go +++ b/internal/ledger/blockedconn_test.go @@ -37,8 +37,8 @@ func TestBlockedRecordConnectionCarriesTimeouts(t *testing.T) { t.Errorf("%s has %d direct dblink_exec call(s); the blocked path belongs in bb._record_blocked", what, n) } } - if n := strings.Count(SchemaData, "PERFORM bb._record_blocked(format("); n != 3 { - t.Errorf("datachanges.sql records %d blocked paths through the helper, want 3 (TRUNCATE, unqualified DML, impact)", n) + if n := strings.Count(SchemaData, "PERFORM bb._record_blocked(format("); n != 4 { + t.Errorf("datachanges.sql records %d blocked paths through the helper, want 4 (TRUNCATE, unqualified DML, impact, guard trigger)", n) } } diff --git a/internal/ledger/datachanges.sql b/internal/ledger/datachanges.sql index e482e010..2b0da7d7 100644 --- a/internal/ledger/datachanges.sql +++ b/internal/ledger/datachanges.sql @@ -24,8 +24,8 @@ -- transition tables, which would copy every changed row of every statement. -- -- The triggers are attached to every existing table here, and to each new one --- by an event trigger (bb_data_attach). Dropping, disabling or renaming one is --- refused unless the guardrail override is in force (a superuser or a db_admin +-- by an event trigger (bb_data_attach). Dropping, disabling, renaming or replacing +-- one is refused unless the guardrail override is in force (a superuser or a db_admin -- member with bb.allow_destructive=on). A superuser can still switch triggers -- off for a session; the Blackbox has always had that limit. @@ -466,7 +466,7 @@ $$; -- the table's owner may set it without being a superuser. CREATE OR REPLACE FUNCTION bb.data_protect_end() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, bb AS $$ -DECLARE r record; bad text; +DECLARE r record; bad text; c record; BEGIN FOR r IN SELECT * FROM pg_event_trigger_ddl_commands() LOOP IF r.object_type NOT IN ('table','trigger') OR r.schema_name IN ('bb','bb_f1','pg_catalog','information_schema') @@ -478,6 +478,31 @@ BEGIN PERFORM bb._attach_data_triggers(r.objid::regclass); END IF; END IF; + -- A trigger named bb_something on a user's table, in any form but the one + -- the Blackbox installs. CREATE OR REPLACE TRIGGER is how a guard gets + -- replaced while it keeps its name, and whoever holds TRIGGER on a table + -- may run it. Recorded like any other refusal, so the attempt outlives the + -- rollback. The engine's own install matches the definition and passes. + IF r.command_tag = 'CREATE TRIGGER' AND r.object_type = 'trigger' AND NOT bb._override_in_force() THEN + SELECT r.object_identity INTO bad + FROM pg_trigger t JOIN pg_class k ON k.oid = t.tgrelid JOIN pg_namespace n ON n.oid = k.relnamespace + WHERE t.oid = r.objid AND t.tgname LIKE 'bb\_%' AND n.nspname NOT IN ('bb','bb_f1') + AND NOT bb._is_own_trigger(t.oid); + IF bad IS NOT NULL THEN + SELECT * INTO c FROM bb._ctx(); + IF bb._holds_chain_lock() THEN + RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; + ELSE + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger + (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) + VALUES (%L,%L,%L,%L,%L,'CREATE TRIGGER','trigger',%L,%L,'BLOCKED','policy')$f$, + c.actor, c.actor_kind, c.tool, c.session, c.branch, bad, current_query())); + END IF; + RAISE EXCEPTION 'guardrail: bb_ trigger names are the Blackbox''s own, and % cannot be created or replaced in this form', bad + USING ERRCODE = 'insufficient_privilege', + HINT = 'Name the trigger something else. Only superusers and members of db_admin may, with bb.allow_destructive=on.'; + END IF; + END IF; -- ALTER TABLE … DISABLE TRIGGER or ENABLE REPLICA TRIGGER, or ALTER -- TRIGGER … RENAME, on one of ours. ENABLE ALWAYS stays allowed: it fires -- in more sessions, not fewer. diff --git a/internal/ledger/guardtrigger_test.go b/internal/ledger/guardtrigger_test.go new file mode 100644 index 00000000..6eedd781 --- /dev/null +++ b/internal/ledger/guardtrigger_test.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "strings" + "testing" +) + +// CREATE OR REPLACE TRIGGER keeps a guard's name and changes what it does, so +// a guard is compared with what the Blackbox installs, not by name. The +// behaviour is proved against a real server; this guards the shape of it. +func TestGuardTriggersCannotBeReplaced(t *testing.T) { + for _, want := range []string{ + "g.tgfoid = to_regprocedure('bb.guard_truncate()') AND g.tgtype = 34", + "g.tgfoid = to_regprocedure('bb.record_dml()') AND g.tgtype = 24", + "g.tgfoid = to_regprocedure('bb.guard_dml()') AND g.tgtype = 26", + "g.tgfoid = to_regprocedure('bb.guard_impact()') AND g.tgtype = 27", + "g.tgfoid = to_regprocedure('bb.track_row()') AND g.tgtype = 23", + // A guard narrowed to one column, or to a condition, is no guard. + "g.tgattr::text <> ''", + "bb._normal_when(bb._impact_when())", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } + for _, want := range []string{ + "r.command_tag = 'CREATE TRIGGER' AND r.object_type = 'trigger' AND NOT bb._override_in_force()", + "AND NOT bb._is_own_trigger(t.oid);", + "t.tgname LIKE 'bb\\_%' AND n.nspname NOT IN ('bb','bb_f1')", + "'CREATE TRIGGER','trigger',%L,%L,'BLOCKED','policy'", + } { + if !strings.Contains(SchemaData, want) { + t.Errorf("datachanges.sql no longer contains %q", want) + } + } +} + +// The recorder's skip list goes by name, so it must also ask for the +// definition, or a replaced guard is passed over as plumbing. +func TestRecorderSkipsOnlyTheInstalledGuards(t *testing.T) { + if !strings.Contains(Schema, "guard_impact) on '\n AND bb._is_own_trigger(r.objid) THEN CONTINUE;") { + t.Error("log_ddl_end skips a trigger by its name alone") + } + // The check is defined in ledger.sql beside its caller, for a recorder + // upgraded before datachanges.sql has been re-run. + if strings.Index(Schema, "FUNCTION bb._is_own_trigger") > strings.Index(Schema, "FUNCTION bb.log_ddl_end") { + t.Error("bb._is_own_trigger must be defined before bb.log_ddl_end") + } +} diff --git a/internal/ledger/ledger.sql b/internal/ledger/ledger.sql index d6afcb73..a16be1a3 100644 --- a/internal/ledger/ledger.sql +++ b/internal/ledger/ledger.sql @@ -629,6 +629,59 @@ BEGIN END; $$; +-- Is this trigger, named bb_something, exactly what the Blackbox installs under +-- that name? A trigger's name is all the recorder's skip list and the data +-- guards go by, and CREATE OR REPLACE TRIGGER keeps the name while changing what +-- it does, so the definition is compared instead: the function, when it fires +-- (tgtype), no column list (tgattr), and the WHEN clause (tgqual). The WHEN is +-- compared as Postgres prints it, with parentheses, casts and spaces taken +-- away from both sides, since the same expression prints differently between +-- major versions. +CREATE OR REPLACE FUNCTION bb._normal_when(e text) RETURNS text +LANGUAGE sql IMMUTABLE AS $$ + SELECT regexp_replace(lower(e), '::[a-z_ ]+(\[\])?|[()[:space:]]', '', 'g'); +$$; + +CREATE OR REPLACE FUNCTION bb._is_own_trigger(t oid) RETURNS boolean +LANGUAGE plpgsql STABLE SET search_path = pg_catalog, public, bb AS $$ +DECLARE g pg_trigger; a text[]; +BEGIN + SELECT * INTO g FROM pg_trigger WHERE oid = t; + IF NOT FOUND THEN + RETURN false; + END IF; + -- UPDATE OF a column nobody writes, a transition table, or a constraint trigger. + IF g.tgattr::text <> '' OR g.tgconstraint <> 0 OR g.tgdeferrable + OR g.tgoldtable IS NOT NULL OR g.tgnewtable IS NOT NULL THEN + RETURN false; + END IF; + CASE g.tgname + WHEN 'bb_guard_truncate' THEN -- BEFORE TRUNCATE, per statement + RETURN g.tgfoid = to_regprocedure('bb.guard_truncate()') AND g.tgtype = 34 AND g.tgqual IS NULL AND g.tgnargs = 0; + WHEN 'bb_record_dml' THEN -- AFTER UPDATE OR DELETE, per statement + RETURN g.tgfoid = to_regprocedure('bb.record_dml()') AND g.tgtype = 24 AND g.tgqual IS NULL AND g.tgnargs = 0; + WHEN 'bb_guard_dml' THEN -- BEFORE UPDATE OR DELETE, per statement + RETURN g.tgfoid = to_regprocedure('bb.guard_dml()') AND g.tgtype = 26 AND g.tgqual IS NULL AND g.tgnargs = 0; + WHEN 'bb_guard_impact' THEN -- BEFORE UPDATE OR DELETE, per row, with bb._impact_when + RETURN g.tgfoid = to_regprocedure('bb.guard_impact()') AND g.tgtype = 27 AND g.tgnargs = 0 + AND bb._normal_when(substring(pg_get_triggerdef(t) FROM ' WHEN \((.*)\) EXECUTE ')) = bb._normal_when(bb._impact_when()); + WHEN 'bb_track' THEN -- BEFORE INSERT OR UPDATE, per row + IF NOT (g.tgfoid = to_regprocedure('bb.track_row()') AND g.tgtype = 23 AND g.tgqual IS NULL AND g.tgnargs IN (3, 4)) THEN + RETURN false; + END IF; + -- Which columns are the owner's choice, but they must be columns: + -- arguments naming nothing would leave a tracking trigger that writes nothing. + a := string_to_array(encode(g.tgargs, 'escape'), '\000'); + RETURN EXISTS (SELECT 1 FROM unnest(a) x WHERE x <> '') + AND NOT EXISTS (SELECT 1 FROM unnest(a) x WHERE x <> '' + AND NOT EXISTS (SELECT 1 FROM pg_attribute + WHERE attrelid = g.tgrelid AND attname = x AND attnum > 0 AND NOT attisdropped)); + ELSE + RETURN false; + END CASE; +END; +$$; + -- ddl_command_end: record CREATE/ALTER changes (drops are recorded in sql_drop). CREATE OR REPLACE FUNCTION bb.log_ddl_end() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, bb AS $$ @@ -652,8 +705,10 @@ BEGIN IF r.command_tag IN ('CREATE SEQUENCE','ALTER SEQUENCE') THEN CONTINUE; END IF; IF r.command_tag = 'CREATE INDEX' AND r.object_identity ~ '_pkey$' THEN CONTINUE; END IF; -- The Blackbox's own data-change triggers (datachanges.sql), attached to - -- every table: plumbing, like the objects in schema bb. - IF r.command_tag = 'CREATE TRIGGER' AND r.object_identity ~ '^bb_(guard_truncate|record_dml|guard_dml|guard_impact) on ' THEN CONTINUE; END IF; + -- every table: plumbing, like the objects in schema bb. Only as installed: + -- one replaced under the override is a change, and is recorded. + IF r.command_tag = 'CREATE TRIGGER' AND r.object_identity ~ '^bb_(guard_truncate|record_dml|guard_dml|guard_impact) on ' + AND bb._is_own_trigger(r.objid) THEN CONTINUE; END IF; st := 'APPLIED'; rk := NULL; IF r.command_tag = 'ALTER TABLE' AND EXISTS (SELECT 1 FROM unnest(t) x WHERE x ~* '\malter\M[^;]*\mtype\M') THEN st := 'FLAGGED'; rk := 'type-change'; diff --git a/internal/ledger/sealing_test.go b/internal/ledger/sealing_test.go index 255ef75f..2f23a927 100644 --- a/internal/ledger/sealing_test.go +++ b/internal/ledger/sealing_test.go @@ -73,9 +73,8 @@ func TestClientCalledFunctionsAreOnTheList(t *testing.T) { } } -// Every SECURITY DEFINER function the free schemas define is either on the -// list, named to be sealed, or simply not; what must hold is that the list -// does not grow by accident to include a definer that writes the ledger. +// bb.act_for is the one SECURITY DEFINER function a client may call. The list +// must not grow, by accident, to include another that writes the ledger. func TestOnlyActForIsAClientCallableDefiner(t *testing.T) { allowed := allowedCallable(t) def := regexp.MustCompile(`(?is)FUNCTION bb\.([a-z0-9_]+)\([^)]*\)[^;$]*?SECURITY DEFINER`)