diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b713c9eb..44412948 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,9 @@ jobs: # occasional impossible value rather than a failing test. - name: Test (race detector) run: go test -race ./... - # The release workflows share one SHA256SUMS; this checks the script that - # merges into it never drops another job's entries. - - name: Release checksum merging + # Each release workflow signs only its own build's checksums; this checks + # the script never signs anything read back from a release. + - name: Release checksum signing run: bash scripts/test_release_checksums.sh # The release job's check that every file `fox update` needs was published. - name: Release assets for fox update diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ebd90dcd..c962cfed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -285,11 +285,10 @@ jobs: # only integrity guarantee is TLS, and these assets are executed as root # inside the user's distro. # - # SHA256SUMS is shared with wsl-distro.yml, which runs on the same tag and - # can finish before or after this job. The script merges this job's entries - # into the published file by exact name, keeps the other job's, and checks - # again after uploading — so neither job can drop the other's checksums - # (v0.8.1 lost the distro image's entry that way). + # SHA256SUMS belongs to this job alone and lists only what it built here; + # wsl-distro.yml, on its own schedule, signs SHA256SUMS.distro for the + # distro image. Merging them meant signing lines read back from the + # release, so anyone able to upload an asset could get a hash signed. - name: Publish checksums env: GH_TOKEN: ${{ github.token }} @@ -298,7 +297,7 @@ jobs: FOX_RELEASE_SIGNING_KEY: ${{ secrets.FOX_RELEASE_SIGNING_KEY }} run: | cd dist - bash ../scripts/release-checksums.sh "$TAG" fox-* foxbyte-docker-context.tar.gz foxbyte-sbom.spdx.json + bash ../scripts/release-checksums.sh "$TAG" SHA256SUMS fox-* foxbyte-docker-context.tar.gz foxbyte-sbom.spdx.json - name: Attach binaries env: diff --git a/.github/workflows/wsl-distro.yml b/.github/workflows/wsl-distro.yml index a9309796..3db24a6e 100644 --- a/.github/workflows/wsl-distro.yml +++ b/.github/workflows/wsl-distro.yml @@ -73,10 +73,9 @@ jobs: gh release view "$TAG" >/dev/null 2>&1 || gh release create "$TAG" --prerelease --generate-notes --title "$TAG" - # SHA256SUMS is signed with the release key (audit v2 G22): fox update and - # install.sh refuse a release whose signature does not check. A missing - # secret fails the release here, rather than publishing one nobody can - # install. (cmd/releasesign also refuses a key fox would not accept.) + # This job's checksums are signed with the release key (audit v2 G22). A + # missing secret fails the job here, rather than publishing an image with + # no signature. (cmd/releasesign also refuses a key fox would not accept.) - name: Build the release signer env: FOX_RELEASE_SIGNING_KEY: ${{ secrets.FOX_RELEASE_SIGNING_KEY }} @@ -84,11 +83,11 @@ jobs: test -n "$FOX_RELEASE_SIGNING_KEY" || { echo "::error::the FOX_RELEASE_SIGNING_KEY secret is not set (see cmd/releasesign)"; exit 1; } go build -o "$RUNNER_TEMP/releasesign" ./cmd/releasesign - # Merged, not replaced: release.yml publishes sums for the binaries into - # the same SHA256SUMS. Every asset the installer or setup downloads must be - # listed, or verification silently skips it. The shared script matches - # entries by exact name and re-checks after uploading, so whichever job - # finishes last keeps the other's entries. + # A signed file of this job's own, because this workflow can run at any + # time for an existing tag and release.yml owns SHA256SUMS. Only the hash + # of the image built above is signed; nothing is read back from the + # release. install.ps1 looks the image up here, then in SHA256SUMS for + # releases made before this file existed. - name: Publish checksums env: GH_TOKEN: ${{ github.token }} @@ -97,7 +96,7 @@ jobs: FOX_RELEASE_SIGNING_KEY: ${{ secrets.FOX_RELEASE_SIGNING_KEY }} run: | cd dist - bash ../scripts/release-checksums.sh "$TAG" foxbyte-distro.tar.zst + bash ../scripts/release-checksums.sh "$TAG" SHA256SUMS.distro foxbyte-distro.tar.zst - name: Attach the distro image env: diff --git a/.github/workflows/wsl-zfs.yml b/.github/workflows/wsl-zfs.yml index 0495400d..3c88dc02 100644 --- a/.github/workflows/wsl-zfs.yml +++ b/.github/workflows/wsl-zfs.yml @@ -59,12 +59,11 @@ jobs: set -eux gh release upload "$TAG" dist/foxbyte-zfs-modules-*.tar.gz --clobber - # Keep SHA256SUMS complete, or setup skips verifying this bundle. + # A checksum beside each bundle, not in SHA256SUMS: that file is signed + # and belongs to release.yml, and rewriting it here without the key + # broke its signature, so fox update then refused the whole release. cd dist - gh release download "$TAG" -p SHA256SUMS -O existing.txt 2>/dev/null || : > existing.txt - sha256sum foxbyte-zfs-modules-*.tar.gz > mine.txt - awk '{print $2}' mine.txt > names.txt - grep -vFf names.txt existing.txt > SHA256SUMS || : > SHA256SUMS - cat mine.txt >> SHA256SUMS - sort -k2 -o SHA256SUMS SHA256SUMS - gh release upload "$TAG" SHA256SUMS --clobber + for f in foxbyte-zfs-modules-*.tar.gz; do + sha256sum "$f" > "$f.sha256" + gh release upload "$TAG" "$f.sha256" --clobber + done diff --git a/README.md b/README.md index 21cc1700..fcac7904 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,9 @@ Every release's `SHA256SUMS` is signed with the FoxByte release key it carries — which comes from this repository, not from the release — wherever `openssl` can do Ed25519 (OpenSSL 3; macOS's LibreSSL cannot, and the installer says so; `FOX_REQUIRE_SIGNATURE=1` refuses instead). `fox update` checks it on -every update and never installs a release that fails. Each file also has a +every update, `fox setup` on the engine it downloads, and neither installs a +release that fails. The Windows distro image, built by its own workflow, has +its own signed `SHA256SUMS.distro`. Each file also has a GitHub build attestation and the release an SBOM (`foxbyte-sbom.spdx.json`): ```bash diff --git a/cmd/fox/main.go b/cmd/fox/main.go index 3556160c..12e2755f 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -696,6 +696,12 @@ func pipelineCmd(args []string) { if target == "" { target = "pl-" + strings.TrimSuffix(filepath.Base(specPath), filepath.Ext(specPath)) } + must(branch.CheckNewName(target)) + // A run deletes and re-creates its target. A branch an account owns is not + // this command's to replace; deleting it first is a decision made out loud. + if branch.Exists(target) && accountOwnsBranch(target) { + must(fmt.Errorf("the branch %q belongs to an account: delete it first, or choose another name with --as", target)) + } res, err := branch.RunPipeline(&branch.Progress{Log: os.Stdout}, spec, target) must(err) if res.Failed { @@ -784,6 +790,7 @@ func branchCmd(args []string) { fmt.Println("usage: fox branch create [--from ] [--masked]") os.Exit(2) } + must(branch.CheckNewName(name)) started := time.Now() if hasFlag(args[1:], "--masked") { maskedBranchCreate(name, optValue(args[1:], "--from"), started) @@ -1482,6 +1489,18 @@ func forgetOwner(name string) { } } +// accountOwnsBranch reports whether an account is recorded as a branch's owner. +// Best-effort, like forgetOwner: with no store there are no accounts. +func accountOwnsBranch(name string) bool { + s, err := auth.OpenFromEnv() + if err != nil { + return false + } + defer s.Close() + _, owned := s.BranchOwner(name) + return owned +} + // userCmd is `fox user`: accounts, for whoever runs the engine (audit v2 G16). func userCmd(args []string) { usage := func() { diff --git a/deploy/install.Tests.ps1 b/deploy/install.Tests.ps1 index 97190f45..e8050a4d 100644 --- a/deploy/install.Tests.ps1 +++ b/deploy/install.Tests.ps1 @@ -86,6 +86,7 @@ Describe 'FoxByte release verification' { BeforeEach { $env:FOX_NO_VERIFY = $null Set-Variable -Name FoxSums -Scope Script -Value $script:listing + Set-Variable -Name FoxDistroSums -Scope Script -Value '' } AfterAll { $env:FOX_NO_VERIFY = $null } @@ -97,6 +98,25 @@ Describe 'FoxByte release verification' { Get-FoxChecksum 'foxbyte-distro.tar.gz' | Should -Be ('b' * 64) } + It 'reads the distro image from its own listing' { + Set-Variable -Name FoxDistroSums -Scope Script -Value (('c' * 64) + " foxbyte-distro.tar.zst`n") + Get-FoxChecksum 'foxbyte-distro.tar.zst' | Should -Be ('c' * 64) + } + + It 'prefers the distro listing over an older entry in SHA256SUMS' { + Set-Variable -Name FoxDistroSums -Scope Script -Value (('c' * 64) + " foxbyte-distro.tar.gz`n") + Get-FoxChecksum 'foxbyte-distro.tar.gz' | Should -Be ('c' * 64) + } + + It 'never lets the distro listing answer for another file' { + Set-Variable -Name FoxDistroSums -Scope Script -Value (('c' * 64) + " fox-windows-amd64.exe`n") + Get-FoxChecksum 'fox-windows-amd64.exe' | Should -Be ('a' * 64) + } + + It 'refuses a distro image neither listing names' { + { Get-FoxChecksum 'foxbyte-distro.tar.zst' } | Should -Throw '*not listed in SHA256SUMS*' + } + It 'refuses an asset the listing does not name' { { Get-FoxChecksum 'fox-linux-amd64' } | Should -Throw '*not listed in SHA256SUMS*' } diff --git a/deploy/install.ps1 b/deploy/install.ps1 index 4025a09a..47c562a4 100644 --- a/deploy/install.ps1 +++ b/deploy/install.ps1 @@ -280,7 +280,12 @@ function Assert-UpstreamChecksum([string]$Path, [string]$SumsUrl, [string]$Name) # download), not authorship -- signatures would be needed for that. The files # below are executed as root inside the distro, so an unverifiable one is not # installed. FOX_NO_VERIFY=1 skips the check deliberately. +# +# The distro image is built by its own workflow, on its own schedule, so its +# checksum is in SHA256SUMS.distro. Releases made before that file existed list +# the image in SHA256SUMS, which is why both are looked at for it, and only for it. $script:FoxSums = $null +$script:FoxDistroSums = $null function Get-FoxSums { if ($env:FOX_NO_VERIFY -eq '1') { return $null } @@ -295,16 +300,43 @@ function Get-FoxSums { return $script:FoxSums } +# Get-FoxDistroSums returns the distro image's own listing, or '' for a release +# without one (the distro workflow has not finished, or the release predates it). +function Get-FoxDistroSums { + if ($null -ne $script:FoxDistroSums) { return $script:FoxDistroSums } + try { + $timeout = $script:WebTimeout + $body = (Invoke-WebRequest -UseBasicParsing -Uri (Get-FoxAsset 'SHA256SUMS.distro') @timeout).Content + $script:FoxDistroSums = if ($body -is [byte[]]) { [System.Text.Encoding]::UTF8.GetString($body) } else { [string]$body } + } catch { + $script:FoxDistroSums = '' + } + return $script:FoxDistroSums +} + +# Find-FoxChecksum returns the SHA256 a listing gives for $Name, or $null. +function Find-FoxChecksum([string]$Listing, [string]$Name) { + foreach ($line in ($Listing -split "`n")) { + if ($line -match '^([0-9a-fA-F]{64})\s+\*?(.+?)\s*$' -and $Matches[2] -eq $Name) { + return $Matches[1].ToLower() + } + } + return $null +} + # Get-FoxChecksum returns the expected SHA256 for a release asset, or $null when -# verification is switched off. +# verification is switched off. The distro's own listing goes first, so a distro +# rebuilt for an older release is checked against its current hash; it can never +# answer for any other file. function Get-FoxChecksum([string]$Name) { $sums = Get-FoxSums if ($null -eq $sums) { return $null } - foreach ($line in ($sums -split "`n")) { - if ($line -match '^([0-9a-fA-F]{64})\s+\*?(.+?)\s*$' -and $Matches[2] -eq $Name) { - return $Matches[1].ToLower() - } + if ($Name -like 'foxbyte-distro.*') { + $want = Find-FoxChecksum (Get-FoxDistroSums) $Name + if ($want) { return $want } } + $want = Find-FoxChecksum $sums $Name + if ($want) { return $want } throw "$Name is not listed in SHA256SUMS for $Version -- nothing was installed.`nSet FOX_NO_VERIFY=1 to install without checking (not recommended)." } diff --git a/docs/windows-setup.md b/docs/windows-setup.md index 772c586e..cba0a1c0 100644 --- a/docs/windows-setup.md +++ b/docs/windows-setup.md @@ -127,7 +127,7 @@ costs one ~2 MB bundle rather than another 740 MB image. **To cover a new kernel on an existing release**, run the **wsl-zfs** workflow manually with the kernel tag and the release tag. It builds just that bundle -(~1 hour), attaches it, and updates `SHA256SUMS` — no full rebuild. Add the +(~1 hour) and attaches it with a `.sha256` file beside it — no full rebuild. Add the kernel to the matrix as well, so future releases keep covering it. ## Building the ZFS bundle (maintainers) diff --git a/enterprise/auditevidence/evidence.go b/enterprise/auditevidence/evidence.go index aa6145ae..204f8c3e 100644 --- a/enterprise/auditevidence/evidence.go +++ b/enterprise/auditevidence/evidence.go @@ -34,7 +34,8 @@ type inputs struct { keyID string forward *branch.ForwardStatus findings []branch.AuditFinding - records []branch.AuditRecord + activity branch.EvidenceActivity + records []branch.AuditRecord // only when they were asked for anchorSet []ledger.Anchor publicKey string } @@ -85,7 +86,7 @@ func gather(o branch.EvidenceOptions) (inputs, error) { } } in.findings = branch.AuditFindings(marks) - if in.records, err = branch.AuditRecordsBetween(o.Since, o.Until); err != nil { + if in.activity, in.records, err = branch.AuditActivityBetween(o.Since, o.Until, o.Records); err != nil { return in, err } if o.Records { @@ -108,7 +109,7 @@ func assemble(o branch.EvidenceOptions, in inputs) branch.AuditEvidence { }, Integrity: branch.EvidenceIntegrity{ChainReport: in.chain, AnchorsKept: in.anchors, KeyID: in.keyID}, Delivery: branch.EvidenceDelivery{Forward: in.forward, Findings: in.findings}, - Activity: branch.EvidenceActivity{ByKind: map[string]int{}, ByBranch: map[string]int{}, Gaps: []branch.AuditRecord{}}, + Activity: in.activity, Changes: []auth.SecurityEvent{}, Limits: branch.EvidenceLimits, } @@ -121,19 +122,6 @@ func assemble(o branch.EvidenceOptions, in inputs) branch.AuditEvidence { if ev.Delivery.Findings == nil { ev.Delivery.Findings = []branch.AuditFinding{} } - for _, r := range in.records { - ev.Activity.Records++ - ev.Activity.ByKind[r.Kind]++ - ev.Activity.ByBranch[r.Branch]++ - switch r.Kind { - case "outcome": - ev.Activity.Errors++ - case "heartbeat": - ev.Activity.Heartbeats++ - case "gap": - ev.Activity.Gaps = append(ev.Activity.Gaps, r) - } - } for _, e := range o.Events { if !strings.HasPrefix(e.Kind, "audit-log.") { continue diff --git a/enterprise/auditevidence/evidence_test.go b/enterprise/auditevidence/evidence_test.go index e0a85110..06f4623a 100644 --- a/enterprise/auditevidence/evidence_test.go +++ b/enterprise/auditevidence/evidence_test.go @@ -23,6 +23,10 @@ func TestAssemble(t *testing.T) { {Seq: 1, Kind: "statement", Branch: "main"}, {Seq: 2, Kind: "outcome", Branch: "main"}, {Seq: 3, Kind: "heartbeat", Branch: "dev"}, {Seq: 4, Kind: "gap", Branch: "dev", Missing: 2}, }} + in.activity = branch.NewAuditActivity() + for _, r := range in.records { + branch.TallyAuditActivity(&in.activity, r) + } o := branch.EvidenceOptions{Since: since, Until: until, By: "ada@example.com", Events: []auth.SecurityEvent{ {Kind: auth.EvAuditLogOn, At: "2026-11-02T10:00:00Z", Actor: "ada@example.com", IP: "10.0.0.9"}, {Kind: auth.EvAuditMarked, At: "2026-10-30T10:00:00Z", Subject: "main", Detail: "auditing writes of public.t"}, // before the period diff --git a/enterprise/promote/promote.go b/enterprise/promote/promote.go index 227102cc..03afd2e8 100644 --- a/enterprise/promote/promote.go +++ b/enterprise/promote/promote.go @@ -10,8 +10,14 @@ package promote import ( + "context" + "errors" "fmt" "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/thefoxbyte/foxbyte/internal/auth" "github.com/thefoxbyte/foxbyte/internal/branch" @@ -126,6 +132,9 @@ func build(source, target string, history []auth.ChangeRequest) (entries []branc return nil, 0, fmt.Errorf("%w: %q has no schema change that %q does not already have", branch.ErrNothingToPromote, source, target) } + if err := checkReplayable(entries); err != nil { + return nil, 0, err + } xids, err := runXids(source, forkAfterID) if err != nil { return nil, 0, fmt.Errorf("reading %q's Blackbox: %w", source, err) @@ -215,10 +224,23 @@ func checkPolicyFor(target string, entries []branch.RequestEntry) error { // and the approver is recorded beside it: the Blackbox entry's session says which // request applied it, and the request says who approved it. One transaction means // the target is never left half-merged — either every statement is in, or none is. +// +// It logs in as the shared client role, over pgx. Ownership is why that role: +// applied as the superuser, a promoted table belongs to the admin role, and the +// next `ALTER TABLE` through the gateway fails with "must be owner of table", so +// running as what the gateway runs every client as (proxy.realUser) gives the +// object exactly the owner it has on the source. Logging in as it, rather than +// SET ROLE from a superuser's session, is why nothing in a statement can get back +// out. The Blackbox's attribution is unaffected: for a session logged in as this +// role it reads bb.actor, as it did for the superuser. func apply(requestID int64, target, approver string, entries []branch.RequestEntry) (int, error) { if len(entries) == 0 { return 0, branch.ErrNothingToPromote } + // Again here: what is applied is a stored snapshot, and may predate this check. + if err := checkReplayable(entries); err != nil { + return 0, err + } target, err := branch.ResolveBranch(target) if err != nil { return 0, err @@ -226,42 +248,96 @@ func apply(requestID int64, target, approver string, entries []branch.RequestEnt if _, err := branch.EnsureRunning(target); err != nil { return 0, err } - if err := branch.ExecScript(target, applyScript(requestID, approver, entries)); err != nil { + ctx := context.Background() + openCtx, cancel := context.WithTimeout(ctx, openTimeout) + defer cancel() + conn, err := branch.OpenClient(openCtx, target) + if err != nil { return 0, fmt.Errorf("applying the request to %q (nothing was committed): %w", target, err) } + defer conn.Close(ctx) + + replayed := false + err = pgx.BeginFunc(ctx, conn, func(tx pgx.Tx) error { + inTx := func() bool { return conn.PgConn().TxStatus() != 'I' } + if err := replay(ctx, tx, inTx, requestID, approver, entries); err != nil { + return err + } + replayed = true + return nil + }) + if err != nil { + return 0, applyFailure(target, replayed, err) + } return len(entries), nil } -// applyScript is the one transaction a request is applied in. -func applyScript(requestID int64, approver string, entries []branch.RequestEntry) string { - var b strings.Builder - b.WriteString("BEGIN;\n") +// openTimeout bounds the login. The statements themselves have no limit: an ALTER +// on a large table has always been allowed to take what it takes. +const openTimeout = 30 * time.Second + +// errTransactionEnded is a statement having ended the transaction it was given, +// which checkReplayable exists to prevent. If one ever gets past that, the apply +// stops there and says so, rather than report a rollback that did not happen. +var errTransactionEnded = errors.New("the statement ended the transaction it was applied in") + +// applyFailure says what a failed apply left behind. Every statement's error +// comes before the commit and rolls the transaction back, and a commit the server +// refused did too. Only a commit whose answer never arrived leaves it open. +func applyFailure(target string, replayed bool, err error) error { + var pgErr *pgconn.PgError + switch { + case errors.Is(err, errTransactionEnded): + return fmt.Errorf("applying the request to %q: part of it may be committed; compare the target with its Blackbox: %w", target, err) + case replayed && !errors.As(err, &pgErr) && !errors.Is(err, pgx.ErrTxCommitRollback): + return fmt.Errorf("applying the request to %q: every statement ran but the commit was not confirmed, so it may or may not be in; "+ + "look for this request's session in the target's Blackbox: %w", target, err) + } + return fmt.Errorf("applying the request to %q (nothing was committed): %w", target, err) +} + +// execer is the one call replay needs of a transaction. +type execer interface { + Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) +} + +// replay runs a request's statements in tx, each under its own author. inTx says +// whether the connection is still in that transaction. +// +// A statement goes to the server as its own message, exactly as recorded and with +// no arguments, which pgx sends by the simple protocol: one message may hold +// several statements, and nothing but the server reads it, so a :name or a +// backslash in it is only text. The settings are bound values for the same +// reason; set_config(…, true) is SET LOCAL. +func replay(ctx context.Context, tx execer, inTx func() bool, requestID int64, approver string, entries []branch.RequestEntry) error { // Set once for the whole transaction: which request this was, and that the - // engine applied it rather than a client. - fmt.Fprintf(&b, "SET LOCAL bb.session = %s;\n", branch.QuoteLiteral(fmt.Sprintf("request-%d", requestID))) - fmt.Fprintf(&b, "SET LOCAL application_name = %s;\n", branch.QuoteLiteral(brand.CLI+" request approve (by "+approver+")")) - // As the shared client role, which is what the gateway runs every client as - // (proxy.realUser). Ownership is the reason: applied as the superuser, a - // promoted table belongs to the admin role, and the next `ALTER TABLE` through - // the gateway fails with "must be owner of table" — the change arrives and then - // cannot be worked on. Running as the same role the gateway uses gives the - // object exactly the owner it has on the source. - // - // session_user stays the superuser, so the Blackbox's attribution — which reads - // session_user and falls back to bb.actor — is unaffected. - fmt.Fprintf(&b, "SET LOCAL ROLE %s;\n", branch.QuoteIdent(branch.ClientRole)) + // engine applied it rather than a client. standard_conforming_strings because + // the check read the statements with it on, whatever the role's default is. + if _, err := tx.Exec(ctx, "SELECT set_config('standard_conforming_strings', 'on', true), "+ + "set_config('bb.session', $1, true), set_config('application_name', $2, true)", + fmt.Sprintf("request-%d", requestID), brand.CLI+" request approve (by "+approver+")"); err != nil { + return fmt.Errorf("setting up the transaction: %w", err) + } for _, e := range entries { if e.SameRun != 0 { continue // its statement ran with the entry it shares it with } // Per statement: the actor is whoever ran it on the source, so the target's // record names them and not the approver. - fmt.Fprintf(&b, "SET LOCAL bb.actor = %s;\n", branch.QuoteLiteral(e.Actor)) + if _, err := tx.Exec(ctx, "SELECT set_config('bb.actor', $1, true)", e.Actor); err != nil { + return fmt.Errorf("entry %d: %w", e.ID, err) + } if e.ActorKind != "" { - fmt.Fprintf(&b, "SET LOCAL bb.actor_kind = %s;\n", branch.QuoteLiteral(e.ActorKind)) + if _, err := tx.Exec(ctx, "SELECT set_config('bb.actor_kind', $1, true)", e.ActorKind); err != nil { + return fmt.Errorf("entry %d: %w", e.ID, err) + } + } + if _, err := tx.Exec(ctx, strings.TrimSpace(e.Statement)); err != nil { + return fmt.Errorf("entry %d: %w", e.ID, err) + } + if !inTx() { + return fmt.Errorf("entry %d: %w", e.ID, errTransactionEnded) } - b.WriteString(strings.TrimRight(strings.TrimSpace(e.Statement), ";") + ";\n") } - b.WriteString("RESET ROLE;\nCOMMIT;\n") - return b.String() + return nil } diff --git a/enterprise/promote/promote_test.go b/enterprise/promote/promote_test.go index 6f296bb0..9f2785d4 100644 --- a/enterprise/promote/promote_test.go +++ b/enterprise/promote/promote_test.go @@ -5,9 +5,15 @@ package promote import ( + "context" + "errors" + "os" "strings" "testing" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/thefoxbyte/foxbyte/internal/auth" "github.com/thefoxbyte/foxbyte/internal/branch" ) @@ -70,9 +76,41 @@ func TestBuildRequestChecksNames(t *testing.T) { } } +// recordingTx stands in for the transaction: what replay asked of it, in order. +type recordingTx struct { + calls []execCall + failOn string +} + +type execCall struct { + sql string + args []any +} + +func (r *recordingTx) Exec(_ context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + r.calls = append(r.calls, execCall{sql, args}) + if r.failOn != "" && strings.Contains(sql, r.failOn) { + return pgconn.CommandTag{}, errors.New("boom") + } + return pgconn.CommandTag{}, nil +} + +// The statements replay sent, without the settings between them. +func (r *recordingTx) statements() []string { + var out []string + for _, c := range r.calls { + if !strings.HasPrefix(c.sql, "SELECT set_config(") { + out = append(out, c.sql) + } + } + return out +} + +func alwaysInTx() bool { return true } + // One run of two statements is two entries, each holding the run's whole -// text: the script runs it once, at the first, under its author. -func TestApplyScriptRunsASharedStatementOnce(t *testing.T) { +// text: replay runs it once, at the first, under its author. +func TestReplayRunsASharedStatementOnce(t *testing.T) { run := "CREATE TABLE t (id int); CREATE FUNCTION t_add(i int) RETURNS void LANGUAGE sql AS $$ INSERT INTO t VALUES (i) $$" entries := []branch.RequestEntry{ {ID: 171, Actor: "ada@example.com", CommandTag: "CREATE TABLE", Statement: run}, @@ -80,14 +118,105 @@ func TestApplyScriptRunsASharedStatementOnce(t *testing.T) { {ID: 173, Actor: "bob@example.com", CommandTag: "CREATE INDEX", Statement: "CREATE INDEX ON t (id)"}, } branch.ReplayOnce(entries, map[int64]int64{171: 900, 172: 900, 173: 901}) - s := applyScript(5, "carol@example.com", entries) - if n := strings.Count(s, "CREATE TABLE t (id int)"); n != 1 { - t.Fatalf("the shared run is in the script %d times:\n%s", n, s) + tx := &recordingTx{} + if err := replay(context.Background(), tx, alwaysInTx, 5, "carol@example.com", entries); err != nil { + t.Fatal(err) + } + got := tx.statements() + if len(got) != 2 || got[0] != run || got[1] != "CREATE INDEX ON t (id)" { + t.Fatalf("statements = %q", got) + } + var actors []any + for _, c := range tx.calls { + if strings.Contains(c.sql, "'bb.actor'") { + actors = append(actors, c.args...) + } + } + if len(actors) != 2 || actors[0] != "ada@example.com" || actors[1] != "bob@example.com" { + t.Fatalf("actors = %v", actors) } - if !strings.Contains(s, "CREATE INDEX ON t (id);") || !strings.Contains(s, "SET LOCAL bb.actor = 'bob@example.com'") { - t.Fatalf("the entry of its own is missing:\n%s", s) +} + +// The apply path is not psql: each statement is one Exec, verbatim and with no +// arguments, so nothing but the server reads it. The transaction is the +// caller's, and replay issues none of its own control. +func TestReplayNeverGoesThroughPsql(t *testing.T) { + stmt := "COMMENT ON TABLE t IS E':var \\\\! rm -rf / \\\\gexec'" + tx := &recordingTx{} + if err := replay(context.Background(), tx, alwaysInTx, 7, "carol@example.com", + []branch.RequestEntry{{ID: 1, Actor: "ada@example.com", Statement: " " + stmt + "\n"}}); err != nil { + t.Fatal(err) } - if strings.Index(s, "CREATE TABLE t") > strings.Index(s, "CREATE INDEX") { - t.Fatalf("out of order:\n%s", s) + if got := tx.statements(); len(got) != 1 || got[0] != stmt { + t.Fatalf("the statement was changed on its way: %q", got) + } + for _, c := range tx.calls { + if c.sql == stmt && len(c.args) != 0 { + t.Fatalf("the statement was sent with arguments: %v", c.args) + } + for _, control := range []string{"BEGIN", "COMMIT", "SET LOCAL", "RESET ROLE", "SET ROLE"} { + if strings.Contains(c.sql, control) { + t.Fatalf("replay issued %s itself: %q", control, c.sql) + } + } + } + src, err := os.ReadFile("promote.go") + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(src), "ExecScript(") { + t.Fatal("the apply path goes through branch.ExecScript, which is psql over stdin") + } +} + +// The statement after the one that fails is not attempted, and the failure +// names the entry. +func TestReplayStopsAtTheFirstFailure(t *testing.T) { + tx := &recordingTx{failOn: "CREATE VIEW"} + err := replay(context.Background(), tx, alwaysInTx, 1, "c", []branch.RequestEntry{ + {ID: 1, Statement: "CREATE TABLE a ()"}, {ID: 2, Statement: "CREATE VIEW b AS SELECT 1"}, {ID: 3, Statement: "CREATE TABLE c ()"}, + }) + if err == nil || !strings.Contains(err.Error(), "entry 2") { + t.Fatalf("err = %v", err) + } + if got := tx.statements(); len(got) != 2 { + t.Fatalf("carried on after the failure: %q", got) + } +} + +// If a statement ever did end the transaction, the next must not run. +func TestReplayStopsIfTheTransactionEnded(t *testing.T) { + tx := &recordingTx{} + err := replay(context.Background(), tx, func() bool { return false }, 1, "c", []branch.RequestEntry{ + {ID: 1, Statement: "CREATE TABLE a ()"}, {ID: 2, Statement: "CREATE TABLE b ()"}, + }) + if !errors.Is(err, errTransactionEnded) { + t.Fatalf("err = %v", err) + } + if got := tx.statements(); len(got) != 1 { + t.Fatalf("carried on outside the transaction: %q", got) + } +} + +// What a failure says was committed has to be true. +func TestApplyFailureSaysWhatWasCommitted(t *testing.T) { + refused := &pgconn.PgError{Message: "deferred constraint"} + lost := errors.New("connection reset") + cases := []struct { + name string + replayed bool + err error + want string + }{ + {"a statement failed", false, lost, "nothing was committed"}, + {"the server refused the commit", true, refused, "nothing was committed"}, + {"the server rolled back at commit", true, pgx.ErrTxCommitRollback, "nothing was committed"}, + {"the commit's answer never came", true, lost, "may or may not be in"}, + {"a statement ended the transaction", false, errTransactionEnded, "part of it may be committed"}, + } + for _, c := range cases { + if got := applyFailure("main", c.replayed, c.err).Error(); !strings.Contains(got, c.want) { + t.Errorf("%s: %q does not say %q", c.name, got, c.want) + } } } diff --git a/enterprise/promote/replaycheck.go b/enterprise/promote/replaycheck.go new file mode 100644 index 00000000..9a399f67 --- /dev/null +++ b/enterprise/promote/replaycheck.go @@ -0,0 +1,517 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package promote + +import ( + "errors" + "fmt" + "strings" + + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/brand" +) + +// What a request may carry. +// +// An entry's statement is the source's whole current_query(): everything its +// author sent in one message, which can be more than the change it was recorded +// for. Replayed onto the target, a COMMIT in it ends the one transaction the +// request is applied in, and a SET ROLE or SET bb.actor changes who the rest +// runs as, or is recorded as. So a request is refused if any statement in any +// entry would do that: when it is asked for, and again when it is applied, +// because what is applied is a stored snapshot. +// +// This is the Go counterpart of bb._sql_statements, and stricter, because here a +// misread is a way in rather than a wrong risk label. It follows the server's +// own lexer where that function does not: a line comment also ends at a carriage +// return, an E'' string keeps its escapes across a newline-separated +// continuation, an identifier runs through any '$' in it, and a semicolon inside +// parentheses or a BEGIN ATOMIC body does not end the statement. Text it cannot +// read is refused, not passed. +// +// What it cannot see is SQL assembled at run time, or held in a function that the +// same message calls. Those run with the applying role's rights and nothing more: +// a COMMIT there is an error inside a transaction block, and the role has no +// other role to become (apply). + +type tokenKind int + +const ( + tokWord tokenKind = iota // a keyword or unquoted name, in lower case + tokQuoted // a "quoted name", in lower case: settings ignore case + tokEscaped // a U&"name" with escapes, whose meaning is not worked out + tokString // any string literal + tokPunct // one character of anything else +) + +type token struct { + kind tokenKind + text string + // plain is a string token whose text is exactly its value: no escapes to + // interpret. Set for '…' and $…$ only. + plain bool +} + +func (t token) is(kind tokenKind, text string) bool { return t.kind == kind && t.text == text } + +var errUnreadable = errors.New("its quoting or comments do not close") + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' +} +func isDigit(c byte) bool { return c >= '0' && c <= '9' } +func isTagStart(c byte) bool { return c == '_' || c >= 0x80 || lower(c) >= 'a' && lower(c) <= 'z' } +func isIdentCont(c byte) bool { return isTagStart(c) || isDigit(c) || c == '$' } + +func lower(c byte) byte { + if c >= 'A' && c <= 'Z' { + return c + 'a' - 'A' + } + return c +} + +// lexSQL splits a message into tokens as the server's scanner would. +func lexSQL(q string) ([]token, error) { + if strings.IndexByte(q, 0) >= 0 { + return nil, errors.New("it holds a NUL byte") + } + var out []token + n := len(q) + for i := 0; i < n; { + c := q[i] + switch { + case isSpace(c): + i++ + case strings.HasPrefix(q[i:], "--"): + i = lineEnd(q, i) + case strings.HasPrefix(q[i:], "/*"): + end, err := blockCommentEnd(q, i) + if err != nil { + return nil, err + } + i = end + case c == '\'': + t, end, err := lexString(q, i, false) + if err != nil { + return nil, err + } + t.plain = true + out, i = append(out, t), end + case c == '"': + t, end, err := lexQuoted(q, i) + if err != nil { + return nil, err + } + out, i = append(out, t), end + case c == '$': + t, end, err := lexDollar(q, i) + if err != nil { + return nil, err + } + out, i = append(out, t), end + case isDigit(c): + for i < n && isDigit(q[i]) { + i++ + } + case isTagStart(c): + t, end, err := lexWord(q, i) + if err != nil { + return nil, err + } + out, i = append(out, t), end + default: + out = append(out, token{kind: tokPunct, text: string(c)}) + i++ + } + } + return out, nil +} + +func lineEnd(q string, i int) int { + for i < len(q) && q[i] != '\n' && q[i] != '\r' { + i++ + } + return i +} + +// blockCommentEnd finds the end of a /* comment */, which nests. +func blockCommentEnd(q string, i int) (int, error) { + depth := 1 + for i += 2; i < len(q); { + switch { + case strings.HasPrefix(q[i:], "/*"): + depth++ + i += 2 + case strings.HasPrefix(q[i:], "*/"): + depth-- + i += 2 + if depth == 0 { + return i, nil + } + default: + i++ + } + } + return 0, errUnreadable +} + +// lexWord reads a name or keyword, and the literal it prefixes if it is one: +// the name is taken whole first, so `note'x'` is not an E” string. +func lexWord(q string, i int) (token, int, error) { + start := i + for i < len(q) && isIdentCont(q[i]) { + i++ + } + word := strings.ToLower(q[start:i]) + if i < len(q) && q[i] == '\'' { + switch word { + case "e": + return lexString(q, i, true) + case "b", "x", "n": + return lexString(q, i, false) + } + } + if word == "u" && i+1 < len(q) && q[i] == '&' { + switch q[i+1] { + case '\'': + return lexString(q, i+1, false) + case '"': + t, end, err := lexQuoted(q, i+1) + if err == nil && strings.Contains(t.text, `\`) { + t.kind = tokEscaped + } + return t, end, err + } + } + return token{kind: tokWord, text: word}, i, nil +} + +// lexString reads a string from its opening quote, with any continuation: two +// literals separated by a newline are one, and the first's rules hold in both. +func lexString(q string, i int, escapes bool) (token, int, error) { + var b strings.Builder + for i++; i < len(q); { + c := q[i] + switch { + case escapes && c == '\\': + if i+1 >= len(q) { + return token{}, 0, errUnreadable + } + b.WriteString(q[i : i+2]) + i += 2 + case c == '\'' && i+1 < len(q) && q[i+1] == '\'': + b.WriteByte('\'') + i += 2 + case c == '\'': + next, ok := stringContinues(q, i+1) + if !ok { + return token{kind: tokString, text: b.String()}, i + 1, nil + } + i = next + 1 + default: + b.WriteByte(c) + i++ + } + } + return token{}, 0, errUnreadable +} + +// stringContinues reports whether what follows a closing quote carries on the +// string: whitespace holding a newline, then a quote. It returns that quote. +// Line comments count as whitespace here, block comments do not. +func stringContinues(q string, i int) (int, bool) { + newline := false + for i < len(q) { + switch { + case q[i] == '\n' || q[i] == '\r': + newline = true + i++ + case isSpace(q[i]): + i++ + case strings.HasPrefix(q[i:], "--"): + i = lineEnd(q, i) + default: + return i, newline && q[i] == '\'' + } + } + return i, false +} + +func lexQuoted(q string, i int) (token, int, error) { + var b strings.Builder + for i++; i < len(q); i++ { + if q[i] != '"' { + b.WriteByte(q[i]) + continue + } + if i+1 < len(q) && q[i+1] == '"' { + b.WriteByte('"') + i++ + continue + } + return token{kind: tokQuoted, text: strings.ToLower(b.String())}, i + 1, nil + } + return token{}, 0, errUnreadable +} + +// lexDollar reads a $tag$ string, or a parameter or stray dollar if it is not one. +func lexDollar(q string, i int) (token, int, error) { + j := i + 1 + if j < len(q) && isTagStart(q[j]) { + for j < len(q) && (isTagStart(q[j]) || isDigit(q[j])) { + j++ + } + } + if j < len(q) && q[j] == '$' { + delim := q[i : j+1] + end := strings.Index(q[j+1:], delim) + if end < 0 { + return token{}, 0, errUnreadable + } + return token{kind: tokString, text: q[j+1 : j+1+end], plain: true}, j + 1 + end + len(delim), nil + } + for j = i + 1; j < len(q) && isDigit(q[j]); j++ { + } + return token{kind: tokPunct, text: "$"}, j, nil +} + +// splitStatements cuts a message at its top-level semicolons. A semicolon inside +// parentheses belongs to a CREATE RULE, and one in a BEGIN ATOMIC body to the +// function: neither ends the statement, and the grammar allows neither +// transaction control. Anything else that hides a semicolon that way does not +// parse, and the server rejects the whole message before running any of it. +func splitStatements(toks []token) [][]token { + var out [][]token + start, parens, atomic := 0, 0, 0 + for i, t := range toks { + switch { + case t.is(tokPunct, "("): + parens++ + case t.is(tokPunct, ")"): + if parens > 0 { + parens-- + } + case parens > 0: + case t.is(tokWord, "begin") && atomic == 0 && i+1 < len(toks) && + toks[i+1].is(tokWord, "atomic") && isRoutine(toks[start:]): + atomic = 1 + case t.is(tokWord, "case") && atomic > 0: + atomic++ + case t.is(tokWord, "end") && atomic > 0: + atomic-- + case t.is(tokPunct, ";") && atomic == 0: + if i > start { + out = append(out, toks[start:i]) + } + start = i + 1 + } + } + if start < len(toks) { + out = append(out, toks[start:]) + } + return out +} + +// isRoutine reports whether a statement is CREATE [OR REPLACE] FUNCTION or +// PROCEDURE, the only ones BEGIN ATOMIC can belong to. +func isRoutine(s []token) bool { + word := func(i int) string { return wordAt(s, i) } + if word(0) != "create" { + return false + } + i := 1 + if word(1) == "or" && word(2) == "replace" { + i = 3 + } + return word(i) == "function" || word(i) == "procedure" +} + +func wordAt(s []token, i int) string { + if i < len(s) && s[i].kind == tokWord { + return s[i].text + } + return "" +} + +// transactionControl is every statement that starts, ends or rewinds a +// transaction, by its first word. A savepoint counts: rolling back to one undoes +// entries the request would still report as applied. +var transactionControl = map[string]bool{ + "begin": true, "start": true, "commit": true, "end": true, "rollback": true, "abort": true, + "savepoint": true, "release": true, +} + +// refusedSettings are the settings a replayed statement may not change. A role +// or session authorization is who the statement runs as, the bb.* settings (by +// prefix) and application_name are who the Blackbox records and whether the +// guardrail is overridden, and session_replication_role switches triggers off. +// The two encodings change how the server reads the entries after this one, and +// so whether this check read them the same way. +var refusedSettings = map[string]bool{ + "role": true, "session_authorization": true, "session_replication_role": true, "application_name": true, + "standard_conforming_strings": true, "client_encoding": true, "names": true, "transaction": true, +} + +func refusedSetting(name string) bool { + return refusedSettings[name] || strings.HasPrefix(name, "bb.") +} + +// statementRefusal says why one statement cannot be replayed, or "" if it can. +func statementRefusal(s []token) string { + head := wordAt(s, 0) + switch { + case transactionControl[head]: + return fmt.Sprintf("%s is transaction control, and a request is applied in one transaction", strings.ToUpper(head)) + case head == "prepare" && wordAt(s, 1) == "transaction": + return "PREPARE TRANSACTION is transaction control, and a request is applied in one transaction" + case head == "set": + return setRefusal(s[1:], "SET") + case head == "reset": + return setRefusal(s[1:], "RESET") + case head == "do": + for _, t := range s[1:] { + if t.kind == tokString { + if r := bodyRefusal(t.text); r != "" { + return "its DO block: " + r + } + } + } + } + return setConfigRefusal(s) +} + +// setRefusal judges what follows SET or RESET, down to the name of the setting. +func setRefusal(rest []token, verb string) string { + i := 0 + if w := wordAt(rest, 0); w == "local" || (w == "session" && wordAt(rest, 1) != "authorization" && wordAt(rest, 1) != "characteristics") { + i = 1 + } + var name strings.Builder + for ; i < len(rest); i++ { + if rest[i].kind != tokWord && rest[i].kind != tokQuoted { + return fmt.Sprintf("the setting its %s changes cannot be read", verb) + } + name.WriteString(rest[i].text) + if i+1 >= len(rest) || !rest[i+1].is(tokPunct, ".") { + break + } + name.WriteByte('.') + i++ + } + n := name.String() + switch { + case n == "": + return fmt.Sprintf("the setting its %s changes cannot be read", verb) + case n == "session" && i+1 < len(rest): + if w := wordAt(rest, i+1); w == "authorization" || w == "characteristics" { + return fmt.Sprintf("%s SESSION %s changes what the session is", verb, strings.ToUpper(w)) + } + case n == "all" && verb == "RESET": + return "RESET ALL clears the settings that attribute the request" + case refusedSetting(n): + return fmt.Sprintf("%s %s changes who it runs or is recorded as, or how the rest is read", verb, n) + } + return "" +} + +// setConfigRefusal finds set_config() anywhere in a statement: SET by another +// name, and common enough with a harmless first argument that only the refused +// settings, or one that cannot be read, stop it. +func setConfigRefusal(s []token) string { + for i, t := range s { + if (t.kind != tokWord && t.kind != tokQuoted) || t.text != "set_config" || + i+1 >= len(s) || !s[i+1].is(tokPunct, "(") { + continue + } + if i+3 >= len(s) || s[i+2].kind != tokString || !s[i+2].plain || !s[i+3].is(tokPunct, ",") { + return "the setting its set_config() changes cannot be read" + } + if name := strings.ToLower(s[i+2].text); refusedSetting(name) { + return fmt.Sprintf("set_config() on %s changes who it runs or is recorded as", name) + } + } + return "" +} + +// bodyRefusal looks inside a DO block for the same things, in literal form. A +// block is not split into statements: BEGIN and END are its own syntax there, so +// transaction control is left to the server, which refuses it inside a +// transaction block. A SET counts only where a PL/pgSQL statement can start, so +// that `UPDATE t SET role = 1` is left alone. +func bodyRefusal(body string) string { + toks, err := lexSQL(body) + if err != nil { + return "it cannot be read: " + err.Error() + } + return tokensRefusal(toks, 0) +} + +func tokensRefusal(toks []token, depth int) string { + if r := setConfigRefusal(toks); r != "" { + return r + } + for i, t := range toks { + if t.kind == tokWord && (t.text == "set" || t.text == "reset") && startsStatement(toks, i) { + if r := setRefusal(toks[i+1:], strings.ToUpper(t.text)); r != "" { + return r + } + } + // EXECUTE 'SET ROLE …': a string in the block may be SQL. One that does + // not lex is not SQL, so it says nothing either way. + if t.kind == tokString && depth < 2 { + if inner, err := lexSQL(t.text); err == nil { + if r := tokensRefusal(inner, depth+1); r != "" { + return r + } + } + } + } + return "" +} + +func startsStatement(toks []token, i int) bool { + if i == 0 || toks[i-1].is(tokPunct, ";") { + return true + } + switch wordAt(toks, i-1) { + case "begin", "then", "else", "loop": + return true + } + return false +} + +// replayRefusal says why a whole message cannot be replayed, or "" if it can. +func replayRefusal(statement string) string { + toks, err := lexSQL(statement) + if err != nil { + return "it cannot be read: " + err.Error() + } + for _, t := range toks { + if t.kind == tokEscaped || t.is(tokWord, "uescape") { + return `it uses a U&"…" name with escapes, whose meaning this check does not work out` + } + } + for _, s := range splitStatements(toks) { + if r := statementRefusal(s); r != "" { + return r + } + } + return "" +} + +// checkReplayable refuses a request holding any entry that cannot be replayed. +func checkReplayable(entries []branch.RequestEntry) error { + var bad []string + for _, e := range entries { + if r := replayRefusal(e.Statement); r != "" { + bad = append(bad, fmt.Sprintf("entry %d (%s %s): %s", e.ID, e.CommandTag, e.Object, r)) + } + } + if len(bad) == 0 { + return nil + } + return fmt.Errorf("%w:\n %s\nNothing was applied. Make the change again on the source without those statements "+ + "(`%s blackbox` shows what it ran), or apply it by hand", branch.ErrNotReplayable, strings.Join(bad, "\n "), brand.CLI) +} diff --git a/enterprise/promote/replaycheck_test.go b/enterprise/promote/replaycheck_test.go new file mode 100644 index 00000000..a8334396 --- /dev/null +++ b/enterprise/promote/replaycheck_test.go @@ -0,0 +1,118 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package promote + +import ( + "errors" + "strings" + "testing" + + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// Every class the replay must refuse, in the forms a message can hide it in. +func TestReplayRefusal(t *testing.T) { + for _, q := range []string{ + // Transaction control. + "COMMIT", "commit;", "END", "END WORK", "ABORT", "ROLLBACK", "ROLLBACK TO SAVEPOINT s", + "BEGIN", "BEGIN ISOLATION LEVEL SERIALIZABLE", "START TRANSACTION", "SAVEPOINT s", "RELEASE SAVEPOINT s", + "PREPARE TRANSACTION 'x'", "COMMIT PREPARED 'x'", "ROLLBACK PREPARED 'x'", + "SET TRANSACTION READ ONLY", "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + // Role and session. + "SET ROLE dbadmin", "set role none", "SET LOCAL ROLE dbadmin", "SET SESSION ROLE dbadmin", "SET role = dbadmin", + `SET "role" TO dbadmin`, `SET "ROLE" TO dbadmin`, "RESET ROLE", "SET SESSION AUTHORIZATION dbadmin", + "SET LOCAL SESSION AUTHORIZATION dbadmin", "SET session_authorization = dbadmin", "RESET SESSION AUTHORIZATION", + "RESET session_authorization", "SET session_replication_role = replica", + // Attribution and the guardrail's override. + "SET bb.actor = 'x'", "SET LOCAL bb.actor_kind = 'human'", "set BB.ALLOW_DESTRUCTIVE = on", "SET bb . actor = 'x'", + `SET "bb".actor = 'x'`, `SET "bb.actor" = 'x'`, `SET "BB"."Actor" = 'x'`, "RESET bb.actor", "RESET ALL", + "SET application_name = 'x'", "SET bb.policy_allow = 'r1'", + "SELECT set_config('bb.actor', 'x', true)", "SELECT pg_catalog.set_config('role', 'dbadmin', true)", + `SELECT "set_config"('bb.session', 'x', false)`, "SELECT set_config('BB.Actor', 'x', true)", + "SELECT set_config(lower('ROLE'), 'dbadmin', true)", "SELECT set_config(E'bb.actor', 'x', true)", + // What changes how later statements are read. + "SET standard_conforming_strings = off", "SET client_encoding = 'SJIS'", "SET NAMES 'SJIS'", + // Behind comments, or where one seems to be. + "/* hello */ COMMIT", "-- hello\nCOMMIT", "/* a /* nested */ b */ RESET ROLE", "COM/* */MIT;SET ROLE x", + "-- a comment ended by a carriage return\rCOMMIT", + "CREATE TABLE t (); /* */ COMMIT; --", + // After a literal that seems to swallow it. + "SELECT 'a'; COMMIT", "SELECT $$a$$; COMMIT", "SELECT $x$ $y$ $x$; COMMIT", `SELECT "a;"; COMMIT`, + "SELECT 'it''s'; COMMIT", "SELECT E'it\\'s'; COMMIT", + // E'' escapes carry across a continuation, so this string ends where it seems not to. + "SELECT E'a'\n'\\''; COMMIT; --'", + "SELECT E'a' -- comment\n'\\''; COMMIT; --'", + // A dollar after a number starts a quote; inside a name it does not. + "SELECT 1$$a$$; COMMIT", + "SELECT a$b$; COMMIT; --$b$", + // Multi-statement messages, wherever it falls. + "CREATE TABLE t (id int); COMMIT; DROP TABLE u", "COMMIT; CREATE TABLE t (id int)", + "CREATE TABLE t (id int);;\n;SET ROLE dbadmin", + // BEGIN ATOMIC is only a body after CREATE FUNCTION, and ends at its END. + "CREATE FUNCTION begin() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; COMMIT", + "CREATE FUNCTION f() RETURNS int BEGIN ATOMIC SELECT 1; END; COMMIT", + // A DO block is not a way round. + "DO $$ BEGIN SET ROLE dbadmin; END $$", "DO $$ BEGIN SET LOCAL bb.actor = 'x'; END $$", + "DO $$ BEGIN PERFORM set_config('bb.actor', 'x', true); END $$", + "DO $$ BEGIN EXECUTE 'SET ROLE dbadmin'; END $$", "DO LANGUAGE plpgsql 'BEGIN RESET ROLE; END'", + // What cannot be read is not passed. + "CREATE TABLE t (a text DEFAULT 'oops", `CREATE TABLE "t (a int)`, "CREATE TABLE t () /* open", "SELECT $x$ open", + "SELECT 1\x00; COMMIT", `SET U&"r\006Fle" = dbadmin`, `CREATE TABLE U&"t!0041" UESCAPE '!' ()`, + } { + if replayRefusal(q) == "" { + t.Errorf("not refused: %q", q) + } + } +} + +// Ordinary changes, and the words above where they mean nothing. +func TestReplayAccepted(t *testing.T) { + for _, q := range []string{ + "CREATE TABLE t (id int PRIMARY KEY, note text)", "ALTER TABLE t ADD COLUMN x int;", "DROP INDEX i", + "CREATE INDEX ON t (id)", "GRANT SELECT ON t TO bb_audit", "COMMENT ON TABLE t IS 'do not COMMIT this; ROLLBACK later'", + // Multi-statement without transaction control. + "CREATE TABLE a (id int); CREATE VIEW v AS SELECT id FROM a; ALTER TABLE a ADD b text", + "SET search_path = app; CREATE TABLE a (id int)", "SET LOCAL lock_timeout = '5s'; ALTER TABLE a ADD b int", + "SELECT pg_catalog.set_config('search_path', '', false); CREATE TABLE a (id int)", + // Only in comments or quoting. + "CREATE TABLE t (id int) -- then COMMIT;", "/* SET ROLE x; */ CREATE TABLE t (id int)", + "CREATE TABLE t (a text DEFAULT '; COMMIT; --')", "CREATE TABLE t (a text DEFAULT $$; RESET ROLE;$$)", + `CREATE TABLE "commit" (id int)`, `CREATE TABLE t ("set role" int); ALTER TABLE t RENAME "set role" TO "begin"`, + "ALTER TABLE t ADD COLUMN \"end\" int", + // A string the E'' continuation really does keep going. + "SELECT E'a'\n'\\'; COMMIT; --'", + // PREPARE of a query, a column called role, a routine body with BEGIN and END. + "PREPARE p AS SELECT 1; CREATE TABLE a (id int)", + "UPDATE t SET role = 'x'; ALTER TABLE t ADD y int", + "CREATE FUNCTION f() RETURNS int LANGUAGE plpgsql AS $$ BEGIN COMMIT; RETURN 1; END $$", + "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC SELECT 1; SELECT CASE WHEN true THEN 1 ELSE 2 END; END", + "CREATE OR REPLACE FUNCTION f() RETURNS int BEGIN ATOMIC SELECT 1; END; CREATE TABLE a (id int)", + "CREATE RULE r AS ON INSERT TO t DO ALSO (SELECT 1; NOTIFY x)", + "DO $$ BEGIN CREATE TABLE a (id int); EXCEPTION WHEN duplicate_table THEN NULL; END $$", + "DO $$ BEGIN UPDATE t SET role = 'x'; END $$", + "DO $$ DECLARE r regclass := 't'::regclass; BEGIN EXECUTE format('GRANT SELECT ON %s TO bb_audit', r); END $$", + `CREATE TABLE U&"plain" (id int)`, + } { + if r := replayRefusal(q); r != "" { + t.Errorf("refused %q: %s", q, r) + } + } +} + +// Refused by name, and before any container is looked for: this test has none. +func TestApplyRefusesAnUnsafeStatement(t *testing.T) { + _, err := apply(1, "main", "someone@example.com", []branch.RequestEntry{ + entry(1, "a", "human", "CREATE TABLE", "public.t", "CREATE TABLE t (id int)"), + entry(2, "a", "human", "CREATE TABLE", "public.u", "CREATE TABLE u (id int); COMMIT"), + }) + if !errors.Is(err, branch.ErrNotReplayable) { + t.Fatalf("err = %v", err) + } + for _, want := range []string{"entry 2", "public.u", "COMMIT", "Nothing was applied"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("%q does not say %q", err, want) + } + } +} diff --git a/enterprise/realtime/decode.go b/enterprise/realtime/decode.go index 3ab0ed4f..865df015 100644 --- a/enterprise/realtime/decode.go +++ b/enterprise/realtime/decode.go @@ -48,9 +48,10 @@ type Decoder struct { // streaming sends only complete transactions, in commit order, so there is // exactly one of these at a time and no interleaving to untangle. tx txState - // acked is the last position every subscriber has been given. Only this is - // reported to Postgres, so WAL behind it stays reclaimable and WAL ahead of - // it stays available for a subscriber that reconnects. + // acked is the last position every subscriber has been given, or the + // server's own position once nothing is owed to anyone (advanceIfQuiet). + // Only this is reported to Postgres, so WAL behind it stays reclaimable and + // WAL ahead of it stays available for a subscriber that reconnects. acked pglogrepl.LSN } @@ -134,6 +135,7 @@ func (d *Decoder) Run(ctx context.Context, since pglogrepl.LSN) error { } } d.acked = since + noteQuietStart(d.slot, since, d.hub.Floor()) if err := pglogrepl.StartReplication(ctx, d.conn, d.slot, since, pglogrepl.StartReplicationOptions{PluginArgs: []string{ "proto_version '1'", "publication_names '" + d.publication + "'", @@ -166,41 +168,81 @@ func (d *Decoder) Run(ctx context.Context, since pglogrepl.LSN) error { if !ok { continue // a notice or similar; nothing to decode } - switch cd.Data[0] { - case pglogrepl.PrimaryKeepaliveMessageByteID: - ka, err := pglogrepl.ParsePrimaryKeepaliveMessage(cd.Data[1:]) - if err != nil { - return err - } - if ka.ReplyRequested { - next = time.Now() // answer on the next turn of the loop - } - case pglogrepl.XLogDataByteID: - xld, err := pglogrepl.ParseXLogData(cd.Data[1:]) - if err != nil { - return err - } - if err := d.handle(xld); err != nil { - return err - } - // A subscriber overflowed inside the transaction that has just - // finished, and is still attached for exactly this moment: report - // the position before it goes, because the last subscriber leaving - // is what stops this decoder, and a transaction nobody could be - // given has to be behind us when that happens. Otherwise the next - // subscriber starts in front of it and overflows in the same place. - if d.hub.Releasable() { - if err := pglogrepl.SendStandbyStatusUpdate(ctx, d.conn, - pglogrepl.StandbyStatusUpdate{WALWritePosition: d.acked}); err != nil { - return fmt.Errorf("reporting the position of a transaction that could not be delivered: %w", err) - } - next = time.Now().Add(StandbyInterval) - d.hub.ReleasePending() + reply, err := d.handleCopyData(cd.Data) + if err != nil { + return err + } + if reply { + next = time.Now() // answer on the next turn of the loop + } + // A subscriber overflowed inside the transaction that has just + // finished, and is still attached for exactly this moment: report + // the position before it goes, because the last subscriber leaving + // is what stops this decoder, and a transaction nobody could be + // given has to be behind us when that happens. Otherwise the next + // subscriber starts in front of it and overflows in the same place. + if d.hub.Releasable() { + if err := pglogrepl.SendStandbyStatusUpdate(ctx, d.conn, + pglogrepl.StandbyStatusUpdate{WALWritePosition: d.acked}); err != nil { + return fmt.Errorf("reporting the position of a transaction that could not be delivered: %w", err) } + next = time.Now().Add(StandbyInterval) + d.hub.ReleasePending() } } } +// handleCopyData takes in one message from the replication stream and reports +// whether Postgres asked for an answer straight away. +func (d *Decoder) handleCopyData(data []byte) (bool, error) { + if len(data) == 0 { + return false, nil // nothing a walsender sends, and nothing to index + } + switch data[0] { + case pglogrepl.PrimaryKeepaliveMessageByteID: + ka, err := pglogrepl.ParsePrimaryKeepaliveMessage(data[1:]) + if err != nil { + return false, err + } + d.advanceIfQuiet(ka.ServerWALEnd) + return ka.ReplyRequested, nil + case pglogrepl.XLogDataByteID: + xld, err := pglogrepl.ParseXLogData(data[1:]) + if err != nil { + return false, err + } + if err := d.handle(xld); err != nil { + return false, err + } + d.advanceIfQuiet(xld.ServerWALEnd) + } + return false, nil +} + +// advanceIfQuiet acknowledges a position the server has reported, when that +// is safe to do. +// +// A table nobody writes to produces no commits here, so without this the +// position stands still while the rest of the database keeps writing, the WAL +// behind it cannot be reclaimed, and max_slot_wal_keep_size eventually +// invalidates the slot. Nobody could be told what happened: there was nothing +// to deliver. +// +// The server only sends a position once everything committing before it has +// been written to this connection, and a message arrives after the ones sent +// before it. So once no transaction is open here, and everything handed to +// subscribers has left their buffers, nothing before that position is still +// owed to anyone. Transactions not yet committed at that position are not +// affected: they are sent when they commit, and Postgres keeps their WAL for +// that whatever is acknowledged here. +func (d *Decoder) advanceIfQuiet(end pglogrepl.LSN) { + if d.tx.commitLSN != "" || end <= d.acked || !d.hub.Drained() { + return + } + d.acked = end + noteQuietAcked(d.slot, end) +} + // Close ends the replication connection. The slot stays: it is what a // reconnecting subscriber resumes from, and sweeping it is a decision made // elsewhere, where "is anyone still interested" can be answered. @@ -284,10 +326,16 @@ func (d *Decoder) handleMessage(m pglogrepl.Message, lsn string) error { At: msg.CommitTime.UTC().Format(time.RFC3339Nano), Changes: d.tx.changes}) } - d.tx = txState{} // Everything in this transaction has been handed to the hub, so the // position is now safe to acknowledge. + published := d.tx.begun + d.tx = txState{} d.acked = msg.CommitLSN + if published { + noteQuietPublished(d.slot, msg.CommitLSN) + } else { + noteQuietAcked(d.slot, msg.CommitLSN) + } } return nil } diff --git a/enterprise/realtime/hub.go b/enterprise/realtime/hub.go index 54b8c926..943042da 100644 --- a/enterprise/realtime/hub.go +++ b/enterprise/realtime/hub.go @@ -230,6 +230,26 @@ func (h *Hub) Stats() (subscribers int, delivered int64, peak int) { return len(h.subs), h.delivered, h.peak } +// Drained reports whether somebody is attached and has been given everything +// published so far: no buffer holds an event, and nobody is waiting to be told +// their stream is over. +// +// False with no subscribers at all. Nobody has been delivered anything then, +// and the decoder is either about to stop or about to be subscribed to. +func (h *Hub) Drained() bool { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.subs) == 0 { + return false + } + for _, s := range h.subs { + if len(s.ch) > 0 || s.isPending() { + return false + } + } + return true +} + // Publish sends to every subscriber without blocking on any of them. func (h *Hub) Publish(ev any) { done := false // a transaction finished in this call diff --git a/enterprise/realtime/quiet.go b/enterprise/realtime/quiet.go new file mode 100644 index 00000000..fe1e6efb --- /dev/null +++ b/enterprise/realtime/quiet.go @@ -0,0 +1,104 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package realtime + +import ( + "sync" + + "github.com/jackc/pglogrepl" +) + +// What a quiet stretch of a slot's position leaves behind for a resume. +// +// A slot acknowledged on keepalives moves past positions where nothing was +// published, which is what stops a quiet table's WAL from piling up. But the +// resume check reads the slot's confirmed position as "everything before this +// was somebody else's": a client reconnecting with the commit position it +// last saw is judged against it, and the slot having moved on would tell it to +// refetch, for a gap that never held a single change. A client on a quiet table +// reconnects at least hourly, so that would be a refetch an hour for nothing. +// +// So this keeps, per slot, the span the decoder has acknowledged without +// publishing anything: from the last position a change was published at +// (from) to the furthest position acknowledged (to). A position inside that +// span has missed nothing, and QuietSpan hands it to the resume check. +// +// In memory, per process, and only ever evidence that a position is safe: with +// nothing recorded (a restarted control plane, a slot read by something else) +// the check goes back to refusing whatever is before the slot's position, which +// costs a client one refetch and never a silent gap. A span from a slot that +// has since been dropped or moved by something else is refused by QuietSpan, +// because the slot's position is then past anything this process acknowledged. +type quietSpan struct{ from, to pglogrepl.LSN } + +var quiet = struct { + mu sync.Mutex + m map[string]quietSpan +}{m: map[string]quietSpan{}} + +// noteQuietStart records a decoder starting on a slot. The first decoder on a +// slot since this process began starts the span at the position it was asked +// for, or else at the hub's floor; a later one continues the span. +func noteQuietStart(slot string, since, floor pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s, ok := quiet.m[slot] + if !ok { + s.from = since + if s.from == 0 { + s.from = floor + } + } + if since > s.to { + s.to = since + } + quiet.m[slot] = s +} + +// noteQuietAcked records a position acknowledged with nothing published. +func noteQuietAcked(slot string, lsn pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s := quiet.m[slot] + if lsn > s.to { + s.to = lsn + } + quiet.m[slot] = s +} + +// noteQuietPublished records a position a change was published at: everything +// before it has been somebody's to receive, so the span starts again here. +func noteQuietPublished(slot string, lsn pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s := quiet.m[slot] + s.from = lsn + if lsn > s.to { + s.to = lsn + } + quiet.m[slot] = s +} + +// QuietSpan is a slot's quiet span for a resume check, given the slot's +// confirmed position as read just now. Both are zero when there is nothing +// trustworthy to go on. A confirmed position past what this process ever +// acknowledged means the slot is not the one the span describes -- dropped and +// made again, or moved by something else -- so the span is forgotten. +func QuietSpan(slot string, confirmed pglogrepl.LSN, haveSlot bool) (from, to pglogrepl.LSN) { + quiet.mu.Lock() + defer quiet.mu.Unlock() + s, ok := quiet.m[slot] + if !ok { + return 0, 0 + } + if !haveSlot || confirmed > s.to { + delete(quiet.m, slot) + return 0, 0 + } + if s.from == 0 { + return 0, 0 + } + return s.from, s.to +} diff --git a/enterprise/realtime/quiet_test.go b/enterprise/realtime/quiet_test.go new file mode 100644 index 00000000..f41744f9 --- /dev/null +++ b/enterprise/realtime/quiet_test.go @@ -0,0 +1,275 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package realtime + +import ( + "encoding/binary" + "fmt" + "testing" + + "github.com/jackc/pglogrepl" +) + +// A decoder on a test's own slot, with one subscriber that has read everything +// so far unless the test leaves its events in the buffer. +func quietDecoder(t *testing.T, slot string) (*Decoder, *Subscriber) { + t.Helper() + quiet.mu.Lock() + delete(quiet.m, slot) + quiet.mu.Unlock() + h := NewHub() + s := h.Subscribe() + t.Cleanup(s.Close) + d := &Decoder{hub: h, slot: slot, relations: map[uint32]Relation{}} + d.relations[1] = Relation{Schema: "public", Name: "t", ReplicaIdentity: 'd', + Columns: []Column{{Name: "id", TypeOID: 20, Key: true}}} + return d, s +} + +func keepaliveAt(end pglogrepl.LSN, reply bool) []byte { + b := make([]byte, 18) + b[0] = pglogrepl.PrimaryKeepaliveMessageByteID + binary.BigEndian.PutUint64(b[1:], uint64(end)) + if reply { + b[17] = 1 + } + return b +} + +// xlogAt wraps pgoutput's own bytes for a message as a walsender sends them. +func xlogAt(start, end pglogrepl.LSN, wal []byte) []byte { + b := make([]byte, 25, 25+len(wal)) + b[0] = pglogrepl.XLogDataByteID + binary.BigEndian.PutUint64(b[1:], uint64(start)) + binary.BigEndian.PutUint64(b[9:], uint64(end)) + return append(b, wal...) +} + +func beginWAL(final pglogrepl.LSN, xid uint32) []byte { + b := make([]byte, 21) + b[0] = 'B' + binary.BigEndian.PutUint64(b[1:], uint64(final)) + binary.BigEndian.PutUint32(b[17:], xid) + return b +} + +func commitWAL(commit, end pglogrepl.LSN) []byte { + b := make([]byte, 26) + b[0] = 'C' + binary.BigEndian.PutUint64(b[2:], uint64(commit)) + binary.BigEndian.PutUint64(b[10:], uint64(end)) + return b +} + +func insertWAL(rel uint32, value string) []byte { + b := []byte{'I', 0, 0, 0, 0, 'N', 0, 1, 't', 0, 0, 0, byte(len(value))} + binary.BigEndian.PutUint32(b[1:], rel) + return append(b, value...) +} + +func drain(s *Subscriber) (n int) { + for { + select { + case <-s.Events(): + n++ + default: + return n + } + } +} + +func mustCopy(t *testing.T, d *Decoder, data []byte) bool { + t.Helper() + reply, err := d.handleCopyData(data) + if err != nil { + t.Fatal(err) + } + return reply +} + +func TestAKeepaliveAdvancesAQuietSlot(t *testing.T) { + d, _ := quietDecoder(t, "ka") + if mustCopy(t, d, keepaliveAt(1000, false)) { + t.Error("a keepalive not asking for a reply got one") + } + if d.acked != 1000 { + t.Fatalf("acked = %v, want the keepalive's position", d.acked) + } + if !mustCopy(t, d, keepaliveAt(1500, true)) { + t.Error("a keepalive asking for a reply did not get one") + } + // Never backwards: a server can report what it has already reported. + mustCopy(t, d, keepaliveAt(1200, false)) + if d.acked != 1500 { + t.Errorf("acked = %v after an older keepalive, want 1500", d.acked) + } + // The XLogData header carries a position too, and a message arriving when + // nothing is open advances on it just the same. + mustCopy(t, d, xlogAt(1500, 1800, []byte("O\x00\x00\x00\x00\x00\x00\x00\x00x\x00"))) + if d.acked != 1800 { + t.Errorf("acked = %v after XLogData's position, want 1800", d.acked) + } +} + +// Empty CopyData has no type byte: indexing it would panic in the decoder that +// every subscriber on the branch is reading from. +func TestEmptyCopyDataIsHarmless(t *testing.T) { + d, _ := quietDecoder(t, "empty") + if reply, err := d.handleCopyData(nil); reply || err != nil { + t.Fatalf("empty CopyData: %v %v", reply, err) + } + if reply, err := d.handleCopyData([]byte{}); reply || err != nil { + t.Fatalf("empty CopyData: %v %v", reply, err) + } + if d.acked != 0 { + t.Errorf("acked moved on nothing: %v", d.acked) + } + // Short messages are refused by their parsers, not read past their end. + if _, err := d.handleCopyData([]byte{pglogrepl.PrimaryKeepaliveMessageByteID}); err == nil { + t.Error("a truncated keepalive was accepted") + } +} + +// The one that must not happen: a position past a change someone has not been +// given. Inside a transaction a keepalive may be ahead of the commit. +func TestAKeepaliveDoesNotAcknowledgeInsideATransaction(t *testing.T) { + d, s := quietDecoder(t, "intx") + mustCopy(t, d, keepaliveAt(1000, false)) + mustCopy(t, d, xlogAt(1000, 1100, beginWAL(2000, 7))) + mustCopy(t, d, keepaliveAt(1900, false)) + if d.acked != 1000 { + t.Fatalf("a keepalive inside a transaction moved acked to %v", d.acked) + } + mustCopy(t, d, xlogAt(1100, 1200, insertWAL(1, "1"))) + drain(s) + mustCopy(t, d, keepaliveAt(1950, false)) + if d.acked != 1000 { + t.Fatalf("a keepalive after a row moved acked to %v", d.acked) + } + mustCopy(t, d, xlogAt(1200, 2000, commitWAL(2000, 2010))) + if d.acked != 2000 { + t.Fatalf("a commit acknowledged %v, want the commit's position", d.acked) + } + drain(s) + mustCopy(t, d, keepaliveAt(3000, false)) + if d.acked != 3000 { + t.Errorf("acked = %v once quiet again, want 3000", d.acked) + } +} + +func TestAKeepaliveWaitsForSubscribersToBeGivenEverything(t *testing.T) { + d, s := quietDecoder(t, "buffered") + d.hub.Publish(Commit{Type: "commit"}) // an event still in the buffer + mustCopy(t, d, keepaliveAt(1000, false)) + if d.acked != 0 { + t.Fatalf("acked = %v with an event still buffered", d.acked) + } + drain(s) + mustCopy(t, d, keepaliveAt(1000, false)) + if d.acked != 1000 { + t.Errorf("acked = %v once drained, want 1000", d.acked) + } +} + +// With nobody attached, nobody has been given anything. +func TestAKeepaliveDoesNotAdvanceWithNoSubscribers(t *testing.T) { + d, s := quietDecoder(t, "nosubs") + s.Close() + mustCopy(t, d, keepaliveAt(1000, false)) + if d.acked != 0 { + t.Errorf("acked = %v with nobody attached", d.acked) + } +} + +// The resume check against the positions acknowledged on keepalives, which is +// what the keepalive advance puts at risk: a client that saw the last commit, +// on a table nobody has written to since, must still be able to come back. +func TestAResumeSurvivesAQuietSlotMovingOn(t *testing.T) { + d, s := quietDecoder(t, "resume") + noteQuietStart(d.slot, 1000, 1000) + d.acked = 1000 + mustCopy(t, d, xlogAt(1000, 1100, beginWAL(2000, 7))) + mustCopy(t, d, xlogAt(1100, 1200, insertWAL(1, "1"))) + mustCopy(t, d, xlogAt(1200, 2000, commitWAL(2000, 2010))) + drain(s) + mustCopy(t, d, keepaliveAt(9000, false)) + if d.acked != 9000 { + t.Fatalf("acked = %v", d.acked) + } + + window := func(confirmed pglogrepl.LSN) Window { + w := Window{HaveSlot: true, Confirmed: confirmed, WALEnd: 10000} + w.QuietFrom, w.QuietTo = QuietSpan(d.slot, confirmed, true) + return w + } + w := window(9000) + if got := JudgeResume(2000, w); got != ResumeFrom { + t.Errorf("a client at the last commit, after only keepalives: %v, want ResumeFrom", got) + } + if got := JudgeResume(5000, w); got != ResumeFrom { + t.Errorf("a position inside the quiet span: %v, want ResumeFrom", got) + } + if got := JudgeResume(1500, w); got != ResumeExpired { + t.Errorf("a client from before the last commit has missed a change: %v, want ResumeExpired", got) + } + + // A slot gone past what this process acknowledged is a different slot: + // the span is no evidence about it, and is forgotten. + w = window(12000) + if w.QuietFrom != 0 || w.QuietTo != 0 { + t.Fatalf("a span survived a slot that moved past it: %+v", w) + } + if got := JudgeResume(2000, Window{HaveSlot: true, Confirmed: 12000, WALEnd: 20000}); got != ResumeExpired { + t.Errorf("no evidence must mean a refetch: %v", got) + } +} + +func TestAQuietSpanIsForgottenWithItsSlot(t *testing.T) { + const slot = "dropped" + noteQuietPublished(slot, 100) + noteQuietAcked(slot, 500) + if from, to := QuietSpan(slot, 500, true); from != 100 || to != 500 { + t.Fatalf("span: %v %v", from, to) + } + if from, _ := QuietSpan(slot, 0, false); from != 0 { + t.Fatal("a span survived its slot") + } + if from, _ := QuietSpan(slot, 500, true); from != 0 { + t.Fatal("a forgotten span came back") + } + if from, to := QuietSpan("unknown-slot", 500, true); from != 0 || to != 0 { + t.Fatalf("a span for a slot never seen: %v %v", from, to) + } + // A fan-out that could not say where it started has no span to give. + noteQuietStart("no-floor", 0, 0) + if from, _ := QuietSpan("no-floor", 0, true); from != 0 { + t.Errorf("a span from nowhere: %v", from) + } + // A later decoder continues a span rather than restarting it. + noteQuietPublished("continued", 800) + noteQuietStart("continued", 900, 900) + if from, to := QuietSpan("continued", 900, true); from != 800 || to != 900 { + t.Errorf("continued span: %v %v", from, to) + } +} + +func TestJudgeResumeQuietNeedsTheSpanToCoverTheSlot(t *testing.T) { + cases := []struct { + w Window + want ResumeVerdict + }{ + {Window{HaveSlot: true, Confirmed: 5000, QuietFrom: 1000, QuietTo: 5000}, ResumeFrom}, + {Window{HaveSlot: true, Confirmed: 5000, QuietFrom: 1000, QuietTo: 6000}, ResumeFrom}, // the slot lags what was reported + {Window{HaveSlot: true, Confirmed: 7000, QuietFrom: 1000, QuietTo: 6000}, ResumeExpired}, + {Window{HaveSlot: true, Confirmed: 5000, QuietFrom: 3000, QuietTo: 6000}, ResumeExpired}, + {Window{HaveSlot: true, Confirmed: 5000}, ResumeExpired}, + {Window{HaveSlot: false, Confirmed: 0, QuietFrom: 1000, QuietTo: 6000}, ResumeExpired}, + } + for i, c := range cases { + if got := JudgeResume(2000, c.w); got != c.want { + t.Errorf("case %d (%s): %v, want %v", i, fmt.Sprint(c.w), got, c.want) + } + } +} diff --git a/enterprise/realtime/resume.go b/enterprise/realtime/resume.go index 3dd55275..70bbf33d 100644 --- a/enterprise/realtime/resume.go +++ b/enterprise/realtime/resume.go @@ -72,6 +72,15 @@ type Window struct { // changes are published to the subscribers already attached. Meaningless // unless Live. Floor pglogrepl.LSN + // QuietFrom and QuietTo are the span the slot has been acknowledged across + // with nothing published (QuietSpan), or zero when that is not known. + QuietFrom, QuietTo pglogrepl.LSN +} + +// quiet reports whether a position lies in the span acknowledged with nothing +// published, and that span reaches as far as the slot has. +func (w Window) quiet(since pglogrepl.LSN) bool { + return w.QuietFrom != 0 && since >= w.QuietFrom && w.Confirmed <= w.QuietTo } // JudgeResume decides what to do with a subscriber's ?since=. @@ -109,7 +118,12 @@ func JudgeResume(since pglogrepl.LSN, w Window) ResumeVerdict { // Postgres will not go back further than the slot has been acknowledged to, // and it does not say so. Asking for an older position there is how the // stream silently became "from the slot" instead. - if !w.HaveSlot || since < w.Confirmed { + // + // Older than the slot is still fine when the slot only moved past it on + // keepalives: nothing was published in between, so the position has missed + // nothing, and starting from the confirmed position is starting in the + // right place. Quiet is what says so; see quiet.go. + if !w.HaveSlot || (since < w.Confirmed && !w.quiet(since)) { return ResumeExpired } return ResumeFrom diff --git a/enterprise/schema/policy.sql b/enterprise/schema/policy.sql index 457e63f8..9759f44c 100644 --- a/enterprise/schema/policy.sql +++ b/enterprise/schema/policy.sql @@ -228,7 +228,7 @@ BEGIN -- the rollback the error below causes (the guardrail's mechanism). eval_id := NULL; BEGIN - conn := 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user; + conn := bb._blocked_conn(); -- Not when this transaction already wrote to Blackbox: the second connection -- would wait for this one forever (bb._holds_chain_lock). The evaluation row -- below takes no such lock and is still recorded. diff --git a/enterprise/schema/schema_test.go b/enterprise/schema/schema_test.go index 1c42c4be..776d6634 100644 --- a/enterprise/schema/schema_test.go +++ b/enterprise/schema/schema_test.go @@ -97,6 +97,11 @@ func TestTheGateChecksTheChainLockBeforeWriting(t *testing.T) { if blocked := strings.Index(schemaPolicy[lock:], "'BLOCKED','policy'"); blocked < 0 { t.Error("policy.sql (gate): no BLOCKED write follows the lock check") } + // And through the connection with timeouts, so no wait there can hang the + // statement being refused. + if !strings.Contains(schemaPolicy, "conn := bb._blocked_conn();") { + t.Error("policy.sql (gate): the blocked attempt must use bb._blocked_conn") + } } // Rules match the statement that runs, not everything a client sent with it. diff --git a/internal/auth/accounts.go b/internal/auth/accounts.go index 2fd00481..1659aa1f 100644 --- a/internal/auth/accounts.go +++ b/internal/auth/accounts.go @@ -185,13 +185,19 @@ func (s *Store) BranchOwner(branch string) (int64, bool) { } // ForgetBranch drops what the store held about a branch that has been deleted: -// its owner, and any change request still waiting on it. One place decides what +// its owner, a pipeline's claim on its name, and any change request still +// waiting on it. One place decides what // forgetting a branch means, so every caller — the CLI, the API, the agent API — // gets the same thing. func (s *Store) ForgetBranch(branch string) error { if _, err := s.db.Exec(`DELETE FROM branch_owners WHERE branch=?`, branch); err != nil { return err } + // A claim that outlived the branch let its pipeline delete whatever somebody + // else later made under that name. + if _, err := s.db.Exec(`DELETE FROM pipeline_targets WHERE branch=?`, branch); err != nil { + return err + } // Readings go with the branch, unlike decided requests, which are history // and stay. A branch name can be reused, and a newly made branch that // inherited the old one's readings would report work it never did — diff --git a/internal/auth/audit.go b/internal/auth/audit.go index a6c6db9b..32f6cb2b 100644 --- a/internal/auth/audit.go +++ b/internal/auth/audit.go @@ -25,7 +25,9 @@ import ( // SQLite triggers refuse UPDATE and DELETE, and the checkpointer anchors the // chain's head in a signed file outside the store (branch.CheckpointSecurityLog), // so rewriting history — even recomputing every hash — breaks either the chain -// or an anchor. `fox audit verify` checks both. +// or an anchor. `fox audit verify` checks both over every event; a checkpoint +// checks only what has been added since the last anchor (see +// branch.CheckpointSecurityLog for exactly what that does and does not cover). // Event kinds. const ( @@ -435,8 +437,13 @@ func (s *Store) LastLicenseEvent() (kind, subject string, ok bool, err error) { // CheckEventChain recomputes the chain. It returns the id of the first event // that does not match, or 0 when all do. func CheckEventChain(evs []SecurityEvent) (int64, string) { - prev := "" - var lastID int64 + return CheckEventChainFrom(evs, 0, "") +} + +// CheckEventChainFrom recomputes the chain over evs, taking them to follow the +// event lastID, whose row hash is prev. Like CheckEventChain, it returns the id +// of the first event that does not match, or 0 when all do. +func CheckEventChainFrom(evs []SecurityEvent, lastID int64, prev string) (int64, string) { for _, e := range evs { switch { case e.ID != lastID+1: diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 42a0bdee..8744f27f 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -57,6 +57,7 @@ type Store struct { db *sql.DB cfg Config throttle *throttle + gateway *gatewayGuard // historyTrimmed is when query history was last trimmed to its retention, // in Unix nanoseconds (history.go). historyTrimmed atomic.Int64 @@ -142,7 +143,9 @@ func Open(cfg Config) (*Store, error) { db.Close() return nil, err } - return &Store{db: db, cfg: cfg, throttle: newThrottle(loginMaxFails, loginWindow)}, nil + s := &Store{db: db, cfg: cfg, throttle: newThrottle(loginMaxFails, loginWindow)} + s.gateway = newGatewayGuard(s) + return s, nil } // initSchema creates and migrates the store. @@ -283,6 +286,9 @@ func (s *Store) Close() error { if s == nil || s.db == nil { return nil } + if s.gateway != nil { + s.gateway.close() + } return s.db.Close() } diff --git a/internal/auth/gateway.go b/internal/auth/gateway.go new file mode 100644 index 00000000..8166c448 --- /dev/null +++ b/internal/auth/gateway.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "fmt" + "net/netip" + "sync" + "time" +) + +// The Gateway's refusals, kept from flooding the security log. +// +// Every invalid key used to be one event, and one event is a chained row +// written under SQLite's write lock in a log that is never trimmed, so anyone +// who could reach the port could grow it without limit and queue every other +// writer behind them. Two things stop that, both per client address. +// +// The refusals are coalesced: one event per address per window, the first as +// it happens, carrying how many were suppressed since the event before it, so +// the log still says that an address was hammering the door and roughly how +// hard. A window's suppressed count is written when the window ends even if +// the address has gone quiet by then; a process killed inside a window loses +// that one count, never the event that opened it. +// +// And an address that has used up its invalid keys is refused before any key +// is looked up, until its window has passed. Only invalid keys count towards +// that. A valid key that is refused (a realtime key, a branch its owner may +// not open) is coalesced but never held against the address, because it proves +// the holder is a real client with a mistake to fix, and a success does not +// clear the count, because an attacker with one working key could then keep +// guessing for ever. +// +// Nothing here is keyed on anything an attacker can choose about someone else: +// a count belongs to the address that earned it, so refusals from one address +// never lock out another, and the check runs only when a connection opens, so +// a session already in progress is never ended by it. Clients sharing an +// address (behind one NAT) do share its count; that is the cost of the address +// being the only identity there is before a key has been checked. +// +// In memory and per process, like the sign-in throttle: a brake, not a record. +const ( + gatewayMaxFails = 30 + // gatewayWindow is both how long an address's invalid keys are counted and + // how often a refusing address can add a row to a log nothing trims: at + // most 288 rows a day per address, against thousands a second before. A + // client whose key was wrong is back within five minutes of fixing it. + gatewayWindow = 5 * time.Minute +) + +// gatewayGuard is the Gateway's throttle and its coalescer. +type gatewayGuard struct { + store *Store + limit *throttle + window time.Duration + now func() time.Time + + mu sync.Mutex + runs map[string]*refusalRun + timer *time.Timer // set while a run is waiting to be written + closed bool +} + +// refusalRun is one address's refusals of one sort in the current window: the +// event that opened it, and how many have been suppressed since. +type refusalRun struct { + first time.Time + suppressed int + kind, actor, subject, ip string + detail string +} + +func newGatewayGuard(s *Store) *gatewayGuard { + return &gatewayGuard{store: s, limit: newThrottle(gatewayMaxFails, gatewayWindow), + window: gatewayWindow, now: time.Now, runs: map[string]*refusalRun{}} +} + +// gatewayBucket is what an address is counted under. An IPv6 client is +// normally handed a whole /64, so counting single addresses would let it pick +// a fresh one for every attempt. +func gatewayBucket(ip string) string { + a, err := netip.ParseAddr(ip) + if err != nil { + return ip + } + a = a.Unmap() + if a.Is6() { + if p, err := a.Prefix(64); err == nil { + return p.String() + } + } + return a.String() +} + +// gatewayExempt is an address the throttle does not hold: this machine, whose +// one address is shared by every process on it and by anything forwarding +// connections to the Gateway, so holding it would let one of them lock out the +// rest. Its refusals are still coalesced. +func gatewayExempt(ip string) bool { + a, err := netip.ParseAddr(ip) + return err == nil && a.Unmap().IsLoopback() +} + +// GatewayThrottled reports whether an address has used up its invalid keys, +// and for how much longer. Called before a key is read: a held address costs +// no key lookup. A held attempt is added to the address's coalesced count. +func (s *Store) GatewayThrottled(database, ip string) (bool, time.Duration) { + if s == nil || s.gateway == nil || gatewayExempt(ip) { + return false, 0 + } + held, wait := s.gateway.limit.blocked(gatewayBucket(ip)) + if held { + s.gateway.note(EvGatewayRefused, "", database, ip, "invalid API key") + } + return held, wait +} + +// GatewayInvalidKey records a key that matched nothing: it counts towards the +// address's throttle, and the event is coalesced. +func (s *Store) GatewayInvalidKey(database, ip string) { + if s == nil || s.gateway == nil { + return + } + s.gateway.limit.fail(gatewayBucket(ip)) + s.gateway.note(EvGatewayRefused, "", database, ip, "invalid API key") +} + +// GatewayRefused records a valid key refused at the Gateway: coalesced like an +// invalid one, and never counted against the address. Runs are kept apart by +// kind and actor, so one person's refusals do not hide another's. +func (s *Store) GatewayRefused(kind, actor, subject, ip, detail string) { + if s == nil || s.gateway == nil { + return + } + s.gateway.note(kind, actor, subject, ip, detail) +} + +// note counts one refusal, and writes an event if it opens a window. +func (g *gatewayGuard) note(kind, actor, subject, ip, detail string) { + k := gatewayBucket(ip) + "\x00" + kind + "\x00" + actor + g.mu.Lock() + now := g.now() + r, ok := g.runs[k] + if ok && now.Sub(r.first) < g.window { + r.suppressed++ + g.mu.Unlock() + return + } + prior := 0 + if ok { + prior = r.suppressed + } + g.runs[k] = &refusalRun{first: now, kind: kind, actor: actor, subject: subject, ip: ip, detail: detail} + g.arm() + g.mu.Unlock() + g.store.Audit(kind, actor, subject, ip, coalescedDetail(detail, prior)) +} + +// coalescedDetail says how many refusals an event stands for beyond itself. +func coalescedDetail(detail string, suppressed int) string { + if suppressed == 0 { + return detail + } + return fmt.Sprintf("%s, and %d more refused from this address since the last record", detail, suppressed) +} + +// arm makes sure a timer is due to write what is waiting. The caller holds mu. +func (g *gatewayGuard) arm() { + if g.timer == nil && !g.closed { + g.timer = time.AfterFunc(g.window/4, g.tick) + } +} + +func (g *gatewayGuard) tick() { + g.flush(false) + g.mu.Lock() + defer g.mu.Unlock() + g.timer = nil + if len(g.runs) > 0 { + g.arm() + } +} + +// flush writes the suppressed counts of the windows that have ended, or of +// every window when all is set. A window that ended with nothing suppressed +// is forgotten; one that wrote a count opens the next window, so an address +// still gets at most one event per window however its refusals fall. +func (g *gatewayGuard) flush(all bool) { + g.mu.Lock() + now := g.now() + var due []refusalRun + for k, r := range g.runs { + if !all && now.Sub(r.first) < g.window { + continue + } + if r.suppressed == 0 { + delete(g.runs, k) + continue + } + due = append(due, *r) + r.first, r.suppressed = now, 0 + } + g.mu.Unlock() + for _, r := range due { + g.store.Audit(r.kind, r.actor, r.subject, r.ip, coalescedDetail(r.detail, r.suppressed)) + } +} + +// close writes what is waiting, while there is still a store to write it to. +func (g *gatewayGuard) close() { + g.mu.Lock() + g.closed = true + if g.timer != nil { + g.timer.Stop() + g.timer = nil + } + g.mu.Unlock() + g.flush(true) +} diff --git a/internal/auth/gateway_test.go b/internal/auth/gateway_test.go new file mode 100644 index 00000000..638ff8a3 --- /dev/null +++ b/internal/auth/gateway_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// A store whose gateway guard runs on a clock the test moves. +func guardedStore(t *testing.T) (*Store, *time.Time) { + t.Helper() + s := testStore(t) + now := time.Now() + clock := func() time.Time { return now } + s.gateway.now, s.gateway.limit.now = clock, clock + return s, &now +} + +func gatewayEvents(t *testing.T, s *Store) []SecurityEvent { + t.Helper() + evs, err := s.SecurityEvents(0, 0) + if err != nil { + t.Fatal(err) + } + return evs +} + +// One event per window however many refusals, the first when it happens, and +// the next one saying how many it stands for. +func TestGatewayRefusalsAreCoalesced(t *testing.T) { + s, now := guardedStore(t) + const ip = "203.0.113.7" + for i := 0; i < 1000; i++ { + s.GatewayInvalidKey("main", ip) + } + evs := gatewayEvents(t, s) + if len(evs) != 1 || evs[0].Kind != EvGatewayRefused || evs[0].IP != ip || evs[0].Detail != "invalid API key" { + t.Fatalf("after 1000 refusals in one window: %+v", evs) + } + *now = now.Add(gatewayWindow) + s.GatewayInvalidKey("main", ip) + evs = gatewayEvents(t, s) + if len(evs) != 2 || !strings.Contains(evs[1].Detail, "999 more") { + t.Fatalf("the next window's event: %+v", evs) + } + if id, why := CheckEventChain(evs); id != 0 { + t.Fatalf("the chain: %s", why) + } +} + +// An address that stops mid-window still gets its count written, without +// waiting for a refusal that may never come, and no extra event. +func TestGatewayCountIsWrittenWhenTheWindowEnds(t *testing.T) { + s, now := guardedStore(t) + for i := 0; i < 5; i++ { + s.GatewayInvalidKey("main", "203.0.113.7") + } + s.gateway.flush(false) + if n := len(gatewayEvents(t, s)); n != 1 { + t.Fatalf("written before the window ended: %d events", n) + } + *now = now.Add(gatewayWindow) + s.gateway.flush(false) + evs := gatewayEvents(t, s) + if len(evs) != 2 || !strings.Contains(evs[1].Detail, "4 more") { + t.Fatalf("after the window: %+v", evs) + } + // The count opened the next window: that one wrote nothing more, and it + // then goes away. + *now = now.Add(gatewayWindow) + s.gateway.flush(false) + if n := len(gatewayEvents(t, s)); n != 2 { + t.Fatalf("a quiet window added an event: %d", n) + } + if len(s.gateway.runs) != 0 { + t.Fatalf("a quiet address was remembered: %d runs", len(s.gateway.runs)) + } +} + +// Closing the store writes what was waiting. +func TestGatewayCountIsWrittenOnClose(t *testing.T) { + s, _ := guardedStore(t) + s.GatewayInvalidKey("main", "203.0.113.7") + s.GatewayInvalidKey("main", "203.0.113.7") + s.gateway.close() + evs := gatewayEvents(t, s) + if len(evs) != 2 || !strings.Contains(evs[1].Detail, "1 more") { + t.Fatalf("after close: %+v", evs) + } +} + +// A throttled address is held for its window, then let go, and its held +// attempts are counted without a row each. +func TestGatewayThrottleHoldsAnAddress(t *testing.T) { + s, now := guardedStore(t) + const ip = "203.0.113.7" + for i := 0; i < gatewayMaxFails; i++ { + if held, _ := s.GatewayThrottled("main", ip); held { + t.Fatalf("held after %d failures", i) + } + s.GatewayInvalidKey("main", ip) + } + for i := 0; i < 500; i++ { + if held, wait := s.GatewayThrottled("main", ip); !held || wait <= 0 { + t.Fatalf("not held once its failures were used up: %v %v", held, wait) + } + } + if n := len(gatewayEvents(t, s)); n != 1 { + t.Fatalf("held attempts wrote %d events", n) + } + *now = now.Add(gatewayWindow) + if held, _ := s.GatewayThrottled("main", ip); held { + t.Fatal("still held after its window") + } +} + +// One address's refusals cost nobody else anything: not another address, and +// not a run of refusals naming a database or an account. +func TestGatewayThrottleIsPerAddress(t *testing.T) { + s, _ := guardedStore(t) + for i := 0; i < gatewayMaxFails*2; i++ { + s.GatewayInvalidKey(fmt.Sprintf("db%d", i), "203.0.113.7") + } + if held, _ := s.GatewayThrottled("main", "203.0.113.8"); held { + t.Fatal("an attacker's refusals held another address") + } + if held, _ := s.GatewayThrottled("main", "2001:db8::1"); held { + t.Fatal("an attacker's refusals held another family's address") + } + if held, _ := s.GatewayThrottled("anything", "203.0.113.7"); !held { + t.Fatal("the attacker was not held whatever database it named") + } +} + +// A valid key that was refused is logged and coalesced, and is never held +// against the address: a client asking for a branch it may not open must not +// lose the branches it may. +func TestGatewayValidKeyRefusalsDoNotThrottle(t *testing.T) { + s, _ := guardedStore(t) + const ip = "203.0.113.7" + for i := 0; i < gatewayMaxFails*3; i++ { + s.GatewayRefused(EvDenied, "a@x.com", "secret", ip, "gateway") + s.GatewayRefused(EvGatewayRefused, "a@x.com", "secret", ip, "realtime key at the gateway") + } + if held, _ := s.GatewayThrottled("main", ip); held { + t.Fatal("refusals of a valid key throttled the address") + } + if n := len(gatewayEvents(t, s)); n != 2 { + t.Fatalf("want one event per kind in the window, got %d", n) + } + // One person's refusals do not hide another's. + s.GatewayRefused(EvDenied, "b@x.com", "secret", ip, "gateway") + if n := len(gatewayEvents(t, s)); n != 3 { + t.Fatalf("a second actor was folded into the first: %d events", n) + } +} + +// This machine's address is shared by everything on it and by anything +// forwarding to the Gateway, so it is coalesced and not held. +func TestGatewayDoesNotHoldLoopback(t *testing.T) { + s, _ := guardedStore(t) + for _, ip := range []string{"127.0.0.1", "::1"} { + for i := 0; i < gatewayMaxFails*2; i++ { + s.GatewayInvalidKey("main", ip) + } + if held, _ := s.GatewayThrottled("main", ip); held { + t.Errorf("%s was held", ip) + } + } + if n := len(gatewayEvents(t, s)); n != 2 { + t.Fatalf("loopback refusals: %d events, want one per address", n) + } +} + +func TestGatewayBucket(t *testing.T) { + for in, want := range map[string]string{ + "203.0.113.7": "203.0.113.7", + "::ffff:203.0.113.7": "203.0.113.7", + "2001:db8:1:2:3:4:5:6": "2001:db8:1:2::/64", + "2001:db8:1:2:aaaa:bbbb:c:d": "2001:db8:1:2::/64", + "2001:db8:1:3:aaaa:bbbb:c:d": "2001:db8:1:3::/64", + "not an address": "not an address", + } { + if got := gatewayBucket(in); got != want { + t.Errorf("gatewayBucket(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCheckEventChainFromTakesUpAfterAnAnchor(t *testing.T) { + s := testStore(t) + for i := 0; i < 6; i++ { + s.Audit(EvLoginOK, "a@x.com", "", "", "") + } + evs := gatewayEvents(t, s) + if id, why := CheckEventChainFrom(evs[3:], evs[2].ID, evs[2].RowHash); id != 0 { + t.Fatalf("a genuine tail: %s", why) + } + if id, _ := CheckEventChainFrom(evs[3:], evs[2].ID, "wrong"); id != evs[3].ID { + t.Fatalf("a tail not following the anchor: %d", id) + } + if id, _ := CheckEventChainFrom(evs[3:], evs[3].ID, evs[2].RowHash); id != evs[3].ID { + t.Fatalf("a tail with the wrong id: %d", id) + } +} diff --git a/internal/auth/pipelines.go b/internal/auth/pipelines.go index a94ff167..596fc429 100644 --- a/internal/auth/pipelines.go +++ b/internal/auth/pipelines.go @@ -141,3 +141,12 @@ func (s *Store) PipelineOwnsTarget(pipelineID, branch string) bool { err := s.db.QueryRow(`SELECT pipeline_id FROM pipeline_targets WHERE branch=?`, branch).Scan(&owner) return err == nil && owner == pipelineID } + +// PipelineMayReplace reports whether a run of pipelineID, which belongs to +// userID, may delete branch and make it again: the pipeline made the branch and +// the branch is still that account's. The claim alone is not enough, since a +// branch can change hands under the same name. +func (s *Store) PipelineMayReplace(pipelineID string, userID int64, branch string) bool { + owner, ok := s.BranchOwner(branch) + return ok && owner == userID && s.PipelineOwnsTarget(pipelineID, branch) +} diff --git a/internal/auth/setup_test.go b/internal/auth/setup_test.go index 4cd08b03..5806a70a 100644 --- a/internal/auth/setup_test.go +++ b/internal/auth/setup_test.go @@ -220,3 +220,55 @@ func TestPipelineTargets(t *testing.T) { t.Error("ownership reads back wrong") } } + +// A branch's name can be reused. A pipeline's claim used to outlive the branch, +// so its next run deleted whatever another account had since made under that +// name, and took it over. +func TestForgetBranchDropsThePipelineClaim(t *testing.T) { + s := testStore(t) + u, _ := s.CreateUser("a@x.com", "password1") + p1, _ := s.CreatePipeline(u.ID, "one", "{}") + p2, _ := s.CreatePipeline(u.ID, "two", "{}") + if err := s.ClaimPipelineTarget(p1.ID, "sales"); err != nil { + t.Fatal(err) + } + if err := s.ForgetBranch("sales"); err != nil { + t.Fatal(err) + } + if s.PipelineOwnsTarget(p1.ID, "sales") { + t.Error("the claim outlived the branch") + } + if err := s.ClaimPipelineTarget(p2.ID, "sales"); err != nil { + t.Errorf("the freed name could not be claimed: %v", err) + } +} + +func TestPipelineMayReplace(t *testing.T) { + s := testStore(t) + a, _ := s.CreateUser("a@x.com", "password1") + b, _ := s.CreateUser("b@x.com", "password1") + p, _ := s.CreatePipeline(a.ID, "one", "{}") + other, _ := s.CreatePipeline(a.ID, "two", "{}") + + if s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("allowed with neither a claim nor an owner") + } + if err := s.ClaimPipelineTarget(p.ID, "sales"); err != nil { + t.Fatal(err) + } + if s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("allowed on a branch with no recorded owner") + } + _ = s.SetBranchOwner("sales", a.ID) + if !s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("a pipeline could not refresh its own branch") + } + if s.PipelineMayReplace(other.ID, a.ID, "sales") { + t.Error("allowed for a pipeline that did not make the branch") + } + // The name changed hands while the claim stayed. + _ = s.SetBranchOwner("sales", b.ID) + if s.PipelineMayReplace(p.ID, a.ID, "sales") { + t.Error("allowed on a branch another account now owns") + } +} diff --git a/internal/branch/auditevidence.go b/internal/branch/auditevidence.go index e1764b45..577c10f6 100644 --- a/internal/branch/auditevidence.go +++ b/internal/branch/auditevidence.go @@ -143,6 +143,7 @@ var ( "SECURITY DEFINER functions are found by a word match on their source: it can name too many, and misses a table name built at run time.", "A record says who and what, and never what a statement held; in forensic mode its text is kept apart from the chain and expires.", "The anchors prove the records against an edit made after they were anchored; records collected since the last anchor rest on the chain alone.", + "A record of the period collected more than a week after it ended, and after a whole later day of newer records had been collected, is not counted here. The chain was still verified over it.", } ) @@ -182,32 +183,104 @@ func EvidenceSummary(ev AuditEvidence) string { return s } +// auditCollectedLateDays is how many days past a period's end its records are +// still looked for, however little those days hold of it. A day file holds what +// was collected that day, and that can be days after a record happened. +const auditCollectedLateDays = 7 + // AuditRecordsBetween reads the records collected in [since, until), in // order, without statement text. func AuditRecordsBetween(since, until time.Time) ([]AuditRecord, error) { var out []AuditRecord + err := ScanAuditRecordsBetween(since, until, func(r AuditRecord) { out = append(out, r) }) + if err != nil { + return nil, err + } + return out, nil +} + +// ScanAuditRecordsBetween hands the records in [since, until) to fn in order, +// without statement text, holding one day at a time at most and none of it +// once fn has been called. +// +// It stops once the period's end is well behind: past auditCollectedLateDays, +// at the first day file with nothing from before until in it. A week is a wide +// margin for a collector that was stopped, and the rule extends itself past +// that when a collector has been down for longer, because the first day it +// runs again still holds records of the period. What it cannot find is a +// record that arrives after a whole later day of newer ones; EvidenceLimits +// says so. +func ScanAuditRecordsBetween(since, until time.Time, fn func(AuditRecord)) error { + first := since.UTC().AddDate(0, 0, -1).Format("2006-01-02") + carry := until.UTC().AddDate(0, 0, auditCollectedLateDays).Format("2006-01-02") for _, day := range storedDays() { // A day file holds what was collected that day, so one before the // period's first day holds nothing in it. - if day < since.UTC().AddDate(0, 0, -1).Format("2006-01-02") { + if day < first { continue } - recs, err := readDay(day) - if err != nil { - return nil, fmt.Errorf("the records of %s cannot all be read (%v)", day, err) - } - for _, r := range recs { + before := false + err := scanDay(day, func(r AuditRecord) { // Parsed, not compared as text: RFC 3339 drops trailing zeros, // so "…01.5Z" sorts after "…01.55Z". at, err := time.Parse(time.RFC3339Nano, r.Time) - if err != nil || at.Before(since) || !at.Before(until) { - continue + if err != nil { + return + } + if at.Before(until) { + before = true + if !at.Before(since) { + r.Text = "" + fn(r) + } } - r.Text = "" + }) + if err != nil { + return fmt.Errorf("the records of %s cannot all be read (%v)", day, err) + } + if day > carry && !before { + break + } + } + return nil +} + +// TallyAuditActivity adds one record to a period's counts. +func TallyAuditActivity(a *EvidenceActivity, r AuditRecord) { + a.Records++ + a.ByKind[r.Kind]++ + a.ByBranch[r.Branch]++ + switch r.Kind { + case "outcome": + a.Errors++ + case "heartbeat": + a.Heartbeats++ + case "gap": + a.Gaps = append(a.Gaps, r) + } +} + +// NewAuditActivity is an empty EvidenceActivity, every part present. +func NewAuditActivity() EvidenceActivity { + return EvidenceActivity{ByKind: map[string]int{}, ByBranch: map[string]int{}, Gaps: []AuditRecord{}} +} + +// AuditActivityBetween counts the period's records in one streaming pass, and +// returns the records themselves only when asked for them: a period's counts +// cost what the counts hold, not what the records do. +func AuditActivityBetween(since, until time.Time, records bool) (EvidenceActivity, []AuditRecord, error) { + a := NewAuditActivity() + var out []AuditRecord + err := ScanAuditRecordsBetween(since, until, func(r AuditRecord) { + TallyAuditActivity(&a, r) + if records { out = append(out, r) } + }) + if err != nil { + return EvidenceActivity{}, nil, err } - return out, nil + return a, out, nil } // AuditBranchCoverage reads a running branch's marks, the definer functions diff --git a/internal/branch/auditread_bounds_test.go b/internal/branch/auditread_bounds_test.go new file mode 100644 index 00000000..3d0ffe62 --- /dev/null +++ b/internal/branch/auditread_bounds_test.go @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "fmt" + "reflect" + "testing" + "time" +) + +// oldReadAudit is ReadAudit before it stopped re-joining everything per day, +// kept to prove the new one returns the same answers. +func oldReadAudit(q AuditQuery) ([]AuditEntry, error) { + if q.Limit <= 0 { + q.Limit = 100 + } + days := storedDays() + var all []AuditRecord + var out []AuditEntry + for i := len(days) - 1; i >= 0 && len(out) < q.Limit; i-- { + if !q.Since.IsZero() && days[i] < q.Since.UTC().Format("2006-01-02") { + break + } + recs, err := readDay(days[i]) + if err != nil { + return nil, err + } + all = append(recs, all...) + out = joinAudit(all, q) + } + if len(out) > q.Limit { + out = out[:q.Limit] + } + attachTexts(out) + return out, nil +} + +// oldAuditRecordsBetween is AuditRecordsBetween before it stopped at until. +func oldAuditRecordsBetween(since, until time.Time) ([]AuditRecord, error) { + var out []AuditRecord + for _, day := range storedDays() { + if day < since.UTC().AddDate(0, 0, -1).Format("2006-01-02") { + continue + } + recs, err := readDay(day) + if err != nil { + return nil, fmt.Errorf("the records of %s cannot all be read (%v)", day, err) + } + for _, r := range recs { + at, err := time.Parse(time.RFC3339Nano, r.Time) + if err != nil || at.Before(since) || !at.Before(until) { + continue + } + r.Text = "" + out = append(out, r) + } + } + return out, nil +} + +func stmtLine(at, session, vxid, user, table string) string { + return fmt.Sprintf(`{"timestamp":"%s UTC","user":"%s","dbname":"appdb","pid":7,"session_id":"%s","line_num":1,"ps":"INSERT","vxid":"%s","error_severity":"LOG","message":"AUDIT: OBJECT,1,1,WRITE,INSERT,TABLE,%s,,"}`, + at, user, session, vxid, table) +} + +func txnLine(at, session, vxid, outcome string) string { + return fmt.Sprintf(`{"timestamp":"%s UTC","user":"alice@x.in","session_id":"%s","vxid":"4/0","error_severity":"LOG","message":"AUDIT: TXN,%s,%s,asserted,shop,\"o'brien, p\""}`, + at, session, vxid, outcome) +} + +func errorLine(at, session, vxid, state string) string { + return fmt.Sprintf(`{"timestamp":"%s UTC","user":"alice@x.in","session_id":"%s","vxid":"%s","error_severity":"ERROR","state_code":"%s","message":"AUDIT: OUTCOME,ERROR,%s"}`, + at, session, vxid, state, state) +} + +// seedAuditFixture is a store with what makes the join and the day boundaries +// interesting: transactions spanning days, a statement filed after the +// transaction that closed it, one key finished twice, errors, and several +// logins and tables. +func seedAuditFixture(t *testing.T) { + t.Helper() + storeIn(t) + settingsIn(t) + // A long transaction: the statement on the 1st, its commit on the 4th. + seed(t, "2026-11-01", + stmtLine("2026-11-01 09:00:00.000", "s1", "1/1", "alice@x.in", "public.orders"), + stmtLine("2026-11-01 09:00:01.000", "s2", "2/1", "bob", "public.users"), + txnLine("2026-11-01 09:00:02.000", "s2", "2/1", "ROLLBACK"), + errorLine("2026-11-01 09:00:03.000", "s2", "2/2", "23505"), + ) + // A transaction filed a day before the statement it closes, as a late raw + // file does; and the same key finished twice, where the last one wins. + seed(t, "2026-11-02", + txnLine("2026-11-02 23:59:58.000", "s3", "3/1", "COMMIT"), + stmtLine("2026-11-02 12:00:00.000", "s4", "4/1", "alice@x.in", "public.orders"), + txnLine("2026-11-02 12:00:01.000", "s4", "4/1", "ROLLBACK"), + stmtLine("2026-11-02 12:00:02.000", "s5", "5/1", "bob", "public.users"), + ) + seed(t, "2026-11-03", + stmtLine("2026-11-02 23:59:59.000", "s3", "3/1", "alice@x.in", "public.orders"), // filed after its transaction + stmtLine("2026-11-03 10:00:00.000", "s5", "5/2", "alice@x.in", "public.users"), + errorLine("2026-11-03 10:00:01.000", "s5", "5/2", "42501"), + txnLine("2026-11-03 10:00:02.000", "s4", "4/1", "COMMIT"), // the same key again + ) + seed(t, "2026-11-04", + txnLine("2026-11-04 08:00:00.000", "s1", "1/1", "COMMIT"), // closes the 1st's statement + stmtLine("2026-11-04 08:00:01.000", "s6", "6/1", "alice@x.in", "public.orders"), + txnLine("2026-11-04 08:00:02.000", "s6", "6/1", "COMMIT"), + errorLine("2026-11-04 08:00:03.000", "s7", "7/1", "22P02"), + stmtLine("2026-11-04 08:00:04.000", "s7", "7/2", "bob", "public.users"), + ) +} + +// The new read against the old one, over filters and limits falling on and +// around every day boundary. +func TestReadAuditMatchesTheOldRead(t *testing.T) { + seedAuditFixture(t) + since := time.Date(2026, 11, 2, 12, 0, 1, 0, time.UTC) + queries := []AuditQuery{ + {}, {Limit: 1}, {Limit: 2}, {Limit: 3}, {Limit: 4}, {Limit: 5}, {Limit: 6}, {Limit: 7}, {Limit: 8}, + {Limit: 9}, {Limit: 10}, {Limit: 1000}, + {Login: "alice@x.in"}, {Login: "alice@x.in", Limit: 2}, {Login: "alice@x.in", Limit: 3}, + {Login: "nobody"}, + {Table: "orders"}, {Table: "orders", Limit: 1}, {Table: "public.users", Limit: 2}, + {Branch: "main", Limit: 3}, {Branch: "other"}, + {Since: since}, {Since: since, Limit: 2}, {Since: since, Table: "users"}, + {Kinds: []string{"outcome"}}, {Kinds: []string{"outcome"}, Limit: 1}, + {Kinds: []string{"transaction"}, Limit: 3}, {Kinds: []string{"statement", "outcome"}, Limit: 5}, + } + for _, q := range queries { + old, oldErr := oldReadAudit(q) + got, err := ReadAudit(q) + if err != nil || oldErr != nil { + t.Fatalf("%+v: %v / %v", q, err, oldErr) + } + if !reflect.DeepEqual(got, old) { + t.Errorf("%+v:\n got %+v\nwant %+v", q, got, old) + } + } + // The cases the fixture exists for, stated, so they cannot pass vacuously. + all, err := ReadAudit(AuditQuery{Limit: 1000}) + if err != nil { + t.Fatal(err) + } + find := func(session, vxid string) AuditEntry { + for _, e := range all { + if e.Session == session && e.VXID == vxid && e.Kind == "statement" { + return e + } + } + t.Fatalf("no statement %s %s", session, vxid) + return AuditEntry{} + } + if e := find("s1", "1/1"); e.TxnOutcome != "COMMIT" { + t.Errorf("a transaction closed three days later: %+v", e) + } + if e := find("s3", "3/1"); e.TxnOutcome != "COMMIT" { + t.Errorf("a transaction filed before its statement: %+v", e) + } + if e := find("s4", "4/1"); e.TxnOutcome != "COMMIT" { + t.Errorf("the last record of a key did not win: %+v", e) + } + if e := find("s5", "5/2"); e.Error != "42501" { + t.Errorf("an error on a statement: %+v", e) + } +} + +// The records and the counts against the old read, including a period whose +// days are not all before its end. +func TestEvidenceReadMatchesTheOldRead(t *testing.T) { + seedAuditFixture(t) + periods := [][2]time.Time{ + {time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 12, 1, 0, 0, 0, 0, time.UTC)}, + {time.Date(2026, 11, 2, 12, 0, 1, 0, time.UTC), time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}, + {time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC), time.Date(2026, 11, 3, 10, 0, 1, 0, time.UTC)}, + {time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 11, 1, 9, 0, 2, 0, time.UTC)}, + {time.Date(2027, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2027, 1, 2, 0, 0, 0, 0, time.UTC)}, + } + for _, p := range periods { + old, err := oldAuditRecordsBetween(p[0], p[1]) + if err != nil { + t.Fatal(err) + } + got, err := AuditRecordsBetween(p[0], p[1]) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, old) { + t.Errorf("%v: records differ:\n got %+v\nwant %+v", p, got, old) + } + want := NewAuditActivity() + for _, r := range old { + TallyAuditActivity(&want, r) + } + a, none, err := AuditActivityBetween(p[0], p[1], false) + if err != nil || none != nil { + t.Fatalf("%v: records came back unasked: %d, %v", p, len(none), err) + } + if !reflect.DeepEqual(a, want) { + t.Errorf("%v: counts differ:\n got %+v\nwant %+v", p, a, want) + } + _, kept, err := AuditActivityBetween(p[0], p[1], true) + if err != nil || len(kept) != len(old) { + t.Fatalf("%v: asked for the records: %d of %d, %v", p, len(kept), len(old), err) + } + } +} + +// countScans wraps scanDay, counting the days opened and the records handed +// over, which is the work a read does. +func countScans(t *testing.T) (days, records *int) { + t.Helper() + old := scanDay + days, records = new(int), new(int) + scanDay = func(day string, fn func(AuditRecord)) error { + *days++ + return old(day, func(r AuditRecord) { *records++; fn(r) }) + } + t.Cleanup(func() { scanDay = old }) + return days, records +} + +// seedDays is a store of n days, each of perDay statements, with one +// statement a filter can find on the very first day only. +func seedDays(t *testing.T, n, perDay int) { + t.Helper() + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for d := 0; d < n; d++ { + day := start.AddDate(0, 0, d) + lines := make([]string, 0, perDay) + for i := 0; i < perDay; i++ { + at := day.Add(time.Duration(i) * time.Second).Format("2006-01-02 15:04:05.000") + table := "public.common" + if d == 0 && i == 0 { + table = "public.rare" + } + lines = append(lines, stmtLine(at, "s", fmt.Sprintf("%d/%d", d, i), "alice@x.in", table)) + } + seed(t, day.Format("2006-01-02"), lines...) + } +} + +// Quadratic growth shows up as records handled per record stored rising with +// the number of days. A rare match walks every day, the worst case, and costs +// two passes over them at most; the old read handled each day once per day +// that followed it, and the fixture asserts that really was growth. +func TestReadAuditWorkIsLinearInTheDaysRead(t *testing.T) { + storeIn(t) + settingsIn(t) + const n, perDay = 40, 10 + seedDays(t, n, perDay) + days, records := countScans(t) + got, err := ReadAudit(AuditQuery{Table: "rare", Limit: 5}) + if err != nil || len(got) != 1 { + t.Fatalf("%d, %v", len(got), err) + } + if want := 2 * n * perDay; *records > want { + t.Errorf("handled %d records of %d stored, want at most %d", *records, n*perDay, want) + } + if *days > 2*n { + t.Errorf("opened %d day files of %d", *days, n) + } + // What the old read handled, by construction: every day re-joined once per + // day walked after it. + if old := perDay * n * (n + 1) / 2; old <= *records { + t.Errorf("the fixture does not separate the two: old %d, new %d", old, *records) + } + + // Most of the time nothing goes back further than it must. + *days, *records = 0, 0 + if _, err := ReadAudit(AuditQuery{Limit: 3}); err != nil { + t.Fatal(err) + } + if *days > 2 { + t.Errorf("a limit met by the newest day opened %d day files", *days) + } +} + +// Held memory is bounded by Limit, even for matches that fill a whole day. +func TestReadAuditHoldsAtMostLimit(t *testing.T) { + storeIn(t) + settingsIn(t) + seedDays(t, 3, 500) + got, err := ReadAudit(AuditQuery{Limit: 7}) + if err != nil { + t.Fatal(err) + } + if len(got) != 7 || got[0].VXID != "2/499" || got[6].VXID != "2/493" { + t.Fatalf("the newest seven: %d, %+v", len(got), got) + } +} + +// Stops at until, plus the days a record can be collected late. Days after +// that are not opened at all, but a period's record collected late is found +// however late, as long as the day it arrives in still holds such a record. +func TestEvidenceReadStopsAfterThePeriod(t *testing.T) { + storeIn(t) + settingsIn(t) + seedDays(t, 40, 3) + since := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + until := time.Date(2026, 1, 4, 0, 0, 0, 0, time.UTC) + days, _ := countScans(t) + recs, err := AuditRecordsBetween(since, until) + if err != nil { + t.Fatal(err) + } + if len(recs) != 6 { + t.Fatalf("records: %d", len(recs)) + } + // from the day before since, through until and its carry-over, and the one + // day that shows the period is behind us. + if want := 2 + 2 + auditCollectedLateDays + 1; *days > want { + t.Errorf("opened %d day files, want at most %d", *days, want) + } +} + +func TestEvidenceReadFindsRecordsCollectedLate(t *testing.T) { + storeIn(t) + settingsIn(t) + // Nothing was collected for three weeks after the period's end, and then + // the backlog arrived. + seed(t, "2026-03-01", stmtLine("2026-03-01 10:00:00.000", "s", "1/1", "alice@x.in", "public.t")) + seed(t, "2026-03-22", stmtLine("2026-03-01 23:59:59.000", "s", "1/2", "alice@x.in", "public.t"), + stmtLine("2026-03-21 12:00:00.000", "s", "1/3", "alice@x.in", "public.t")) + since := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + until := time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC) + got, err := AuditRecordsBetween(since, until) + if err != nil { + t.Fatal(err) + } + old, _ := oldAuditRecordsBetween(since, until) + if len(got) != 2 || !reflect.DeepEqual(got, old) { + t.Fatalf("late records:\n got %+v\nwant %+v", got, old) + } +} + +// BenchmarkReadAudit compares the old read with the new on a rare match that +// walks every day: the old grows with the square of the days, the new with +// the days. +func BenchmarkReadAudit(b *testing.B) { + for _, n := range []int{10, 40, 160} { + dir := b.TempDir() + old := AuditStoreDir + AuditStoreDir = func() string { return dir } + defer func() { AuditStoreDir = old }() + seedBenchDays(b, n, 200) + q := AuditQuery{Table: "rare", Limit: 5} + b.Run(fmt.Sprintf("new/days=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := ReadAudit(q); err != nil { + b.Fatal(err) + } + } + }) + b.Run(fmt.Sprintf("old/days=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := oldReadAudit(q); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func seedBenchDays(b *testing.B, n, perDay int) { + b.Helper() + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for d := 0; d < n; d++ { + day := start.AddDate(0, 0, d) + recs := make([]AuditRecord, 0, perDay) + for i := 0; i < perDay; i++ { + r, ok := recordFromLine("main", []byte(stmtLine(day.Add(time.Duration(i)*time.Second).Format("2006-01-02 15:04:05.000"), + "s", fmt.Sprintf("%d/%d", d, i), "alice@x.in", "public.common"))) + if !ok { + b.Fatal("not a record") + } + if d == 0 && i == 0 { + r.Object = "public.rare" + } + recs = append(recs, r) + } + if _, err := appendDay(day.Format("2006-01-02"), recs); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/branch/auditstore.go b/internal/branch/auditstore.go index 99bcdabf..fee0a7ec 100644 --- a/internal/branch/auditstore.go +++ b/internal/branch/auditstore.go @@ -207,31 +207,40 @@ func appendDay(day string, recs []AuditRecord) (int64, error) { return st.Size(), nil } -// readDay reads a day's records. +// readDay reads a day's records. On an error it returns those it got to, which +// is how verify says how far a damaged file still reads. func readDay(day string) ([]AuditRecord, error) { + var out []AuditRecord + err := scanDay(day, func(r AuditRecord) { out = append(out, r) }) + return out, err +} + +// scanDay hands a day's records to fn in file order, one at a time, so a reader +// that does not need them all does not hold them. A variable so a test can +// count what a read costs. +var scanDay = func(day string, fn func(AuditRecord)) error { f, err := os.Open(dayPath(day)) if err != nil { - return nil, err + return err } defer f.Close() zr, err := gzip.NewReader(f) if err != nil { if err == io.EOF { - return nil, nil + return nil } - return nil, err + return err } defer zr.Close() - var out []AuditRecord dec := json.NewDecoder(zr) for { var r AuditRecord if err := dec.Decode(&r); err == io.EOF { - return out, nil + return nil } else if err != nil { - return out, fmt.Errorf("%s: %w", day, err) + return fmt.Errorf("%s: %w", day, err) } - out = append(out, r) + fn(r) } } @@ -524,26 +533,76 @@ type AuditEntry struct { // ReadAudit reads the newest records that match, newest first: statements // joined to their transaction's outcome and whom it was for, and the // outcome records of errors outside any audited statement. +// +// Two passes over the days, each a day at a time, so the work grows with the +// records read and what is held with Limit, however far back a rare match +// makes it look. The first walks back from the newest day choosing the +// records to show and stops once it has Limit of them. The second reads those +// same days again for the transactions and errors of the statements chosen. +// +// The second pass, and not a window carried across day boundaries, because a +// statement's transaction can be filed on any day the walk read, in either +// direction: a transaction can run for days, and a late raw file can file a +// statement after the transaction that closed it. A window of any fixed width +// would give a different answer for those. Knowing the statements first means +// only their keys are looked for, at most Limit of them. func ReadAudit(q AuditQuery) ([]AuditEntry, error) { if q.Limit <= 0 { q.Limit = 100 } + match := q.matcher() days := storedDays() - var all []AuditRecord - var out []AuditEntry - for i := len(days) - 1; i >= 0 && len(out) < q.Limit; i-- { + var picked []AuditRecord // newest first, at most Limit + oldest := len(days) + for i := len(days) - 1; i >= 0 && len(picked) < q.Limit; i-- { if !q.Since.IsZero() && days[i] < q.Since.UTC().Format("2006-01-02") { break } - recs, err := readDay(days[i]) + // A day is walked backwards, so only its last matches can be shown. + need := q.Limit - len(picked) + var day []AuditRecord + err := scanDay(days[i], func(r AuditRecord) { + if !match(r) { + return + } + if len(day) == 2*need { + day = day[:copy(day, day[need:])] + } + day = append(day, r) + }) if err != nil { return nil, err } - all = append(recs, all...) - out = joinAudit(all, q) + if len(day) > need { + day = day[len(day)-need:] + } + for j := len(day) - 1; j >= 0; j-- { + picked = append(picked, day[j]) + } + oldest = i + } + join := newAuditJoin() + statements := map[auditKey]bool{} + for _, r := range picked { + if r.Kind == "statement" { + statements[keyOf(r)] = true + } + } + if len(statements) > 0 { + for _, day := range days[oldest:] { // oldest first: the last record of a key wins + err := scanDay(day, func(r AuditRecord) { + if statements[keyOf(r)] { + join.add(r) + } + }) + if err != nil { + return nil, err + } + } } - if len(out) > q.Limit { - out = out[:q.Limit] + var out []AuditEntry + for _, r := range picked { + out = append(out, join.entry(r)) } attachTexts(out) return out, nil @@ -551,56 +610,87 @@ func ReadAudit(q AuditQuery) ([]AuditEntry, error) { // joinAudit is ReadAudit's work on records in hand, pure so it is tested. func joinAudit(recs []AuditRecord, q AuditQuery) []AuditEntry { - type key struct{ session, vxid string } - txn := map[key]AuditRecord{} - errs := map[key]string{} + join := newAuditJoin() for _, r := range recs { - k := key{r.Session, r.VXID} - switch { - case r.Kind == "transaction" && r.Outcome != "SUBROLLBACK": - txn[k] = r - case r.Kind == "outcome" && r.SQLState != "": - errs[k] = r.SQLState + join.add(r) + } + match := q.matcher() + var out []AuditEntry + for i := len(recs) - 1; i >= 0; i-- { + if match(recs[i]) { + out = append(out, join.entry(recs[i])) } } + return out +} + +// auditKey is what joins a statement to its transaction's records. +type auditKey struct{ session, vxid string } + +func keyOf(r AuditRecord) auditKey { return auditKey{r.Session, r.VXID} } + +// auditJoin is how each transaction ended and which error it hit, by key. +type auditJoin struct { + txn map[auditKey]AuditRecord + errs map[auditKey]string +} + +func newAuditJoin() auditJoin { + return auditJoin{txn: map[auditKey]AuditRecord{}, errs: map[auditKey]string{}} +} + +// add takes in a record, if it says how a transaction went. +func (j auditJoin) add(r AuditRecord) { + switch { + case r.Kind == "transaction" && r.Outcome != "SUBROLLBACK": + j.txn[keyOf(r)] = r + case r.Kind == "outcome" && r.SQLState != "": + j.errs[keyOf(r)] = r.SQLState + } +} + +// entry is a record as people read it, joined to what was added. +func (j auditJoin) entry(r AuditRecord) AuditEntry { + e := AuditEntry{AuditRecord: r} + if r.Kind == "statement" { + if t, ok := j.txn[keyOf(r)]; ok { + e.TxnOutcome, e.TxnFor = t.Outcome, t.For + } + e.Error = j.errs[keyOf(r)] + } + return e +} + +// matcher is the query's filter over one record. +func (q AuditQuery) matcher() func(AuditRecord) bool { want := map[string]bool{} for _, k := range q.Kinds { want[k] = true } - var out []AuditEntry - for i := len(recs) - 1; i >= 0; i-- { - r := recs[i] + return func(r AuditRecord) bool { if len(want) > 0 && !want[r.Kind] { - continue + return false } if len(want) == 0 && (r.Kind == "transaction" || r.Kind == "heartbeat") { - continue // transactions are shown on the statements they close; heartbeats are the engine's + return false // transactions are shown on the statements they close; heartbeats are the engine's } if q.Branch != "" && r.Branch != q.Branch { - continue + return false } if q.Login != "" && r.Login != q.Login { - continue + return false } if q.Table != "" && !tableMatches(r.Object, q.Table) { - continue + return false } if !q.Since.IsZero() { // Parsed: as text, "…01.5Z" sorts after "…01.55Z". if at, err := time.Parse(time.RFC3339Nano, r.Time); err == nil && at.Before(q.Since) { - continue + return false } } - e := AuditEntry{AuditRecord: r} - if t, ok := txn[key{r.Session, r.VXID}]; ok && r.Kind == "statement" { - e.TxnOutcome, e.TxnFor = t.Outcome, t.For - } - if r.Kind == "statement" { - e.Error = errs[key{r.Session, r.VXID}] - } - out = append(out, e) + return true } - return out } // attachTexts gives each forensic record its statement text, while it is diff --git a/internal/branch/branch.go b/internal/branch/branch.go index 8c6e95c0..be7d33e0 100644 --- a/internal/branch/branch.go +++ b/internal/branch/branch.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/jackc/pgx/v5" "github.com/thefoxbyte/foxbyte/internal/ledger" "github.com/thefoxbyte/foxbyte/internal/secrets" ) @@ -173,8 +174,13 @@ func sudoCmd(d time.Duration, name string, args ...string) (*exec.Cmd, context.C ctx, cancel := context.WithTimeout(context.Background(), t) cmd := exec.CommandContext(ctx, "sudo", full...) // Without this, a killed child that leaves a pipe open keeps Wait blocked — - // the hang this is meant to end, one level down. + // the hang this is meant to end, one level down. Killing sudo does not kill + // the docker client under it, so a hung dockerd reaches this every time, and + // a probe measured in seconds must not then wait ten more. cmd.WaitDelay = 10 * time.Second + if t <= execProbe { + cmd.WaitDelay = time.Second + } return cmd, ctx, cancel } @@ -189,6 +195,7 @@ func execErr(ctx context.Context, d time.Duration, name string, err error) error // run executes a privileged command and streams output to the terminal. func run(name string, args ...string) error { + defer noteIfChange(name, args) cmd, ctx, cancel := sudoCmd(execLong, name, args...) defer cancel() cmd.Stdout = os.Stdout @@ -199,6 +206,7 @@ func run(name string, args ...string) error { // quiet executes a privileged command, discarding output and errors. Used for // best-effort cleanup where "not found" is not a failure. func quiet(name string, args ...string) { + defer noteIfChange(name, args) cmd, _, cancel := sudoCmd(execClean, name, args...) defer cancel() _ = cmd.Run() @@ -228,6 +236,7 @@ func execSudo(name string, args ...string) *exec.Cmd { // text, which is what it echoes on failure, carries no secret. Checked rather // than assumed: no capture call passes a password, a conninfo or a secret. func capture(name string, args ...string) (string, error) { + defer noteIfChange(name, args) cmd, ctx, cancel := sudoCmd(execQuick, name, args...) defer cancel() out, err := cmd.Output() @@ -819,6 +828,7 @@ func LedgerVerify(name string) (string, error) { // captureCombined runs a privileged command and returns its combined // stdout+stderr, so a failing psql includes the server's error text. func captureCombined(name string, args ...string) (string, error) { + defer noteIfChange(name, args) cmd, ctx, cancel := sudoCmd(execQuick, name, args...) defer cancel() out, err := cmd.CombinedOutput() @@ -868,6 +878,9 @@ func Create(name, parent string) error { if err := checkName(parent); err != nil { return err } + // Last, as well as at each docker command: the dataset is part of what the + // console lists. + defer noteStateChange() forgetBranchRoles(name) forgetCatalogScan(name) // Cloning a branch that doesn't exist fails deep in the storage layer with a @@ -878,6 +891,9 @@ func Create(name, parent string) error { if activeStorage().exists(name) { return fmt.Errorf("branch %q already exists", name) } + if err := checkNoContainer(name, ContainerState(name)); err != nil { + return err + } // A masked branch being made, or one that failed, is copied by nobody; // a copy of an open one is masked too (maskedbranch.go). if err := inheritMasked(name, parent); err != nil { @@ -918,6 +934,7 @@ func Delete(name string) error { if name == "main" { return fmt.Errorf("refusing to delete the primary branch 'main'") } + defer noteStateChange() // after the dataset goes, which the console lists forgetBranchRoles(name) forgetCatalogScan(name) forgetBranchUnmasks(name) @@ -1279,8 +1296,8 @@ func agentBranch(id string) string { return "agent-" + id } var ErrBadName = errors.New("invalid name: use letters, digits, '.', '-' and '_' (at most 63, starting with a letter or digit)") // safeNameRe is every name the engine passes to Docker, ZFS or btrfs. Stricter -// rules for new names live where names are made (the control plane's nameRe); -// this is the floor under every path that acts on an existing one. A '/' would +// rules for new names live where names are made (the control plane's nameRe, +// and CheckNewName); this is the floor under every path that acts on an existing one. A '/' would // walk out of the branch directory on btrfs (filepath.Join), '@' would name a // ZFS snapshot of another dataset, and "..", a space or a leading '-' mean // something to one tool or another. @@ -1293,6 +1310,49 @@ func checkName(name string) error { return nil } +// ErrReservedName is returned for a new branch named like one the engine makes +// for itself. +var ErrReservedName = errors.New("reserved branch name") + +// engineName reports whether name is one the engine's own containers and +// branches use: the primary, the HA standby, the two restores, an agent's. +func engineName(name string) bool { + switch name { + case "main", "standby", "restore", restoreCheckContainer: + return true + } + return strings.HasPrefix(name, "agent-") +} + +// CheckNewName is the rule for a name a person or an account picks. A branch's +// container is named after it and creating one starts by removing whatever +// holds that name, so a branch called "standby" used to kill the live HA +// standby. The engine makes these names itself by calling Create directly, and +// checkName must go on accepting them, so the refusal belongs here, where names +// come in from outside. Compared in lower case: an import target is not held to +// the control plane's lower-case nameRe, and Agent-x is still an agent's name +// to anyone reading it. +func CheckNewName(name string) error { + if err := checkName(name); err != nil { + return err + } + if n := strings.ToLower(name); engineName(n) || strings.Contains(n, "-before-") { + return fmt.Errorf("%w: %q is kept for the engine (main, standby, restore, restore-check, agent-*, *-before-*), choose another name", ErrReservedName, name) + } + return nil +} + +// checkNoContainer refuses a new branch whose container name is already taken, +// given that container's state: the storage check cannot see a container that +// has no branch behind it, like the standby's or a restore's, and starting the +// new one would remove it. +func checkNoContainer(name, state string) error { + if state == "absent" { + return nil + } + return fmt.Errorf("%w: a container named %s is already there (%s), and making this branch would remove it", ErrBranchExists, container(name), state) +} + // Exists reports whether a branch's data exists, running or not. func Exists(name string) bool { return checkName(name) == nil && activeStorage().exists(name) } @@ -1440,6 +1500,9 @@ func DeleteAgentBranch(agentID string) error { // BackendAddr returns host:port where a branch's Postgres is reachable, used by // the wire-protocol proxy to route connections. For "main" it resolves the // container currently serving as primary (which changes after an HA failover). +// +// PrimaryContainer is read each time and the memory is per container, so a +// failover run by another process moves "main" at once. func BackendAddr(name string) (string, error) { if name == "" { name = "main" @@ -1448,11 +1511,14 @@ func BackendAddr(name string) (string, error) { if name == "main" { cont = PrimaryContainer() } - ip, err := containerIP(cont) + f, err := containers.lookup(cont) if err != nil { return "", err } - return ip + ":5432", nil + if f.ip == "" { + return "", fmt.Errorf("container %q has no IP on the %q network", cont, network) + } + return f.addr(), nil } // primaryFile records which branch container currently serves as the "main" @@ -1477,6 +1543,7 @@ func primaryBranch() string { // setPrimary records the branch name now serving as primary. func setPrimary(name string) error { + defer noteStateChange() _ = os.MkdirAll(filepath.Dir(primaryFile()), 0o755) return os.WriteFile(primaryFile(), []byte(name), 0o644) } @@ -1498,37 +1565,94 @@ type BranchInfo struct { // Branches returns structured info for every branch (including main), enriched // with container state, connections, and copy-on-write size. +// +// One docker inspect for every branch's state and address, then each running +// branch's connection count, several at a time. A docker that does not answer +// shows every branch as absent, as it always did: this is a report, and nothing +// that removes a container reads it. func Branches() ([]BranchInfo, error) { - usages, err := activeStorage().usage() + usages, err := branchUsages() if err != nil { return nil, err } var infos []BranchInfo + var conts []string for _, u := range usages { name := u.Name if strings.Contains(name, "/") { continue // only direct children } - bi := BranchInfo{ + infos = append(infos, BranchInfo{ Masked: maskedStateOf(name), Name: name, Used: u.Used, Refer: u.Refer, Primary: name == "main", Agent: strings.HasPrefix(name, "agent-"), - State: ContainerState(name), + State: "absent", + }) + conts = append(conts, container(name)) + } + if len(conts) == 0 { + return infos, nil + } + out, _ := inspectContainers(execProbe, conts) // what it found, whatever else it says + facts := parseFacts(out) + forEachBounded(len(infos), func(i int) { + f, ok := facts[conts[i]] + if !ok { + return } - if bi.State == "running" { - bi.Port = "5432" // in-container port; branches are reached via the gateway - if c, err := ActiveConnections(name); err == nil { - bi.Connections = c + infos[i].State = f.state + if f.state == "running" { + infos[i].Port = "5432" // in-container port; branches are reached via the gateway + if c, err := countConnections(infos[i].Name, f.ip); err == nil { + infos[i].Connections = c } } - infos = append(infos, bi) - } + }) return infos, nil } +// branchUsages and countConnections are Branches' other two sources, variables +// so a test can count them. +var ( + branchUsages = func() ([]branchUsage, error) { return activeStorage().usage() } + + countConnections = func(name, ip string) (int, error) { + if ip != "" { + if n, err := connectionsOver(ip); err == nil { + return n, nil + } + } + return activeConnectionsWithin(execProbe, name) + } +) + +// connectionsOver reads a branch's connection count over the docker network, +// which stateConn already does as this role: no sudo, docker or psql process for +// a number the dashboard wants for every branch every few seconds. Its callers +// fall back to the exec, so a branch this cannot reach is no worse off. +func connectionsOver(ip string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cfg, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s@%s:5432/%s", pgUser, ip, pgDatabase)) + if err != nil { + return 0, err + } + cfg.Password = pgPass() + conn, err := pgx.ConnectConfig(ctx, cfg) + if err != nil { + return 0, err + } + defer conn.Close(context.Background()) + var n int + if err := conn.QueryRow(ctx, activeConnectionsSQL).Scan(&n); err != nil { + return 0, err + } + return n, nil +} + // Storage summarises pool usage for the dashboard. type Storage struct { Used string `json:"used"` @@ -1547,8 +1671,14 @@ func StorageInfo() Storage { // --- auto-suspend / auto-resume --- // ContainerState returns "running", "exited"/"created", or "absent". -func ContainerState(name string) string { - out, err := capture("docker", "inspect", "-f", "{{.State.Status}}", container(name)) +// +// Always asks docker: the engine's own decisions are made on this, and +// statecache.go's memory is only for routing a connection. +func ContainerState(name string) string { return containerStateWithin(execQuick, name) } + +// containerStateWithin is ContainerState bounded by d. +func containerStateWithin(d time.Duration, name string) string { + out, err := captureWithin(d, "docker", "inspect", "-f", "{{.State.Status}}", container(name)) if err != nil { return "absent" } @@ -1615,6 +1745,39 @@ func Resume(name string) error { // idle one. var OnUse func(name string) +// OnWake, when set, is told a suspended branch is about to be resumed, and what +// state it was in. The gateway logs it. +var OnWake func(name, state string) + +// recreateBranch and resumeBranch are EnsureRunning's two ways of bringing a +// branch up, and branchExists the check before the first: variables so a test +// can see that a stale or missing answer never reaches them. +var ( + branchExists = func(name string) bool { return activeStorage().exists(name) } + + recreateBranch = func(name string) error { + if err := startContainer(name, name == "main"); err != nil { + return err + } + if err := waitReady(name); err != nil { + return err + } + // A new container, perhaps from a newer image: the SQL too. + syncIdentityBestEffort(name, true) + applyAuditBestEffort(name) + reconcileGuard(name) + return nil + } + + resumeBranch = func(name string) error { + if err := Resume(name); err != nil { + return err + } + reconcileGuard(name) + return nil + } +) + func EnsureRunning(name string) (string, error) { if err := checkName(name); err != nil { return "", err @@ -1634,30 +1797,38 @@ func EnsureRunning(name string) (string, error) { // and cause split-brain. Just route to the current primary. return BackendAddr("main") } - switch ContainerState(name) { + cont := container(name) + // The ordinary case, and the only one remembered answers can settle: it was + // serving a moment ago, and if that has stopped being true the dial fails + // and the caller comes back through here. + if f, err := containers.lookup(cont); err == nil && f.serving() { + return f.addr(), nil + } + // Anything else may start or recreate a container, so it stands on an + // answer asked for here and now. + f, err := probeContainer(cont) + if err != nil { + return "", fmt.Errorf("branch %q: %w", name, err) + } + switch f.state { case "running": // already up case "absent": // No container. If the dataset survives (e.g. after a full stop), // recreate the container on it; otherwise the branch truly doesn't exist. - if !activeStorage().exists(name) { + if !branchExists(name) { return "", fmt.Errorf("branch %q does not exist", name) } - if err := startContainer(name, name == "main"); err != nil { - return "", err - } - if err := waitReady(name); err != nil { + if err := recreateBranch(name); err != nil { return "", err } - // A new container, perhaps from a newer image: the SQL too. - syncIdentityBestEffort(name, true) - applyAuditBestEffort(name) - reconcileGuard(name) default: // exited, created, paused, ... - if err := Resume(name); err != nil { + if OnWake != nil { + OnWake(name, f.state) + } + if err := resumeBranch(name); err != nil { return "", err } - reconcileGuard(name) } return BackendAddr(name) } @@ -1670,10 +1841,14 @@ func Wake(name string) error { // ActiveConnections returns the number of client connections currently open to a // branch (used to avoid suspending a branch that is in use). -func ActiveConnections(name string) (int, error) { - out, err := capture("docker", "exec", "--env-file", pgEnvFile(), container(name), - "psql", "-U", pgUser, "-d", pgDatabase, "-tAc", - "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend' AND pid<>pg_backend_pid();") +func ActiveConnections(name string) (int, error) { return activeConnectionsWithin(execQuick, name) } + +const activeConnectionsSQL = "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend' AND pid<>pg_backend_pid();" + +// activeConnectionsWithin is ActiveConnections bounded by d. +func activeConnectionsWithin(d time.Duration, name string) (int, error) { + out, err := captureWithin(d, "docker", "exec", "--env-file", pgEnvFile(), container(name), + "psql", "-U", pgUser, "-d", pgDatabase, "-tAc", activeConnectionsSQL) if err != nil { return 0, err } diff --git a/internal/branch/branch_before.go b/internal/branch/branch_before.go index ab3ee9e3..64459864 100644 --- a/internal/branch/branch_before.go +++ b/internal/branch/branch_before.go @@ -106,11 +106,9 @@ func validateBeforeName(name string) error { if !beforeNameRe.MatchString(name) { return fmt.Errorf("%w: invalid branch name %q (use lowercase letters, digits, dashes)", ErrInvalidRequest, name) } - switch { - case name == "main" || name == "restore" || name == "standby": + // Not CheckNewName: a name ending in -before- is exactly what this makes. + if engineName(name) { return fmt.Errorf("%w: %q is a reserved branch name", ErrInvalidRequest, name) - case strings.HasPrefix(name, "agent-"): - return fmt.Errorf("%w: names starting with agent- are reserved for agent branches", ErrInvalidRequest) } return nil } @@ -595,7 +593,9 @@ func BranchBeforeEntry(src string, entryID int64, newName string, logf func(form "-v", mountpoint(newName)+":/var/lib/postgresql/data", "--entrypoint", "bash", pgImage(), "-c", restoreBeforeScript) - if out, err := exec.Command("sudo", args...).CombinedOutput(); err != nil { + out, err := exec.Command("sudo", args...).CombinedOutput() + noteStateChange() + if err != nil { return fail(fmt.Errorf("starting the restore: %v: %s", err, strings.TrimSpace(string(out)))) } if err := waitPromoted(newName, envDurationOr("FOX_BRANCH_BEFORE_TIMEOUT", 15*time.Minute)); err != nil { diff --git a/internal/branch/export.go b/internal/branch/export.go index 1325a0ce..fe20e5f7 100644 --- a/internal/branch/export.go +++ b/internal/branch/export.go @@ -458,6 +458,9 @@ func RestoreExport(archive, from, into string) error { if store.exists(into) { return fmt.Errorf("branch %q already exists — restore into a new name", into) } + if err := checkNoContainer(into, ContainerState(into)); err != nil { + return err + } dir, err := os.MkdirTemp("", "fox-restore-") if err != nil { return err diff --git a/internal/branch/ha.go b/internal/branch/ha.go index 1c23d3a4..35a29517 100644 --- a/internal/branch/ha.go +++ b/internal/branch/ha.go @@ -29,12 +29,18 @@ type HAState struct { } // HAInfo reports current HA status. -func HAInfo() HAState { +func HAInfo() HAState { return haInfoWithin(execQuick) } + +// HAInfoProbe is HAInfo for the console's status, which must answer whatever the +// state of docker. +func HAInfoProbe() HAState { return haInfoWithin(execProbe) } + +func haInfoWithin(d time.Duration) HAState { st := HAState{Primary: "main", Sync: SyncOn()} if PrimaryContainer() == container("standby") { st.Primary = "standby" } - cs := ContainerState("standby") + cs := containerStateWithin(d, "standby") if cs == "absent" { return st } @@ -43,7 +49,7 @@ func HAInfo() HAState { // Both facts in one round trip, because this feeds the dashboard and the // status API: a second `docker exec psql` per call would double what the // page waits for to learn one extra bit. - out, _ := capture("docker", "exec", "--env-file", pgEnvFile(), PrimaryContainer(), + out, _ := captureWithin(d, "docker", "exec", "--env-file", pgEnvFile(), PrimaryContainer(), "psql", "-U", pgUser, "-d", pgDatabase, "-tAc", "SELECT (SELECT count(*) FROM pg_stat_replication WHERE state='streaming')::text"+ " || '|' || (current_setting('synchronous_standby_names') <> '')::text") diff --git a/internal/branch/import.go b/internal/branch/import.go index bf682597..0788af72 100644 --- a/internal/branch/import.go +++ b/internal/branch/import.go @@ -144,6 +144,10 @@ func importInstance(p *Progress, target, defName, desc string, a As, load func(s if target == "" { target = defName } + // After the default is filled in: it comes from the source's name. + if err := CheckNewName(target); err != nil { + return "", err + } p.Logf("Creating instance %q…\n", target) if err := Create(target, "main"); err != nil { return "", fmt.Errorf("create target instance %q: %w", target, err) @@ -203,6 +207,9 @@ func ImportContinuousToAs(p *Progress, source, target string, a As) (string, err if target == "" { target = defaultTargetName(source) } + if err := CheckNewName(target); err != nil { + return "", err + } // Every client below, and the subscription's walreceiver, is given the // source's own password or an empty one, never the container's (GW-1). source = noAmbientPassword(source) diff --git a/internal/branch/maskedbranch.go b/internal/branch/maskedbranch.go index 5c1270d8..b0a66bda 100644 --- a/internal/branch/maskedbranch.go +++ b/internal/branch/maskedbranch.go @@ -143,6 +143,7 @@ func putMaskedBranch(r MaskedBranch) error { m[k] = v } m[r.Name] = r + defer noteStateChange() // the console shows a masked branch's state return saveMaskedBranches(m) } diff --git a/internal/branch/names_test.go b/internal/branch/names_test.go index 995887e4..7c666702 100644 --- a/internal/branch/names_test.go +++ b/internal/branch/names_test.go @@ -4,6 +4,7 @@ package branch import ( "errors" + "strings" "testing" ) @@ -67,3 +68,70 @@ func TestPublishedPortsAreLoopback(t *testing.T) { } } } + +// Any account could name a branch "standby", and making it began by removing +// the live HA standby's container. +func TestCheckNewNameRefusesEngineNames(t *testing.T) { + for _, n := range []string{"main", "standby", "restore", "restore-check", "agent-zz", "agent-", + "main-before-606", "x-before-y", "Standby", "AGENT-zz", "X-Before-1"} { + if err := CheckNewName(n); !errors.Is(err, ErrReservedName) { + t.Errorf("CheckNewName(%q) = %v, want ErrReservedName", n, err) + } + } + for _, n := range []string{"qa", "feature-123", "import-db", "agents", "mainline", "standby2", "restored", "before", "x-before"} { + if err := CheckNewName(n); err != nil { + t.Errorf("CheckNewName(%q) = %v, want ok", n, err) + } + } + if err := CheckNewName("x/../main"); !errors.Is(err, ErrBadName) { + t.Errorf("a bad name: %v, want ErrBadName", err) + } + // The floor under the engine's own names is not raised. + for _, n := range []string{"main", "standby", "restore", "restore-check", "agent-zz", "main-before-606"} { + if err := checkName(n); err != nil { + t.Errorf("checkName(%q) = %v: the engine uses this name", n, err) + } + } +} + +// Each way a name comes in from outside is refused before anything is run, so +// these need no Docker. +func TestNewNameEntryPointsRefuseEngineNames(t *testing.T) { + p := &Progress{} + check := func(what string, err error) { + t.Helper() + if !errors.Is(err, ErrReservedName) { + t.Errorf("%s: %v, want ErrReservedName", what, err) + } + } + _, err := ImportToAs(p, "postgres://h/d", "standby", As{}) + check("import target", err) + _, err = ImportToAs(p, "mongodb://h/d", "main-before-3", As{}) + check("mongodb target", err) + _, err = ImportReaderToAs(p, strings.NewReader(""), srcSQL, "x.sql", "agent-zz", As{}) + check("upload target", err) + _, err = ImportContinuousToAs(p, "postgres://h/d", "restore-check", As{}) + check("continuous target", err) +} + +func TestCheckNoContainer(t *testing.T) { + if err := checkNoContainer("qa", "absent"); err != nil { + t.Errorf("no container: %v", err) + } + for _, state := range []string{"running", "exited", "created"} { + if err := checkNoContainer("standby", state); !errors.Is(err, ErrBranchExists) { + t.Errorf("a %s container: %v, want ErrBranchExists", state, err) + } + } +} + +// branch-before makes the *-before-* names, so it must keep accepting them, +// and it now refuses the name it had missed. +func TestBeforeNameFollowsEngineNames(t *testing.T) { + if err := validateBeforeName("restore-check"); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("restore-check: %v", err) + } + if err := validateBeforeName("main-before-12"); err != nil { + t.Errorf("main-before-12: %v", err) + } +} diff --git a/internal/branch/primitives.go b/internal/branch/primitives.go index 4ad7a8f1..56935d94 100644 --- a/internal/branch/primitives.go +++ b/internal/branch/primitives.go @@ -109,8 +109,8 @@ func PolicyQuery(name, sql string) ([]string, error) { return policyLines(name, func ResolveBranch(name string) (string, error) { return ledgerBranchName(name) } // ExecScript runs a multi-statement SQL script on a branch over stdin, aborting -// at the first error. Applying a change request is several statements that must -// stand or fall together, which ExecSQL's one statement at a time cannot do. +// at the first error, as the superuser. For the engine's own scripts only: psql +// reads its input as commands, so text a client wrote must not go through here. func ExecScript(target, sql string) error { return psqlStdin(target, sql) } // SetGuard enables or disables the Blackbox guardrail event trigger on a @@ -153,6 +153,28 @@ func OpenAs(ctx context.Context, name, account string) (*pgx.Conn, error) { return pgx.ConnectConfig(ctx, cfg) } +// OpenClient connects to the branch's database as the shared client role itself: +// what a change request is applied over. db_client owns what clients make, is no +// superuser and is a member of nothing, and the Blackbox takes its actor from +// bb.actor for a session logged in as it, so replayed statements have only a +// client's rights and are still recorded under whoever wrote them. +func OpenClient(ctx context.Context, name string) (*pgx.Conn, error) { + target := ServingBranch(name) + if err := EnsureAppRole(target); err != nil { + return nil, fmt.Errorf("setting %s's password on %s: %w", ClientRole, name, err) + } + ip, err := containerIP(container(target)) + if err != nil { + return nil, fmt.Errorf("finding %s: %w", name, err) + } + cfg, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:5432/%s", ip, pgDatabase)) + if err != nil { + return nil, err + } + cfg.User, cfg.Password = ClientRole, ClientRolePassword() + return pgx.ConnectConfig(ctx, cfg) +} + // QuoteIdent renders a name as a Postgres identifier, preserving its case. func QuoteIdent(s string) string { return sqlIdent(s) } diff --git a/internal/branch/promote.go b/internal/branch/promote.go index 251d8780..9e50c0fc 100644 --- a/internal/branch/promote.go +++ b/internal/branch/promote.go @@ -85,6 +85,13 @@ var ErrConflict = errors.New("both branches changed the same object since they s // target in a state that is neither the old schema nor the source's. var ErrBlockedByPolicy = errors.New("the policy gate blocks a statement in this request") +// ErrNotReplayable is returned when a statement in the request carries +// transaction control or changes who the rest runs or is recorded as: an entry +// holds its author's whole message, and such a statement in it would break the +// one unprivileged transaction a request is applied in. Refused when the request +// is made and again when it is applied. +var ErrNotReplayable = errors.New("a statement in this request cannot be replayed safely") + // MarshalEntries is the snapshot stored with a request. func MarshalEntries(entries []RequestEntry) (string, error) { b, err := json.Marshal(entries) diff --git a/internal/branch/reads.go b/internal/branch/reads.go index 13e482c8..18b0154a 100644 --- a/internal/branch/reads.go +++ b/internal/branch/reads.go @@ -85,7 +85,13 @@ var ErrNoStandbyRead = errors.New("no standby to read from") // StandbyReadAddr is the backend a read-only session should be sent to, or why // it cannot be. func StandbyReadAddr(name string) (string, error) { - if err := standbyReadRefusal(name, primaryBranch(), ContainerState("standby"), IsStandbyNode()); err != nil { + // From the gateway's connect path, so on the short deadline. A refusal is + // all that can come of a poor answer, and the client tries again. + state := "absent" + if f, err := containers.lookup(container("standby")); err == nil { + state = f.state + } + if err := standbyReadRefusal(name, primaryBranch(), state, IsStandbyNode()); err != nil { return "", err } return BackendAddr("standby") diff --git a/internal/branch/replication.go b/internal/branch/replication.go index 15598ed7..cced9e07 100644 --- a/internal/branch/replication.go +++ b/internal/branch/replication.go @@ -59,22 +59,33 @@ func parseReplication(name string, lines func() ([]string, error)) (Replication, return r, nil } +// runningBranchNames and replicationOf are Replicating's sources, variables so a +// test can count them. +var ( + runningBranchNames = RunningBranches + replicationOf = ReplicationStatus +) + // Replicating lists the running branches that have a continuous import. func Replicating() ([]Replication, error) { - names, err := RunningBranches() + names, err := runningBranchNames() if err != nil { return nil, err } - out := []Replication{} - for _, n := range names { - if n == "main" { - continue // an import always lands in its own branch + found := make([]Replication, len(names)) + forEachBounded(len(names), func(i int) { + if names[i] == "main" { + return // an import always lands in its own branch } - r, err := ReplicationStatus(n) - if err != nil || !r.Replicating { - continue + if r, err := replicationOf(names[i]); err == nil { + found[i] = r + } + }) + out := []Replication{} + for _, r := range found { + if r.Replicating { + out = append(out, r) } - out = append(out, r) } return out, nil } @@ -90,6 +101,7 @@ func CutoverReplication(name string) (int, error) { if !r.Replicating { return 0, fmt.Errorf("%w: branch %q has no replication to cut over", ErrNotReplicating, name) } + defer noteStateChange() // the import page must stop listing it if err := ImportCutoverTo(&Progress{}, name); err != nil { return 0, err } diff --git a/internal/branch/seclog.go b/internal/branch/seclog.go index ffe0a199..e53f9179 100644 --- a/internal/branch/seclog.go +++ b/internal/branch/seclog.go @@ -38,8 +38,30 @@ const ( secAnchorAlgorithm = "sha256-chain-v1" // LastRowHash is the head of auth.EventHash's chain ) +// secLogCheckPageSize is how many events a checkpoint holds at once. A variable +// so a test can reach a page boundary without writing thousands of events. +var secLogCheckPageSize = 5000 + // CheckpointSecurityLog anchors the security log's events added since its last // anchor. It returns nil when there is nothing new. +// +// It verifies incrementally, so its cost follows what is new rather than how +// long the log has grown. Before anchoring it checks: +// - that the last anchored event is still there, is unchanged, and has the +// hash the anchor signed; +// - that every event since chains from that hash, with no id missing and no +// event changed. +// +// So it catches any edit after the last anchor, and any rewrite of older +// events that was re-hashed through to the present, because that changes the +// hash the new events chain from. +// +// It does not re-read events before the last anchored one. An older event +// edited in place, with the rows after it left alone, is not noticed here and +// this will go on anchoring; nor are the older anchors or any signature looked +// at. `fox audit verify` (VerifySecurityLog) checks every event against every +// anchor and is what catches those: this is a guard against anchoring a log +// already known to be bad, and passing it says nothing about the older rows. func CheckpointSecurityLog(store *auth.Store) (*ledger.Anchor, error) { dir := SecurityLogAnchorDir() anchors, err := ledger.LoadAnchors(dir) @@ -53,21 +75,46 @@ func CheckpointSecurityLog(store *auth.Store) (*ledger.Anchor, error) { lastTo, prev = a.ToID, a.LastRowHash } } - fresh, err := store.SecurityEvents(lastTo, 0) - if err != nil || len(fresh) == 0 { + page, err := store.SecurityEvents(lastTo, secLogCheckPageSize) + if err != nil || len(page) == 0 { return nil, err } // Refuse to anchor a log that no longer chains: that would bless it. - all, err := store.SecurityEvents(0, 0) - if err != nil { - return nil, err - } - if id, why := auth.CheckEventChain(all); id != 0 { + refuse := func(why string) (*ledger.Anchor, error) { return nil, fmt.Errorf("not anchoring the security log: %s — inspect with: %s audit verify", why, brand.CLI) } - head := fresh[len(fresh)-1] + if lastTo > 0 { + anchored, err := store.SecurityEvents(lastTo-1, 1) + if err != nil { + return nil, err + } + if len(anchored) != 1 || anchored[0].ID != lastTo { + return refuse(fmt.Sprintf("event %d, the last one anchored, is gone", lastTo)) + } + if e := anchored[0]; e.RowHash != prev || auth.EventHash(e) != e.RowHash { + return refuse(fmt.Sprintf("event %d no longer matches its anchor", lastTo)) + } + } + var head auth.SecurityEvent + n, after, hash := 0, lastTo, prev + for { + if id, why := auth.CheckEventChainFrom(page, after, hash); id != 0 { + return refuse(why) + } + head = page[len(page)-1] + n, after, hash = n+len(page), head.ID, head.RowHash + if len(page) < secLogCheckPageSize { + break + } + if page, err = store.SecurityEvents(after, secLogCheckPageSize); err != nil { + return nil, err + } + if len(page) == 0 { + break + } + } a := ledger.Anchor{Format: secAnchorFormat, Algorithm: secAnchorAlgorithm, Branch: secLogAnchorDir, - CheckpointID: head.ID, FromID: lastTo + 1, ToID: head.ID, EntryCount: len(fresh), + CheckpointID: head.ID, FromID: lastTo + 1, ToID: head.ID, EntryCount: n, LastRowHash: head.RowHash, MerkleRoot: head.RowHash, PrevRoot: prev, CreatedAt: time.Now().UTC()} key, err := ledger.LoadOrCreateSigningKey(AnchorKeyPath()) if err != nil { diff --git a/internal/branch/seclog_checkpoint_test.go b/internal/branch/seclog_checkpoint_test.go new file mode 100644 index 00000000..44d66df0 --- /dev/null +++ b/internal/branch/seclog_checkpoint_test.go @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "database/sql" + "fmt" + "path/filepath" + "testing" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/brand" + "github.com/thefoxbyte/foxbyte/internal/ledger" +) + +// tamperEvent changes one row directly in the store, past the append-only +// trigger: what someone with the file and SQLite can do. +func tamperEvent(t *testing.T, path, set string, id int64) { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, q := range []string{ + `DROP TRIGGER security_events_no_update`, + fmt.Sprintf(`UPDATE security_events SET %s WHERE id = %d`, set, id), + `CREATE TRIGGER security_events_no_update BEFORE UPDATE ON security_events + BEGIN SELECT RAISE(ABORT, 'security_events is append-only'); END`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } +} + +func logWith(t *testing.T, events int) (*auth.Store, string) { + t.Helper() + t.Setenv(brand.EnvName("ANCHOR_DIR"), t.TempDir()) + t.Setenv(brand.EnvName("ANCHOR_KEY"), filepath.Join(t.TempDir(), "signing.key")) + path := filepath.Join(t.TempDir(), "a.db") + s, err := auth.Open(auth.Config{DBPath: path, WebOrigin: "http://x"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + addEvents(s, events) + return s, path +} + +func addEvents(s *auth.Store, n int) { + for i := 0; i < n; i++ { + s.Audit(auth.EvLoginOK, "a@x.com", "", "127.0.0.1", "") + } +} + +// A log with its first n events already anchored. +func anchoredLog(t *testing.T, n int) (*auth.Store, string) { + t.Helper() + s, path := logWith(t, n) + if a, err := CheckpointSecurityLog(s); err != nil || a == nil || a.ToID != int64(n) { + t.Fatalf("the first anchor: %+v, %v", a, err) + } + return s, path +} + +func fullVerify(t *testing.T, s *auth.Store) ledger.Report { + t.Helper() + evs, err := s.SecurityEvents(0, 0) + if err != nil { + t.Fatal(err) + } + anchors, err := ledger.LoadAnchors(SecurityLogAnchorDir()) + if err != nil { + t.Fatal(err) + } + return VerifySecurityLog(evs, anchors, nil) +} + +// A checkpoint anchors what is new and leaves a genuine log verifying. +func TestCheckpointAnchorsOnlyWhatIsNew(t *testing.T) { + s, _ := anchoredLog(t, 4) + addEvents(s, 2) + a, err := CheckpointSecurityLog(s) + if err != nil || a == nil || a.FromID != 5 || a.ToID != 6 || a.EntryCount != 2 { + t.Fatalf("the second anchor: %+v, %v", a, err) + } + if rep := fullVerify(t, s); !rep.Intact || rep.UnanchoredRows != 0 { + t.Fatalf("a genuine log: %+v", rep) + } + if again, err := CheckpointSecurityLog(s); err != nil || again != nil { + t.Fatalf("with nothing new: %+v, %v", again, err) + } +} + +// What the incremental check catches, and the line it does not cross: an edit +// before the last anchor, left in place, is passed by the checkpoint and caught +// by the full verify. Asserted in both directions so the documented coverage +// cannot quietly stop being true. +func TestCheckpointChecksFromTheLastAnchor(t *testing.T) { + t.Run("an edit since the anchor is refused", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 2) + tamperEvent(t, path, "actor = 'nobody'", 5) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("anchored over an edited event: %+v, %v", a, err) + } + }) + t.Run("an edit to the anchored event is refused", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 1) + tamperEvent(t, path, "actor = 'nobody'", 4) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("anchored past an edited anchor event: %+v, %v", a, err) + } + }) + t.Run("a rewrite re-hashed up to the new events is refused", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 1) + // Event 5 chaining from something other than the anchored hash is what + // recomputing every hash after an old edit leaves behind. + tamperEvent(t, path, "prev_hash = 'rewritten'", 5) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("anchored a chain not following its anchor: %+v, %v", a, err) + } + }) + t.Run("an edit before the anchor is the full verify's to catch", func(t *testing.T) { + s, path := anchoredLog(t, 4) + addEvents(s, 1) + tamperEvent(t, path, "actor = 'nobody'", 2) + if a, err := CheckpointSecurityLog(s); err != nil || a == nil { + t.Fatalf("the incremental check looked before its anchor: %+v, %v", a, err) + } + if rep := fullVerify(t, s); rep.Intact || rep.FirstBrokenID != 2 { + t.Fatalf("the full verify missed an edit before the last anchor: %+v", rep) + } + }) + t.Run("the first anchor checks everything", func(t *testing.T) { + s, path := logWith(t, 3) + tamperEvent(t, path, "actor = 'nobody'", 1) + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("a first anchor over an edited log: %+v, %v", a, err) + } + }) +} + +// The tail is read in pages, and every page is counted and checked. +func TestCheckpointPagesThroughTheTail(t *testing.T) { + old := secLogCheckPageSize + secLogCheckPageSize = 3 + t.Cleanup(func() { secLogCheckPageSize = old }) + + s, path := anchoredLog(t, 1) + addEvents(s, 7) // a boundary-straddling tail + a, err := CheckpointSecurityLog(s) + if err != nil || a == nil || a.EntryCount != 7 || a.ToID != 8 { + t.Fatalf("a tail longer than a page: %+v, %v", a, err) + } + addEvents(s, 6) // exactly two pages + a, err = CheckpointSecurityLog(s) + if err != nil || a == nil || a.EntryCount != 6 || a.ToID != 14 { + t.Fatalf("a tail of exact pages: %+v, %v", a, err) + } + if rep := fullVerify(t, s); !rep.Intact { + t.Fatalf("after paging: %+v", rep) + } + addEvents(s, 7) + tamperEvent(t, path, "actor = 'nobody'", 20) // in the third page + if a, err := CheckpointSecurityLog(s); err == nil || a != nil { + t.Fatalf("an edit in a later page: %+v, %v", a, err) + } +} diff --git a/internal/branch/snapshot.go b/internal/branch/snapshot.go new file mode 100644 index 00000000..a642d149 --- /dev/null +++ b/internal/branch/snapshot.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "fmt" + "slices" + "sync" + "time" +) + +// The dashboard asks what every branch is doing, and the import page which of +// them are replicating, every few seconds per open tab. Each answer was a +// serial walk over the branches, an inspect and a `docker exec psql` apiece, so +// the cost was tabs times branches, and twenty tabs on two branches was p95 1.2s. +// +// One answer now serves everybody for a few seconds. Nothing per-caller belongs +// in a Snapshot: the API filters what it is given by the caller's access after +// it comes out, and must go on doing that. +const ( + // SnapshotTTL is how old a shared answer may be. It bounds how long a change + // made by another process (the gateway suspending an idle branch, the CLI) + // is unseen on the dashboard. This process's own changes are seen at once. + SnapshotTTL = 5 * time.Second + // SnapshotWait is how long a caller holding the previous answer waits for a + // new one before showing the previous one instead. With no previous answer + // it waits for as long as the read takes, as it always did. + SnapshotWait = 4 * time.Second +) + +// Snapshot is one expensive answer shared between callers: at most one read at +// a time, kept for a TTL, and out of date the moment this process changes +// anything (noteStateChange), so the page that made a change shows it. +// +// A failed read is not kept: the next caller tries again. +type Snapshot[T any] struct { + ttl, wait time.Duration + read func() (T, error) + + mu sync.Mutex + val T + has bool + at time.Time + gen uint64 + flight *snapshotFlight[T] +} + +type snapshotFlight[T any] struct { + gen uint64 + done chan struct{} + val T + err error +} + +// NewSnapshot makes a Snapshot of read. +func NewSnapshot[T any](ttl, wait time.Duration, read func() (T, error)) *Snapshot[T] { + return &Snapshot[T]{ttl: ttl, wait: wait, read: read} +} + +// Get returns the shared answer, reading it first if it has gone out of date. +func (s *Snapshot[T]) Get() (T, error) { + s.mu.Lock() + gen := stateGen.Load() + if s.has && s.gen == gen && stateNow().Sub(s.at) < s.ttl { + v := s.val + s.mu.Unlock() + return v, nil + } + // A read begun before a change may have seen the world from before it, so + // nobody who comes after the change joins it. + fl := s.flight + if fl == nil || fl.gen != gen { + fl = &snapshotFlight[T]{gen: gen, done: make(chan struct{})} + s.flight = fl + go s.fetch(fl) + } + prev, hasPrev := s.val, s.has + s.mu.Unlock() + + if !hasPrev { + <-fl.done + return fl.val, fl.err + } + timer := time.NewTimer(s.wait) + defer timer.Stop() + select { + case <-fl.done: + return fl.val, fl.err + case <-timer.C: + return prev, nil + } +} + +func (s *Snapshot[T]) fetch(fl *snapshotFlight[T]) { + defer close(fl.done) + func() { + // On its own goroutine, where net/http's recovery does not reach: a + // panic here would take the whole control plane with it. + defer func() { + if p := recover(); p != nil { + fl.err = fmt.Errorf("reading a snapshot: %v", p) + } + }() + fl.val, fl.err = s.read() + }() + + s.mu.Lock() + defer s.mu.Unlock() + if s.flight == fl { + s.flight = nil + } + if fl.err == nil && fl.gen == stateGen.Load() { + s.val, s.has, s.at, s.gen = fl.val, true, stateNow(), fl.gen + } +} + +// branchesSnapshot and replicatingSnapshot are the shared answers the console's +// polling reads. +var ( + branchesSnapshot = NewSnapshot(SnapshotTTL, SnapshotWait, func() ([]BranchInfo, error) { return Branches() }) + replicatingSnapshot = NewSnapshot(SnapshotTTL, SnapshotWait, func() ([]Replication, error) { return Replicating() }) +) + +// BranchesSnapshot is Branches as of a few seconds ago, for callers polling it. +// For a display only: anything deciding what to do to a branch uses Branches or +// ContainerState. Every caller gets its own slice. +func BranchesSnapshot() ([]BranchInfo, error) { + bs, err := branchesSnapshot.Get() + return slices.Clone(bs), err +} + +// ReplicatingSnapshot is Replicating as of a few seconds ago, for callers +// polling it. Every caller gets its own slice. +func ReplicatingSnapshot() ([]Replication, error) { + rs, err := replicatingSnapshot.Get() + return slices.Clone(rs), err +} + +// maxProbeFanOut bounds how many branches are asked about at once, so a large +// install does not start every process together. +const maxProbeFanOut = 8 + +// forEachBounded runs f(0..n-1) with at most maxProbeFanOut running at a time, +// and returns when all have finished. +func forEachBounded(n int, f func(i int)) { + sem := make(chan struct{}, maxProbeFanOut) + var wg sync.WaitGroup + wg.Add(n) + for i := range n { + sem <- struct{}{} + go func() { + defer func() { <-sem; wg.Done() }() + f(i) + }() + } + wg.Wait() +} diff --git a/internal/branch/snapshot_test.go b/internal/branch/snapshot_test.go new file mode 100644 index 00000000..7051d8b4 --- /dev/null +++ b/internal/branch/snapshot_test.go @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestSnapshotIsSharedForItsTTL(t *testing.T) { + _, advance := withFakeDocker(t) + reads := 0 + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { reads++; return reads, nil }) + for range 10 { + if v, err := s.Get(); err != nil || v != 1 { + t.Fatalf("Get = %d, %v", v, err) + } + } + if reads != 1 { + t.Fatalf("%d reads for 10 callers, want 1", reads) + } + advance(SnapshotTTL) + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get after the TTL = %d, want a new read", v) + } +} + +// Create, delete and suspend have to show on the very next poll. +func TestSnapshotIsOutOfDateAfterAStateChange(t *testing.T) { + _, _ = withFakeDocker(t) + reads := 0 + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { reads++; return reads, nil }) + _, _ = s.Get() + noteIfChange("docker", []string{"stop", "pg-a"}) + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get = %d: a change this process made was not seen", v) + } +} + +func TestSnapshotConcurrentCallersShareOneRead(t *testing.T) { + _, _ = withFakeDocker(t) + var reads atomic.Int64 + gate := make(chan struct{}) + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { reads.Add(1); <-gate; return 7, nil }) + const n = 50 + var wg sync.WaitGroup + wg.Add(n) + for range n { + go func() { + defer wg.Done() + if v, err := s.Get(); err != nil || v != 7 { + t.Errorf("Get = %d, %v", v, err) + } + }() + } + for reads.Load() == 0 { + time.Sleep(time.Millisecond) + } + time.Sleep(20 * time.Millisecond) + close(gate) + wg.Wait() + if got := reads.Load(); got != 1 { + t.Fatalf("%d reads for %d concurrent callers, want 1", got, n) + } +} + +// A read begun before a change is neither joined nor kept. Otherwise the poll +// that follows a suspend could be given the list from before it. +func TestSnapshotDoesNotKeepAReadFromBeforeAChange(t *testing.T) { + _, _ = withFakeDocker(t) + var reads atomic.Int64 + gate := make(chan struct{}) + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { + if reads.Add(1) == 1 { + <-gate + return 1, nil // read before the change + } + return 2, nil + }) + first := make(chan int, 1) + go func() { v, _ := s.Get(); first <- v }() + for reads.Load() == 0 { + time.Sleep(time.Millisecond) + } + noteStateChange() + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get after the change = %d, want the read begun after it", v) + } + close(gate) + <-first + if v, _ := s.Get(); v != 2 { + t.Fatalf("Get = %d: the older read replaced the newer", v) + } +} + +func TestSnapshotDoesNotKeepAFailure(t *testing.T) { + _, _ = withFakeDocker(t) + fail := true + boom := errors.New("boom") + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { + if fail { + return 0, boom + } + return 1, nil + }) + if _, err := s.Get(); !errors.Is(err, boom) { + t.Fatalf("err = %v", err) + } + fail = false + if v, err := s.Get(); err != nil || v != 1 { + t.Fatalf("Get = %d, %v: a failure was kept", v, err) + } +} + +// With an earlier answer to hand, nobody waits on a hung read for longer than +// SnapshotWait: this is what stands between a stuck docker and the dashboard. +func TestSnapshotServesThePreviousAnswerRatherThanHang(t *testing.T) { + _, _ = withFakeDocker(t) + var reads atomic.Int64 + gate := make(chan struct{}) + defer close(gate) + s := NewSnapshot(SnapshotTTL, 20*time.Millisecond, func() (int, error) { + if reads.Add(1) > 1 { + <-gate + } + return 1, nil + }) + if v, _ := s.Get(); v != 1 { + t.Fatal(v) + } + noteStateChange() // out of date: the next read will hang + done := make(chan int, 1) + go func() { v, _ := s.Get(); done <- v }() + select { + case v := <-done: + if v != 1 { + t.Fatalf("Get = %d, want the previous answer", v) + } + case <-time.After(5 * time.Second): + t.Fatal("hung on a read that will not finish") + } +} + +func TestSnapshotSurvivesAPanickingRead(t *testing.T) { + _, _ = withFakeDocker(t) + s := NewSnapshot(SnapshotTTL, SnapshotWait, func() (int, error) { panic("no") }) + if _, err := s.Get(); err == nil { + t.Fatal("a panicking read returned no error") + } + s.read = func() (int, error) { return 1, nil } + if v, err := s.Get(); err != nil || v != 1 { + t.Fatalf("Get = %d, %v: the failed read stayed in flight", v, err) + } +} + +func TestForEachBoundedRunsEachOnce(t *testing.T) { + var running, peak atomic.Int64 + seen := make([]atomic.Int64, 40) + forEachBounded(len(seen), func(i int) { + now := running.Add(1) + for { + p := peak.Load() + if now <= p || peak.CompareAndSwap(p, now) { + break + } + } + time.Sleep(time.Millisecond) + running.Add(-1) + seen[i].Add(1) + }) + for i := range seen { + if seen[i].Load() != 1 { + t.Fatalf("index %d ran %d times", i, seen[i].Load()) + } + } + if peak.Load() > maxProbeFanOut { + t.Fatalf("%d ran at once, the bound is %d", peak.Load(), maxProbeFanOut) + } +} + +// branchesWorld installs the fakes behind Branches: a pool of datasets and a +// count of what is asked about them. +func branchesWorld(t *testing.T, datasets ...string) (*fakeDocker, *atomic.Int64, func(time.Duration)) { + t.Helper() + d, advance := withFakeDocker(t) + var counted atomic.Int64 + oldUsage, oldCount := branchUsages, countConnections + branchUsages = func() ([]branchUsage, error) { + var u []branchUsage + for _, n := range datasets { + u = append(u, branchUsage{Name: n, Used: "1M", Refer: "2M"}) + } + return u, nil + } + countConnections = func(string, string) (int, error) { counted.Add(1); return 3, nil } + t.Cleanup(func() { branchUsages, countConnections = oldUsage, oldCount }) + return d, &counted, advance +} + +// One inspect however many branches there are, and an answer of the same shape +// the serial walk gave: a stopped branch has no port or count, and one with no +// container is absent. +func TestBranchesUsesOneInspect(t *testing.T) { + d, counted, _ := branchesWorld(t, "main", "dev", "agent-x", "gone") + d.set("pg-main", "running", "10.0.0.1") + d.set("pg-dev", "exited", "") + d.set("pg-agent-x", "running", "10.0.0.3") + + got, err := Branches() + if err != nil { + t.Fatal(err) + } + if n := d.calls.Load(); n != 1 { + t.Fatalf("%d inspects for 4 branches, want 1", n) + } + if n := counted.Load(); n != 2 { + t.Fatalf("%d connection counts, want one per running branch", n) + } + want := []BranchInfo{ + {Name: "main", Primary: true, State: "running", Used: "1M", Refer: "2M", Connections: 3, Port: "5432"}, + {Name: "dev", State: "exited", Used: "1M", Refer: "2M"}, + {Name: "agent-x", Agent: true, State: "running", Used: "1M", Refer: "2M", Connections: 3, Port: "5432"}, + {Name: "gone", State: "absent", Used: "1M", Refer: "2M"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Branches =\n%+v\nwant\n%+v", got, want) + } +} + +// Callers filter and sort what they are given. One caller's doing that must +// not show up in another's list. +func TestBranchesSnapshotGivesEachCallerItsOwnSlice(t *testing.T) { + d, _, _ := branchesWorld(t, "main") + d.set("pg-main", "running", "10.0.0.1") + noteStateChange() // the package-level snapshot, between tests + a, err := BranchesSnapshot() + if err != nil { + t.Fatal(err) + } + a[0].Name = "changed by a caller" + b, _ := BranchesSnapshot() + if b[0].Name != "main" { + t.Fatalf("the shared list was changed through a caller's copy: %+v", b[0]) + } +} + +// The measurement behind the change. 20 tabs on 2 branches for a minute: each +// tab used to poll /api/status and /api/branches every 3s, and each of those +// walked the branches. Now 10s polls of one shared snapshot. Run with -v. +func TestDashboardDockerCalls(t *testing.T) { + d, counted, step := branchesWorld(t, "main", "dev") + d.set("pg-main", "running", "10.0.0.1") + d.set("pg-dev", "running", "10.0.0.2") + noteStateChange() + const clients, minute = 20, 60 + + polls := 0 + var clock time.Duration + for clock < minute*time.Second { + var wg sync.WaitGroup + wg.Add(clients * 2) + for range clients * 2 { // status and branches read the same snapshot + go func() { + defer wg.Done() + if _, err := BranchesSnapshot(); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + polls++ + step(10 * time.Second) + clock += 10 * time.Second + } + inspects, conns := d.calls.Load(), counted.Load() + // Per request before: an inspect and a psql exec per branch, serially. + legacy := clients * (minute / 3) * 2 * (2 + 2) + t.Logf("dashboard: %d tabs, 2 branches, 60s: %d docker inspects + %d connection counts over the network, was %d docker processes", + clients, inspects, conns, legacy) + if inspects > int64(polls)+1 { + t.Fatalf("%d inspects over %d polls: not shared", inspects, polls) + } +} diff --git a/internal/branch/statecache.go b/internal/branch/statecache.go new file mode 100644 index 00000000..58deb07f --- /dev/null +++ b/internal/branch/statecache.go @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Every gateway connection and console request used to ask docker where its +// branch was: an inspect for the state, the same inspect again inside +// EnsureRunning, another for the IP. Three `sudo docker` processes per +// connection, each allowed ten minutes, so a dockerd that stopped answering held +// every request for ten minutes. +// +// So one inspect now answers both questions, it gets seconds rather than +// minutes, and what it found is kept briefly for the callers behind it. +// +// The gateway, the control plane and the CLI are separate processes, and this +// memory is not shared between them. A suspend run elsewhere cannot clear it, so +// the rules below are what make keeping it safe: +// +// - Only "running, at this address" is ever kept. A branch seen stopped or +// absent is not remembered at all, so no stale "absent" can exist to send +// anybody down startContainer's `docker rm -f`. +// - What to do about a branch that is not running is decided by probeContainer, +// which always asks docker and shares its answer with nobody. +// - A docker that did not answer is ErrStateUnknown, never "absent". +// - An address that turns out not to work is forgotten and routed again +// (ForgetBackends; the gateway's connectBackend, the console's connectSession). + +// stateCacheTTL is how long a branch seen running is believed without asking +// again. Short because it is the window in which another process's suspend goes +// unnoticed; long enough that a burst of connections costs one inspect. +const stateCacheTTL = 2 * time.Second + +// ErrStateUnknown: docker could not say what state a container is in. Not a +// statement that the container is missing, and the difference matters: "absent" +// is what recreates one. +var ErrStateUnknown = errors.New("could not read the container's state from docker") + +// containerFacts is what one inspect says about a container. +type containerFacts struct { + state string // running | exited | created | paused | ... | absent + ip string // on the engine's docker network; "" when not attached +} + +// serving reports whether these facts can route a connection. +func (f containerFacts) serving() bool { return f.state == "running" && f.ip != "" } + +func (f containerFacts) addr() string { return f.ip + ":5432" } + +// inspectFormat puts a container's name, state and IP on one line. `index` +// rather than the dotted path containerIP uses, because a container not on the +// network has no such key and a dotted path would fail the whole inspect. +var inspectFormat = `{{.Name}}|{{.State.Status}}|{{with index .NetworkSettings.Networks "` + network + `"}}{{.IPAddress}}{{end}}` + +// inspectContainers is the docker call behind every answer here, a variable so +// a test can count it. docker prints what it found and fails over what it did +// not, so the output is worth reading even with an error. +var inspectContainers = func(d time.Duration, conts []string) (string, error) { + args := append([]string{"inspect", "--type", "container", "-f", inspectFormat}, conts...) + return captureWithin(d, "docker", args...) +} + +// parseFacts reads inspectFormat's lines, keyed by container name. +func parseFacts(out string) map[string]containerFacts { + facts := map[string]containerFacts{} + for _, ln := range strings.Split(out, "\n") { + f := strings.Split(strings.TrimSpace(ln), "|") + if len(f) != 3 || f[1] == "" { + continue + } + facts[strings.TrimPrefix(f[0], "/")] = containerFacts{state: f[1], ip: f[2]} + } + return facts +} + +// isNoSuchContainer tells docker saying a container does not exist from docker +// not saying anything. Only the first may be read as "absent". +func isNoSuchContainer(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), "no such") +} + +// probeContainer asks docker now, on a short deadline, and shares the answer +// with nobody. It is what a decision to start or recreate a container has to +// stand on. +func probeContainer(cont string) (containerFacts, error) { + out, err := inspectContainers(execProbe, []string{cont}) + if f, ok := parseFacts(out)[cont]; ok { + return f, nil + } + if isNoSuchContainer(err) { + return containerFacts{state: "absent"}, nil + } + if err == nil { + err = fmt.Errorf("docker inspect said nothing about %s", cont) + } + return containerFacts{}, fmt.Errorf("%w: %v", ErrStateUnknown, err) +} + +// stateGen counts the state changes this process has made. Whatever was read +// under an earlier value is out of date, however recently it was read. +var stateGen atomic.Uint64 + +// stateNow is the clock of everything in this file, so a test can move it. +var stateNow = time.Now + +// noteStateChange says this process has just changed a container, a dataset or +// which container is the primary. It goes after the change, because a read made +// while a `docker stop` is running still says "running". +func noteStateChange() { + containers.mu.Lock() + defer containers.mu.Unlock() + stateGen.Add(1) + clear(containers.entries) +} + +// ForgetBackends drops every address kept here, for a caller that has just +// failed to reach one. The next routing asks docker again. +func ForgetBackends() { noteStateChange() } + +// changesContainers recognises the docker commands that change what is running. +// Checked where every command is run, rather than at each place that stops or +// starts something, so that one added later cannot forget to. `docker run --rm` +// is a tool in a throwaway container, and the backup listing runs one a minute. +func changesContainers(name string, args []string) bool { + if name != "docker" || len(args) == 0 { + return false + } + switch args[0] { + case "start", "stop", "restart", "kill", "rm", "create", "pause", "unpause", "rename": + return true + case "run": + for _, a := range args[1:] { + if a == "--rm" { + return false + } + } + return true + } + return false +} + +// noteIfChange is noteStateChange for a command that has just finished, +// successfully or not: one that failed may still have done part of its work. +func noteIfChange(name string, args []string) { + if changesContainers(name, args) { + noteStateChange() + } +} + +// probeFlight is one inspect in progress, for the callers waiting on it. +type probeFlight struct { + gen uint64 + done chan struct{} + facts containerFacts + err error +} + +// containerCache is the per-container memory: at most one inspect at a time for +// a container, and its answer kept for stateCacheTTL if the container was +// serving. +type containerCache struct { + mu sync.Mutex + entries map[string]cachedFacts + flights map[string]*probeFlight +} + +type cachedFacts struct { + facts containerFacts + expires time.Time +} + +var containers = &containerCache{entries: map[string]cachedFacts{}, flights: map[string]*probeFlight{}} + +// lookup answers from memory or from the inspect already under way, and starts +// one if there is neither. +// +// A caller may only act on a serving answer. Anything else came from nowhere +// that a decision can stand on: it may be another caller's inspect from a moment +// ago. Those go on to probeContainer. +// +// A flight begun before a state change is not joined, and what it finds is not +// kept: it may have read the world from before the change. +func (c *containerCache) lookup(cont string) (containerFacts, error) { + c.mu.Lock() + if e, ok := c.entries[cont]; ok && stateNow().Before(e.expires) { + c.mu.Unlock() + return e.facts, nil + } + gen := stateGen.Load() + if fl, ok := c.flights[cont]; ok && fl.gen == gen { + c.mu.Unlock() + <-fl.done + return fl.facts, fl.err + } + fl := &probeFlight{gen: gen, done: make(chan struct{})} + c.flights[cont] = fl + c.mu.Unlock() + + fl.facts, fl.err = probeContainer(cont) + + c.mu.Lock() + if c.flights[cont] == fl { + delete(c.flights, cont) + } + if fl.err == nil && fl.facts.serving() && fl.gen == stateGen.Load() { + c.entries[cont] = cachedFacts{facts: fl.facts, expires: stateNow().Add(stateCacheTTL)} + } + c.mu.Unlock() + close(fl.done) + return fl.facts, fl.err +} diff --git a/internal/branch/statecache_test.go b/internal/branch/statecache_test.go new file mode 100644 index 00000000..1b1cf0a8 --- /dev/null +++ b/internal/branch/statecache_test.go @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeDocker stands in for inspectContainers: containers it knows answer with +// their state and IP, anything else the way docker says it has no such thing. +type fakeDocker struct { + mu sync.Mutex + state map[string]containerFacts + calls atomic.Int64 + fail error // when set, docker is not answering + gate chan struct{} +} + +func (d *fakeDocker) inspect(_ time.Duration, conts []string) (string, error) { + d.calls.Add(1) + if d.gate != nil { + <-d.gate + } + d.mu.Lock() + defer d.mu.Unlock() + if d.fail != nil { + return "", d.fail + } + var lines []string + var missing error + for _, c := range conts { + f, ok := d.state[c] + if !ok { + missing = fmt.Errorf("exit status 1: Error: No such object: %s", c) + continue + } + lines = append(lines, fmt.Sprintf("/%s|%s|%s", c, f.state, f.ip)) + } + return strings.Join(lines, "\n"), missing +} + +func (d *fakeDocker) set(cont, state, ip string) { + d.mu.Lock() + defer d.mu.Unlock() + d.state[cont] = containerFacts{state: state, ip: ip} +} + +func (d *fakeDocker) remove(cont string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.state, cont) +} + +// withFakeDocker installs a fake docker and a clean memory, and hands back the +// clock so a test can pass the TTL without sleeping. +func withFakeDocker(t *testing.T) (*fakeDocker, func(time.Duration)) { + t.Helper() + d := &fakeDocker{state: map[string]containerFacts{}} + oldInspect, oldNow := inspectContainers, stateNow + oldExists, oldRecreate, oldResume := branchExists, recreateBranch, resumeBranch + now := time.Unix(1_000_000, 0) + var clock sync.Mutex + inspectContainers = d.inspect + stateNow = func() time.Time { clock.Lock(); defer clock.Unlock(); return now } + branchExists = func(string) bool { return true } + recreateBranch = func(n string) error { t.Fatalf("recreated %q unexpectedly", n); return nil } + resumeBranch = func(n string) error { t.Fatalf("resumed %q unexpectedly", n); return nil } + noteStateChange() + t.Cleanup(func() { + inspectContainers, stateNow = oldInspect, oldNow + branchExists, recreateBranch, resumeBranch = oldExists, oldRecreate, oldResume + noteStateChange() + }) + return d, func(by time.Duration) { clock.Lock(); defer clock.Unlock(); now = now.Add(by) } +} + +func TestParseFacts(t *testing.T) { + got := parseFacts("/pg-a|running|172.18.0.5\n/pg-b|exited|\n\nnot a line") + if got["pg-a"] != (containerFacts{"running", "172.18.0.5"}) { + t.Errorf("pg-a = %+v", got["pg-a"]) + } + if got["pg-b"] != (containerFacts{"exited", ""}) { + t.Errorf("pg-b = %+v", got["pg-b"]) + } + if len(got) != 2 { + t.Errorf("%d containers parsed, want 2: %v", len(got), got) + } +} + +// Within the TTL a connection costs no docker call at all; past it, one. +func TestARunningBranchIsKeptForTheTTL(t *testing.T) { + d, advance := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + for range 5 { + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.1:5432" { + t.Fatalf("EnsureRunning = %q, %v", addr, err) + } + } + if n := d.calls.Load(); n != 1 { + t.Fatalf("%d docker calls for 5 connections, want 1", n) + } + advance(stateCacheTTL + time.Millisecond) + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + if n := d.calls.Load(); n != 2 { + t.Fatalf("%d docker calls after the TTL, want 2", n) + } +} + +// Anything the engine does to a container is seen at once, without waiting for +// the TTL. +func TestAStateChangeInvalidates(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + d.set("pg-a", "running", "10.0.0.9") + noteIfChange("docker", []string{"start", "pg-a"}) + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.9:5432" { + t.Fatalf("EnsureRunning = %q, %v, want the new address", addr, err) + } +} + +// The commands that change what is running, recognised where commands are run, +// so that none needs remembering at its call site. A tool in a throwaway +// container changes nothing anybody is routed to, and reads certainly do not. +func TestChangesContainers(t *testing.T) { + yes := [][]string{{"docker", "stop", "x"}, {"docker", "start", "x"}, {"docker", "restart", "x"}, {"docker", "kill", "-s", "KILL", "x"}, + {"docker", "rm", "-f", "-v", "x"}, {"docker", "run", "-d", "--name", "x", "img"}, {"docker", "rename", "x", "y"}} + no := [][]string{{"docker", "run", "--rm", "img"}, {"docker", "inspect", "x"}, {"docker", "exec", "x", "psql"}, {"docker", "ps"}, + {"docker", "network", "create", "n"}, {"docker"}, {"zfs", "stop"}} + for _, c := range yes { + if !changesContainers(c[0], c[1:]) { + t.Errorf("%v should invalidate", c) + } + } + for _, c := range no { + if changesContainers(c[0], c[1:]) { + t.Errorf("%v should not invalidate", c) + } + } +} + +// A hundred connections arriving together, with nothing remembered, share one +// inspect. +func TestConcurrentCallersShareOneInspect(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + d.gate = make(chan struct{}) + const n = 100 + var wg sync.WaitGroup + wg.Add(n) + errs := make(chan error, n) + for range n { + go func() { + defer wg.Done() + if addr, err := EnsureRunning("a"); err != nil || addr != "10.0.0.1:5432" { + errs <- fmt.Errorf("%q, %v", addr, err) + } + }() + } + // Every caller queued, or as many as will: then let the inspect finish. + for d.calls.Load() == 0 { + time.Sleep(time.Millisecond) + } + time.Sleep(20 * time.Millisecond) + close(d.gate) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + if got := d.calls.Load(); got != 1 { + t.Fatalf("%d inspects for %d concurrent connections, want 1", got, n) + } +} + +// An inspect that began before a state change read the world from before it, +// so nobody joins it afterwards and its answer is not kept. +func TestAnInspectFromBeforeAChangeIsNotKept(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + d.gate = make(chan struct{}) + first := make(chan string, 1) + go func() { + a, _ := EnsureRunning("a") + first <- a + }() + for d.calls.Load() == 0 { + time.Sleep(time.Millisecond) + } + // The change lands while that inspect is out. + d.set("pg-a", "running", "10.0.0.9") + noteStateChange() + close(d.gate) + <-first + d.gate = nil + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.9:5432" { + t.Fatalf("EnsureRunning = %q, %v: kept an answer from before the change", addr, err) + } +} + +// Another process stopped the branch: the memory still says running and the +// caller is given the old address, which fails to dial. ForgetBackends sends it +// back through the real path, which wakes the branch. +func TestAForgottenBackendIsWokenThroughTheRealPath(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + d.set("pg-a", "exited", "") // another process's suspend, unseen here + addr, err := EnsureRunning("a") + if err != nil || addr != "10.0.0.1:5432" { + t.Fatalf("within the TTL: %q, %v, want the remembered address", addr, err) + } + woken := 0 + resumeBranch = func(string) error { woken++; d.set("pg-a", "running", "10.0.0.2"); noteStateChange(); return nil } + ForgetBackends() + addr, err = EnsureRunning("a") + if err != nil || addr != "10.0.0.2:5432" || woken != 1 { + t.Fatalf("after forgetting: %q, %v, woken %d", addr, err, woken) + } +} + +// Nothing but a fresh answer from docker can recreate a container, whose first +// step is `docker rm -f`. +func TestAbsentIsOnlyBelievedFromAFreshAnswer(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + // Absent never goes into the memory: the only way to the destructive path + // is an inspect made during this very call. + d.remove("pg-a") + ForgetBackends() + before := d.calls.Load() + recreated := 0 + recreateBranch = func(string) error { recreated++; d.set("pg-a", "running", "10.0.0.3"); noteStateChange(); return nil } + if addr, err := EnsureRunning("a"); err != nil || addr != "10.0.0.3:5432" || recreated != 1 { + t.Fatalf("EnsureRunning = %q, %v, recreated %d", addr, err, recreated) + } + if n := d.calls.Load() - before; n < 2 { + t.Fatalf("recreated on %d docker calls: the decision needs its own", n) + } + containers.mu.Lock() + for cont, e := range containers.entries { + if !e.facts.serving() { + t.Errorf("%s remembered as %q", cont, e.facts.state) + } + } + containers.mu.Unlock() +} + +// A dockerd that has stopped answering looks like every container vanishing to +// anything that treats an error as "absent", and the next step of that would be +// removing them. It is an error, and nothing is touched. +func TestDockerNotAnsweringIsNeverAbsent(t *testing.T) { + d, _ := withFakeDocker(t) + d.fail = errors.New("docker did not finish within 5s and was stopped") + // withFakeDocker fails the test if either way up is taken. + _, err := EnsureRunning("a") + if !errors.Is(err, ErrStateUnknown) { + t.Fatalf("EnsureRunning = %v, want ErrStateUnknown", err) + } + if _, err := BackendAddr("a"); !errors.Is(err, ErrStateUnknown) { + t.Fatalf("BackendAddr = %v, want ErrStateUnknown", err) + } +} + +// Docker saying there is no such container, by contrast, is an answer. +func TestNoSuchContainerIsAbsent(t *testing.T) { + d, _ := withFakeDocker(t) + f, err := probeContainer("pg-gone") + if err != nil || f.state != "absent" { + t.Fatalf("probeContainer = %+v, %v", f, err) + } + if d.calls.Load() != 1 { + t.Fatalf("%d calls", d.calls.Load()) + } +} + +// A branch that does not exist is still reported as one. +func TestAMissingBranchSaysSo(t *testing.T) { + _, _ = withFakeDocker(t) + branchExists = func(string) bool { return false } + if _, err := EnsureRunning("a"); err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Fatalf("EnsureRunning = %v", err) + } +} + +// A failover changes which container "main" is. Read each time, so a promotion +// done by another process needs no invalidation here. +func TestMainFollowsThePrimaryContainer(t *testing.T) { + d, _ := withFakeDocker(t) + d.set("pg-main", "running", "10.0.0.1") + d.set("pg-standby", "running", "10.0.0.2") + if addr, err := BackendAddr("main"); err != nil || addr != "10.0.0.1:5432" { + t.Fatalf("BackendAddr = %q, %v", addr, err) + } + if err := setPrimary("standby"); err != nil { + t.Fatal(err) + } + defer setPrimary("main") + if addr, err := BackendAddr("main"); err != nil || addr != "10.0.0.2:5432" { + t.Fatalf("after failover BackendAddr = %q, %v", addr, err) + } +} + +// The measurement behind the change: docker processes for 100 connections to a +// running branch, before and after. Run with -v to read the numbers. +func TestConnectPathDockerCalls(t *testing.T) { + d, advance := withFakeDocker(t) + d.set("pg-a", "running", "10.0.0.1") + const connections = 100 + for i := range connections { + if _, err := EnsureRunning("a"); err != nil { + t.Fatal(err) + } + if i%10 == 9 { // ten connections a second + advance(time.Second) + } + } + got := d.calls.Load() + legacy := int64(connections * 3) // state in the proxy, state again, then the IP + t.Logf("connect path: %d connections at 10/s: %d docker calls, was %d", connections, got, legacy) + if got > connections/10 { + t.Fatalf("%d docker calls", got) + } +} diff --git a/internal/controlplane/promote.go b/internal/controlplane/promote.go index 5740e0c4..13c7cdad 100644 --- a/internal/controlplane/promote.go +++ b/internal/controlplane/promote.go @@ -219,7 +219,7 @@ func statusForRequestErr(err error) int { switch { case errors.Is(err, branch.ErrConflict): return 409 - case errors.Is(err, branch.ErrBlockedByPolicy): + case errors.Is(err, branch.ErrBlockedByPolicy), errors.Is(err, branch.ErrNotReplayable): return 422 case errors.Is(err, branch.ErrNothingToPromote): return 400 diff --git a/internal/controlplane/realtime_on.go b/internal/controlplane/realtime_on.go index 923ccdd2..c5c1b810 100644 --- a/internal/controlplane/realtime_on.go +++ b/internal/controlplane/realtime_on.go @@ -235,6 +235,8 @@ func resumeFor(name string, since pglogrepl.LSN) (*realtime.Hub, realtime.Resume floor := w.Confirmed if !w.HaveSlot { floor = w.WALEnd + } else if w.QuietFrom != 0 && w.QuietFrom < floor { + floor = w.QuietFrom // nothing was published between there and the slot } h, _ := startHub(name, 0, floor) return h, realtime.ResumeLive, nil @@ -306,6 +308,9 @@ func resumeWindow(name string) (realtime.Window, error) { if lsn, err := pglogrepl.ParseLSN(rw.WALEnd); err == nil { w.WALEnd = lsn } + // What the slot moved past on keepalives, with nothing published: a position + // in there has missed nothing, however far the slot has gone since. + w.QuietFrom, w.QuietTo = realtime.QuietSpan(branch.SlotName(name, "stream"), w.Confirmed, w.HaveSlot) return w, nil } diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 7469fa7d..fbb78b6c 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -12,6 +12,7 @@ import ( "fmt" "io/fs" "log" + "net" "net/http" "net/url" "path" @@ -39,6 +40,14 @@ import ( var nameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,40}$`) +// checkNewBranchName is the rule for a branch name an account picks. +func checkNewBranchName(name string) error { + if !nameRe.MatchString(name) { + return fmt.Errorf("invalid name (use lowercase letters, digits, dashes)") + } + return branch.CheckNewName(name) +} + // ruleIDRe matches a Blackbox policy rule id (bb.policy_rules.rule_id). var ruleIDRe = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) @@ -203,8 +212,8 @@ func checkBranchName(next http.Handler) http.Handler { }) } -// backupHealth caches branch.CurrentBackupHealth for a minute: reading it lists -// the backups from a container, and the dashboard polls /api/status. +// backupHealth caches the backup judgement for a minute: reading it lists the +// backups from a container, and the dashboard polls /api/status. var backupHealth struct { sync.Mutex at time.Time @@ -215,11 +224,39 @@ func cachedBackupHealth() branch.BackupHealth { backupHealth.Lock() defer backupHealth.Unlock() if time.Since(backupHealth.at) > time.Minute { - backupHealth.h, backupHealth.at = branch.CurrentBackupHealth(), time.Now() + // Listing can start a container, so it gets more than a probe; still + // seconds, because this is answering a page. "Could not tell" is a + // verdict the dashboard already knows how to show. + backupHealth.h, backupHealth.at = branch.CurrentBackupHealthProbe(30*time.Second), time.Now() } return backupHealth.h } +// The console's polling reads these rather than walking the branches itself +// (branch/snapshot.go): variables so a test can count what is behind them. +var ( + listBranches = branch.BranchesSnapshot + listReplicating = branch.ReplicatingSnapshot +) + +// statusFacts is the engine's part of /api/status, the same for every caller. +type statusFacts struct { + branches []branch.BranchInfo + ha branch.HAState + backup branch.BackupHealth + storage branch.Storage +} + +var statusSnapshot = branch.NewSnapshot(branch.SnapshotTTL, branch.SnapshotWait, readStatusFacts) + +func readStatusFacts() (statusFacts, error) { + bs, err := listBranches() + if err != nil { + return statusFacts{}, err + } + return statusFacts{branches: bs, ha: branch.HAInfoProbe(), backup: cachedBackupHealth(), storage: branch.StorageInfo()}, nil +} + // isUpload is the one route whose body is a file rather than JSON; it has its // own, larger cap (maxUpload). func isUpload(r *http.Request) bool { return r.URL.Path == "/api/import/file" } @@ -236,14 +273,14 @@ func maxUpload() int64 { func registerAPI(mux *http.ServeMux) { mux.HandleFunc("GET /api/status", func(w http.ResponseWriter, r *http.Request) { - bs, err := branch.Branches() + st, err := statusSnapshot.Get() if err != nil { writeErr(w, 500, err) return } var nBranch, nAgent int mainReady := false - for _, b := range bs { + for _, b := range st.branches { switch { case b.Primary: mainReady = b.State == "running" @@ -263,9 +300,9 @@ func registerAPI(mux *http.ServeMux) { // and so does an Enterprise build with no licence. "edition": edition.Name(), "features": featureNames(), - "ha": branch.HAInfo(), - "backup": cachedBackupHealth(), - "storage": branch.StorageInfo(), + "ha": st.ha, + "backup": st.backup, + "storage": st.storage, "servers": map[string]bool{ "gateway": daemon.Alive("gateway"), "api": daemon.Alive("api"), @@ -274,13 +311,14 @@ func registerAPI(mux *http.ServeMux) { }) mux.HandleFunc("GET /api/branches", func(w http.ResponseWriter, r *http.Request) { - bs, err := branch.Branches() + bs, err := listBranches() if err != nil { writeErr(w, 500, err) return } // Only the branches the caller may reach: main, their own, and — for - // an admin — all of them. + // an admin — all of them. Here, per request, and never in the shared + // list above, which is the same for everybody. u, _ := auth.UserFrom(r.Context()) visible := []branch.BranchInfo{} for _, b := range bs { @@ -298,8 +336,8 @@ func registerAPI(mux *http.ServeMux) { Masked bool `json:"masked"` // a masked branch (masking step 5): made in the background } _ = json.NewDecoder(r.Body).Decode(&body) - if !nameRe.MatchString(body.Name) { - writeErr(w, 400, fmt.Errorf("invalid name (use lowercase letters, digits, dashes)")) + if err := checkNewBranchName(body.Name); err != nil { + writeErr(w, 400, err) return } if body.From != "" && !nameRe.MatchString(body.From) { @@ -497,7 +535,7 @@ func registerAPI(mux *http.ServeMux) { // Continuous imports (logical replication into a branch): where each stands, // and the cutover that turns one into a standalone branch. mux.HandleFunc("GET /api/replication", func(w http.ResponseWriter, r *http.Request) { - list, err := branch.Replicating() + list, err := listReplicating() if err != nil { writeErr(w, 500, err) return @@ -612,6 +650,42 @@ func registerAPI(mux *http.ServeMux) { // statement. const maskedStatementSQL = `'(hidden from masked accounts: a statement can hold real values)'::text` +// The console's way out to a branch, variables so a test can stand in for one +// that moved. +var ( + pgConnect = pgx.ConnectConfig + sessionRoute = branch.EnsureRunning +) + +// connectSession connects, and if the address could not be dialled, forgets it, +// routes the branch again and tries once more. The address may be seconds old +// (branch/statecache.go), and a suspend run by the gateway's idle reaper leaves +// it behind without this process hearing about it. Only for a failure to reach +// the address: a refused password or a full server is an answer from a backend +// that is there. +func connectSession(ctx context.Context, cfg *pgx.ConnConfig, name string) (*pgx.Conn, error) { + conn, err := pgConnect(ctx, cfg) + if err == nil || name == "" || !isDialFailure(err) { + return conn, err + } + branch.ForgetBackends() + addr, rerr := sessionRoute(name) + if rerr != nil { + return nil, err + } + again, perr := pgx.ParseConfig(fmt.Sprintf("postgres://%s/%s", addr, branch.Database)) + if perr != nil { + return nil, err + } + again.User, again.Password = cfg.User, cfg.Password + return pgConnect(ctx, again) +} + +func isDialFailure(err error) bool { + var op *net.OpError + return errors.As(err, &op) && op.Op == "dial" +} + func sqlEsc(s string) string { return strings.ReplaceAll(s, "'", "''") } // ledgerSQL builds a filtered, bounded query over bb.schema_ledger. Filter @@ -867,7 +941,7 @@ func openSession(ctx context.Context, addr string, as queryAs) (*pgx.Conn, error } cfg.User, cfg.Password = as.Actor, branch.UserRolePassword(as.Actor) } - conn, err := pgx.ConnectConfig(ctx, cfg) + conn, err := connectSession(ctx, cfg, as.Branch) if err != nil { return nil, err } @@ -1236,14 +1310,20 @@ func registerPipelines(mux *http.ServeMux, store *auth.Store) { return } target := pipelineBranch(pl) + // Before anything else: the engine begins by deleting whatever has this + // name, and a run is recorded as owning its target even when it fails. + if err := branch.CheckNewName(target); err != nil { + send("error", map[string]string{"message": fmt.Sprintf("rename the pipeline: %v", err)}) + return + } // A spec saved before sources were checked may still name a file. if err := checkPipelineSource(spec.Source); err != nil { send("error", map[string]string{"message": err.Error()}) return } // A run deletes and re-creates its target, so it may only do that to - // a branch this pipeline made. - if branch.Exists(target) && !store.PipelineOwnsTarget(pl.ID, target) { + // a branch this pipeline made and this account still owns. + if branch.Exists(target) && !store.PipelineMayReplace(pl.ID, u.ID, target) { send("error", map[string]string{"message": fmt.Sprintf( "a branch named %q already exists and was not made by this pipeline — rename the pipeline, or delete that branch first", target)}) return diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go index 9a327716..9869f539 100644 --- a/internal/controlplane/server_test.go +++ b/internal/controlplane/server_test.go @@ -3,12 +3,14 @@ package controlplane import ( + "errors" "net/http" "net/http/httptest" "strings" "testing" "testing/fstest" + "github.com/thefoxbyte/foxbyte/internal/branch" "github.com/thefoxbyte/foxbyte/internal/httpx" ) @@ -108,3 +110,20 @@ func TestTheConsolePageCarriesItsPolicy(t *testing.T) { } } } + +// POST /api/branches: the shape rule and the engine's names are both refused. +func TestCheckNewBranchName(t *testing.T) { + for _, n := range []string{"qa", "feature-123"} { + if err := checkNewBranchName(n); err != nil { + t.Errorf("%q: %v", n, err) + } + } + for _, n := range []string{"main", "standby", "restore", "restore-check", "agent-zz", "main-before-1"} { + if err := checkNewBranchName(n); !errors.Is(err, branch.ErrReservedName) { + t.Errorf("%q: %v, want ErrReservedName", n, err) + } + } + if err := checkNewBranchName("Bad"); err == nil || errors.Is(err, branch.ErrReservedName) { + t.Errorf("Bad: %v", err) + } +} diff --git a/internal/controlplane/snapshot_test.go b/internal/controlplane/snapshot_test.go new file mode 100644 index 00000000..d9106277 --- /dev/null +++ b/internal/controlplane/snapshot_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controlplane + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "sync/atomic" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// Everybody reads one shared list and each still sees only their own part of +// it. This is the property a snapshot could quietly lose: the list is cached, +// and who may see what must never be. +func TestSharedBranchListIsFilteredPerCaller(t *testing.T) { + s, admin, alice, bob := withACL(t) + var reads atomic.Int64 + snap := branch.NewSnapshot(time.Hour, time.Second, func() ([]branch.BranchInfo, error) { + reads.Add(1) + return []branch.BranchInfo{{Name: "main", Primary: true}, {Name: "alice-dev"}, {Name: "bobs-secret"}}, nil + }) + old := listBranches + listBranches = snap.Get + t.Cleanup(func() { listBranches = old }) + + api := http.NewServeMux() + registerAPI(api) + h := s.Authn(api) + + get := func(u auth.User) []string { + t.Helper() + key, _, err := s.CreateAPIKey(u.ID, "t") + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest("GET", "/api/branches", nil) + r.Header.Set("Authorization", "Bearer "+key) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("%s: %d %s", u.Email, w.Code, w.Body) + } + var out []branch.BranchInfo + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + names := []string{} + for _, b := range out { + names = append(names, b.Name) + } + sort.Strings(names) + return names + } + // Bob first, then the owner, then Bob again: an order that would show a + // result kept per list rather than per caller. + if got, want := get(bob), []string{"main"}; !reflect.DeepEqual(got, want) { + t.Errorf("bob sees %v, want %v", got, want) + } + if got, want := get(alice), []string{"alice-dev", "main"}; !reflect.DeepEqual(got, want) { + t.Errorf("alice sees %v, want %v", got, want) + } + if got, want := get(admin), []string{"alice-dev", "bobs-secret", "main"}; !reflect.DeepEqual(got, want) { + t.Errorf("the admin sees %v, want %v", got, want) + } + if got, want := get(bob), []string{"main"}; !reflect.DeepEqual(got, want) { + t.Errorf("bob sees %v afterwards, want %v", got, want) + } + if n := reads.Load(); n != 1 { + t.Errorf("%d reads of the branches for 4 requests, want 1", n) + } +} + +// And the same for what the import page polls. +func TestSharedReplicationListIsFilteredPerCaller(t *testing.T) { + s, _, alice, bob := withACL(t) + old := listReplicating + listReplicating = func() ([]branch.Replication, error) { + return []branch.Replication{{Branch: "alice-dev", Replicating: true}}, nil + } + t.Cleanup(func() { listReplicating = old }) + api := http.NewServeMux() + registerAPI(api) + h := s.Authn(api) + seen := func(u auth.User) int { + t.Helper() + key, _, _ := s.CreateAPIKey(u.ID, "t") + r := httptest.NewRequest("GET", "/api/replication", nil) + r.Header.Set("Authorization", "Bearer "+key) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("%s: %d %s", u.Email, w.Code, w.Body) + } + var out []branch.Replication + _ = json.Unmarshal(w.Body.Bytes(), &out) + return len(out) + } + if n := seen(bob); n != 0 { + t.Errorf("bob sees %d replications on a branch that is not his", n) + } + if n := seen(alice); n != 1 { + t.Errorf("alice sees %d, want 1", n) + } +} diff --git a/internal/controlplane/stale_test.go b/internal/controlplane/stale_test.go new file mode 100644 index 00000000..8e06c509 --- /dev/null +++ b/internal/controlplane/stale_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controlplane + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +// A branch the gateway's reaper suspended leaves the console holding an address +// that no longer answers. The console routes again rather than showing an error. +func TestConsoleRecoversFromAStaleAddress(t *testing.T) { + oldConnect, oldRoute := pgConnect, sessionRoute + t.Cleanup(func() { pgConnect, sessionRoute = oldConnect, oldRoute }) + + var hosts []string + pgConnect = func(_ context.Context, c *pgx.ConnConfig) (*pgx.Conn, error) { + hosts = append(hosts, c.Host) + if c.Host == "10.0.0.1" { + return nil, &net.OpError{Op: "dial", Err: errors.New("no route to host")} + } + return nil, nil + } + sessionRoute = func(string) (string, error) { return "10.0.0.2:5432", nil } + + cfg, err := pgx.ParseConfig("postgres://u:p@10.0.0.1:5432/d") + if err != nil { + t.Fatal(err) + } + if _, err := connectSession(context.Background(), cfg, "dev"); err != nil { + t.Fatalf("a stale address reached the user: %v", err) + } + if len(hosts) != 2 || hosts[1] != "10.0.0.2" { + t.Fatalf("connected to %v, want the old address and then the new one", hosts) + } +} + +// An answer from a backend that is there is not a stale address. +func TestConsoleDoesNotRetryAnAnswer(t *testing.T) { + oldConnect, oldRoute := pgConnect, sessionRoute + t.Cleanup(func() { pgConnect, sessionRoute = oldConnect, oldRoute }) + calls := 0 + refused := errors.New("password authentication failed") + pgConnect = func(context.Context, *pgx.ConnConfig) (*pgx.Conn, error) { calls++; return nil, refused } + sessionRoute = func(string) (string, error) { t.Fatal("routed again after an answer"); return "", nil } + cfg, _ := pgx.ParseConfig("postgres://u:p@10.0.0.1:5432/d") + if _, err := connectSession(context.Background(), cfg, "dev"); !errors.Is(err, refused) || calls != 1 { + t.Fatalf("err %v after %d connects", err, calls) + } +} + +// The check has to recognise the error pgx really returns, not just one built +// to look like it. +func TestIsDialFailureOnARealRefusal(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip(err) + } + addr := ln.Addr().String() + ln.Close() // nothing listens there now + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = pgx.Connect(ctx, "postgres://u:p@"+addr+"/d?sslmode=disable") + if err == nil { + t.Skip("something answered on a closed port") + } + if !isDialFailure(err) { + t.Fatalf("not recognised as a dial failure: %T %v", err, err) + } +} diff --git a/internal/host/host.go b/internal/host/host.go index 81634678..beeba3bc 100644 --- a/internal/host/host.go +++ b/internal/host/host.go @@ -266,8 +266,10 @@ func refreshEngineBinary(arch string) string { } // The engine is installed into the VM and run as root, so it is downloaded - // through the same verified path as `fox update`: the release's SHA256SUMS - // decides, and a file that doesn't match is never kept. + // through the same verified path as `fox update`: the release's signed + // SHA256SUMS decides, and a file that doesn't match is never kept. A release + // that doesn't verify (or a build with no release key) lands in keep below, + // so setup carries on with the installer's engine; it never goes unchecked. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() c := update.NewClient(os.Getenv, filepath.Join(cacheDir(), "update-check.json")) @@ -291,7 +293,7 @@ func refreshEngineBinary(arch string) string { _ = os.Remove(path) return keep("%s in release %s is not a Linux binary", asset, offer.Release.Tag) } - fmt.Printf("Engine %s from %s verified against SHA256SUMS.\n", asset, offer.Release.Tag) + fmt.Printf("Engine %s from %s verified against the signed SHA256SUMS.\n", asset, offer.Release.Tag) return path } diff --git a/internal/ledger/blockedconn_test.go b/internal/ledger/blockedconn_test.go new file mode 100644 index 00000000..6b26c5cd --- /dev/null +++ b/internal/ledger/blockedconn_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "strings" + "testing" +) + +// A blocked attempt is recorded through a second connection, and a wait there +// is invisible to Postgres' deadlock detector: another transaction holding the +// append lock used to hang the refused client. Every path goes through the one +// connection that carries timeouts, and failing to record stays a WARNING. +func TestBlockedRecordConnectionCarriesTimeouts(t *testing.T) { + if !strings.Contains(Schema, `options='-c lock_timeout=2s -c statement_timeout=5s'`) { + t.Error("the blocked-attempt connection lost its timeouts") + } + if !strings.Contains(Schema, "'host=/var/run/postgresql dbname='") { + t.Error("the blocked-attempt connection no longer uses the Unix socket") + } + i := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb._record_blocked(stmt text)") + if i < 0 { + t.Fatal("bb._record_blocked is gone") + } + body := Schema[i:] + body = body[:strings.Index(body, "$$;")] + if !strings.Contains(body, "EXCEPTION WHEN OTHERS THEN\n RAISE WARNING") { + t.Error("a failure to record a blocked attempt must be a WARNING, so the refusal still happens") + } + if strings.Contains(body, "SECURITY DEFINER") { + t.Error("bb._record_blocked must not run as its owner: a client could write entries through it") + } + + // Nothing talks to dblink except the helpers. + for what, sql := range map[string]string{"ledger.sql": Schema, "datachanges.sql": SchemaData} { + if n := strings.Count(sql, "PERFORM dblink_exec("); (what == "ledger.sql" && n != 1) || (what != "ledger.sql" && n != 0) { + t.Errorf("%s has %d direct dblink_exec call(s); the blocked path belongs in bb._record_blocked", what, n) + } + } + if n := strings.Count(SchemaData, "PERFORM bb._record_blocked(format("); n != 4 { + t.Errorf("datachanges.sql records %d blocked paths through the helper, want 4 (TRUNCATE, unqualified DML, impact, guard trigger)", n) + } +} + +// The defaults for client logins, and that an administrator's choice survives +// the re-run on every start. +func TestClientRolesGetTimeoutDefaults(t *testing.T) { + for _, want := range []string{ + "ARRAY['lock_timeout=60s', 'idle_in_transaction_session_timeout=10min']", + "WHERE d.setrole = r.oid AND d.setdatabase = 0", + "pg_has_role(oid, 'db_client', 'MEMBER')", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } + if strings.Contains(Schema, "statement_timeout=0") { + t.Error("client logins should not have their statement timeout touched") + } +} diff --git a/internal/ledger/chainhead_test.go b/internal/ledger/chainhead_test.go new file mode 100644 index 00000000..4ea83b32 --- /dev/null +++ b/internal/ledger/chainhead_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "strings" + "testing" +) + +// chainRowBody is bb.chain_row's source, as ledger.sql writes it. +func chainRowBody(t *testing.T) string { + t.Helper() + i := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb.chain_row()") + if i < 0 { + t.Fatal("bb.chain_row is gone") + } + body := Schema[i:] + return body[:strings.Index(body, "$$;")] +} + +// In REPEATABLE READ a snapshot can be older than the append lock, so the tail +// must not be read through it: two rows would chain off one parent and verify +// would call it tampering. The head row's UPDATE is what turns a stale snapshot +// into a serialization error. Proved against a real server; this guards the +// shape it rests on. +func TestChainHeadIsTakenByUpdateUnderTheLock(t *testing.T) { + body := chainRowBody(t) + lock := strings.Index(body, "pg_advisory_xact_lock(hashtext('bb.schema_ledger'))") + head := strings.Index(body, "UPDATE bb.ledger_head SET last_id = last_id + 1 RETURNING last_id, last_hash INTO NEW.id, prev;") + hash := strings.Index(body, "NEW.row_hash := bb._ledger_hash(NEW);") + if lock < 0 || head < 0 || hash < 0 || !(lock < head && head < hash) { + t.Errorf("bb.chain_row must lock, then take the head, then hash (lock %d, head %d, hash %d)", lock, head, hash) + } + if strings.Contains(body, "ORDER BY id DESC") { + t.Error("bb.chain_row reads the tail through the transaction's snapshot again") + } + if !strings.Contains(body, "UPDATE bb.ledger_head SET last_hash = NEW.row_hash;") { + t.Error("bb.chain_row no longer moves the head to the row it wrote") + } +} + +// The upgrade: the head and the function that needs it arrive together, under +// the lock, seeded from the existing tail, and a re-run leaves a head that is +// already right alone. Nothing may rewrite a row that exists. +func TestChainHeadUpgradesAnExistingLedger(t *testing.T) { + begin := strings.Index(Schema, "BEGIN;\nDO $$ BEGIN PERFORM pg_advisory_xact_lock(hashtext('bb.schema_ledger')); END $$;") + table := strings.Index(Schema, "CREATE TABLE IF NOT EXISTS bb.ledger_head") + fn := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb.chain_row()") + commit := strings.Index(Schema, "FOR EACH ROW EXECUTE FUNCTION bb.chain_row();\nCOMMIT;") + if begin < 0 || table < begin || fn < table || commit < fn { + t.Errorf("the head and bb.chain_row must be installed in one locked transaction (begin %d, table %d, function %d, commit %d)", begin, table, fn, commit) + } + for _, want := range []string{ + "(SELECT id, row_hash FROM bb.schema_ledger ORDER BY id DESC LIMIT 1) t ON true", + "ON CONFLICT (id) DO UPDATE SET last_id = EXCLUDED.last_id, last_hash = EXCLUDED.last_hash\n WHERE EXCLUDED.last_id > bb.ledger_head.last_id;", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } +} + +// A masked branch's ledger is emptied by the engine, past the triggers +// (enterprise/masking, freshBlackboxSQL), and verify expects the first row of +// a ledger to link to nothing. +func TestChainRestartsOnAnEmptiedLedger(t *testing.T) { + body := chainRowBody(t) + head := strings.Index(body, "UPDATE bb.ledger_head SET last_id") + empty := strings.Index(body, "IF NOT EXISTS (SELECT 1 FROM bb.schema_ledger) THEN\n prev := '';") + if empty < 0 || empty < head { + t.Error("an emptied ledger must start a new chain, judged after the head was taken") + } +} diff --git a/internal/ledger/datachanges.sql b/internal/ledger/datachanges.sql index 8b6e6985..2b0da7d7 100644 --- a/internal/ledger/datachanges.sql +++ b/internal/ledger/datachanges.sql @@ -24,8 +24,8 @@ -- transition tables, which would copy every changed row of every statement. -- -- The triggers are attached to every existing table here, and to each new one --- by an event trigger (bb_data_attach). Dropping, disabling or renaming one is --- refused unless the guardrail override is in force (a superuser or a db_admin +-- by an event trigger (bb_data_attach). Dropping, disabling, renaming or replacing +-- one is refused unless the guardrail override is in force (a superuser or a db_admin -- member with bb.allow_destructive=on). A superuser can still switch triggers -- off for a session; the Blackbox has always had that limit. @@ -66,9 +66,7 @@ BEGIN IF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) VALUES (%L,%L,%L,%L,%L,'TRUNCATE','table',%L,%L,'BLOCKED','policy')$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, obj, current_query())); @@ -125,9 +123,7 @@ BEGIN IF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) VALUES (%L,%L,%L,%L,%L,%L,'table',%L,%L,'BLOCKED','unqualified')$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, TG_OP, obj, q)); @@ -308,9 +304,7 @@ BEGIN ELSIF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) VALUES (%L,%L,%L,%L,%L,%L,'table',%L,%L,'BLOCKED',%L)$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, op, obj, current_query(), risk)); @@ -472,7 +466,7 @@ $$; -- the table's owner may set it without being a superuser. CREATE OR REPLACE FUNCTION bb.data_protect_end() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, bb AS $$ -DECLARE r record; bad text; +DECLARE r record; bad text; c record; BEGIN FOR r IN SELECT * FROM pg_event_trigger_ddl_commands() LOOP IF r.object_type NOT IN ('table','trigger') OR r.schema_name IN ('bb','bb_f1','pg_catalog','information_schema') @@ -484,6 +478,31 @@ BEGIN PERFORM bb._attach_data_triggers(r.objid::regclass); END IF; END IF; + -- A trigger named bb_something on a user's table, in any form but the one + -- the Blackbox installs. CREATE OR REPLACE TRIGGER is how a guard gets + -- replaced while it keeps its name, and whoever holds TRIGGER on a table + -- may run it. Recorded like any other refusal, so the attempt outlives the + -- rollback. The engine's own install matches the definition and passes. + IF r.command_tag = 'CREATE TRIGGER' AND r.object_type = 'trigger' AND NOT bb._override_in_force() THEN + SELECT r.object_identity INTO bad + FROM pg_trigger t JOIN pg_class k ON k.oid = t.tgrelid JOIN pg_namespace n ON n.oid = k.relnamespace + WHERE t.oid = r.objid AND t.tgname LIKE 'bb\_%' AND n.nspname NOT IN ('bb','bb_f1') + AND NOT bb._is_own_trigger(t.oid); + IF bad IS NOT NULL THEN + SELECT * INTO c FROM bb._ctx(); + IF bb._holds_chain_lock() THEN + RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; + ELSE + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger + (actor,actor_kind,tool,session,branch,command_tag,object_type,object_identity,statement,status,risk) + VALUES (%L,%L,%L,%L,%L,'CREATE TRIGGER','trigger',%L,%L,'BLOCKED','policy')$f$, + c.actor, c.actor_kind, c.tool, c.session, c.branch, bad, current_query())); + END IF; + RAISE EXCEPTION 'guardrail: bb_ trigger names are the Blackbox''s own, and % cannot be created or replaced in this form', bad + USING ERRCODE = 'insufficient_privilege', + HINT = 'Name the trigger something else. Only superusers and members of db_admin may, with bb.allow_destructive=on.'; + END IF; + END IF; -- ALTER TABLE … DISABLE TRIGGER or ENABLE REPLICA TRIGGER, or ALTER -- TRIGGER … RENAME, on one of ours. ENABLE ALWAYS stays allowed: it fires -- in more sessions, not fewer. @@ -521,11 +540,10 @@ BEGIN END; $$; --- No REVOKE here: ledger.sql grants every bb function to db_client on each --- start, so a REVOKE would have to run on each start too, and a privilege --- change after the event triggers are on is recorded as an unattributed --- change (integration-v2 §9). Calling bb._attach_data_triggers only adds --- these protective triggers, which is harmless. +-- No REVOKE here: a privilege change after the event triggers are on is +-- recorded as an unattributed change (integration-v2 §9). None is needed: +-- bb.seal_new_functions (ledger.sql) takes EXECUTE from clients on each +-- SECURITY DEFINER function here as it is made. -- Every existing table, now. (Creating these triggers is DDL; ledger.sql's -- recorder skips the Blackbox's own triggers, so a start records nothing.) diff --git a/internal/ledger/guardtrigger_test.go b/internal/ledger/guardtrigger_test.go new file mode 100644 index 00000000..6eedd781 --- /dev/null +++ b/internal/ledger/guardtrigger_test.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "strings" + "testing" +) + +// CREATE OR REPLACE TRIGGER keeps a guard's name and changes what it does, so +// a guard is compared with what the Blackbox installs, not by name. The +// behaviour is proved against a real server; this guards the shape of it. +func TestGuardTriggersCannotBeReplaced(t *testing.T) { + for _, want := range []string{ + "g.tgfoid = to_regprocedure('bb.guard_truncate()') AND g.tgtype = 34", + "g.tgfoid = to_regprocedure('bb.record_dml()') AND g.tgtype = 24", + "g.tgfoid = to_regprocedure('bb.guard_dml()') AND g.tgtype = 26", + "g.tgfoid = to_regprocedure('bb.guard_impact()') AND g.tgtype = 27", + "g.tgfoid = to_regprocedure('bb.track_row()') AND g.tgtype = 23", + // A guard narrowed to one column, or to a condition, is no guard. + "g.tgattr::text <> ''", + "bb._normal_when(bb._impact_when())", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } + for _, want := range []string{ + "r.command_tag = 'CREATE TRIGGER' AND r.object_type = 'trigger' AND NOT bb._override_in_force()", + "AND NOT bb._is_own_trigger(t.oid);", + "t.tgname LIKE 'bb\\_%' AND n.nspname NOT IN ('bb','bb_f1')", + "'CREATE TRIGGER','trigger',%L,%L,'BLOCKED','policy'", + } { + if !strings.Contains(SchemaData, want) { + t.Errorf("datachanges.sql no longer contains %q", want) + } + } +} + +// The recorder's skip list goes by name, so it must also ask for the +// definition, or a replaced guard is passed over as plumbing. +func TestRecorderSkipsOnlyTheInstalledGuards(t *testing.T) { + if !strings.Contains(Schema, "guard_impact) on '\n AND bb._is_own_trigger(r.objid) THEN CONTINUE;") { + t.Error("log_ddl_end skips a trigger by its name alone") + } + // The check is defined in ledger.sql beside its caller, for a recorder + // upgraded before datachanges.sql has been re-run. + if strings.Index(Schema, "FUNCTION bb._is_own_trigger") > strings.Index(Schema, "FUNCTION bb.log_ddl_end") { + t.Error("bb._is_own_trigger must be defined before bb.log_ddl_end") + } +} diff --git a/internal/ledger/identity.sql b/internal/ledger/identity.sql index 807f84a8..deb2a151 100644 --- a/internal/ledger/identity.sql +++ b/internal/ledger/identity.sql @@ -30,10 +30,10 @@ SET session_replication_role = replica; -- ── The private half ──────────────────────────────────────────────────────── --- ledger.sql grants db_client EXECUTE on every function in schema bb, on every --- start. The MAC and the secret must never be callable by a client (a client --- that could compute a MAC could forge an actor), so they live in a schema no --- one but its owner can use. +-- The MAC and the secret must never be callable by a client (a client that +-- could compute a MAC could forge an actor), so they live in a schema no one +-- but its owner can use, rather than relying on bb's list of functions a +-- client may call (ledger.sql, bb._client_callable). CREATE SCHEMA IF NOT EXISTS bb_f1; REVOKE ALL ON SCHEMA bb_f1 FROM PUBLIC; diff --git a/internal/ledger/ledger.go b/internal/ledger/ledger.go index eb49271e..389123e8 100644 --- a/internal/ledger/ledger.go +++ b/internal/ledger/ledger.go @@ -18,8 +18,8 @@ const SchemaName = "bb" // PrivateSchemaName holds what no client may reach: application identity's // install secret and the function that makes its MACs (identity.sql). A schema -// of its own because ledger.sql grants db_client EXECUTE on everything in -// SchemaName. Like SchemaName it is the engine's, never a user's: everything +// of its own so that the secret never depends on what SchemaName exposes to +// db_client. Like SchemaName it is the engine's, never a user's: everything // that lists, counts, guards or tracks a user's tables leaves both out. const PrivateSchemaName = "bb_f1" diff --git a/internal/ledger/ledger.sql b/internal/ledger/ledger.sql index 0a2bba3e..a16be1a3 100644 --- a/internal/ledger/ledger.sql +++ b/internal/ledger/ledger.sql @@ -17,6 +17,7 @@ DROP EVENT TRIGGER IF EXISTS bb_guard_start; DROP EVENT TRIGGER IF EXISTS bb_log_end; DROP EVENT TRIGGER IF EXISTS bb_log_drop; +DROP EVENT TRIGGER IF EXISTS bb_seal_functions; CREATE EXTENSION IF NOT EXISTS dblink; CREATE SCHEMA IF NOT EXISTS bb; @@ -568,6 +569,33 @@ LANGUAGE sql STABLE AS $$ AND ((classid::bigint << 32) | objid::bigint) = hashtext('bb.schema_ledger')::bigint); $$; +-- The second connection a blocked attempt is recorded through. A wait inside it +-- is one Postgres cannot see as part of any deadlock (the client's backend is +-- waiting on a socket, not a lock), so nothing would ever break one: another +-- transaction sitting on the Blackbox's append lock used to hang the refused +-- client for as long as it stayed open. The timeouts are for that connection +-- alone, and short: a refusal should come back at once, and it comes back +-- whether or not the record could be written. +CREATE OR REPLACE FUNCTION bb._blocked_conn() RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user + || $o$ options='-c lock_timeout=2s -c statement_timeout=5s'$o$; +$$; + +-- Write a blocked attempt through that connection. Failing to record it is a +-- WARNING and never the reason the statement stops: every caller raises its +-- own refusal next, and a refusal must not turn into a hang or another error. +-- Not SECURITY DEFINER, and a client can't reach dblink: it is only of use from +-- the triggers, which run as the owner. +CREATE OR REPLACE FUNCTION bb._record_blocked(stmt text) RETURNS void +LANGUAGE plpgsql SET search_path = pg_catalog, public, bb AS $$ +BEGIN + PERFORM dblink_exec(bb._blocked_conn(), stmt); +EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox: %', SQLERRM; +END; +$$; + -- ddl_command_start: enforce guardrails BEFORE the command runs. -- SECURITY DEFINER: the ledger's triggers record history with the owner's -- privileges, so a non-superuser client can trigger them (by running DDL) without @@ -590,9 +618,7 @@ BEGIN IF bb._holds_chain_lock() THEN RAISE WARNING 'guardrail: this blocked attempt is not recorded in Blackbox, because this transaction has already written to it'; ELSE - PERFORM dblink_exec( - 'host=/var/run/postgresql dbname=' || current_database() || ' user=' || current_user, - format($f$INSERT INTO bb.schema_ledger + PERFORM bb._record_blocked(format($f$INSERT INTO bb.schema_ledger (actor,actor_kind,tool,session,branch,command_tag,statement,status,risk) VALUES (%L,%L,%L,%L,%L,%L,%L,'BLOCKED','policy')$f$, c.actor, c.actor_kind, c.tool, c.session, c.branch, TG_TAG, current_query())); @@ -603,6 +629,59 @@ BEGIN END; $$; +-- Is this trigger, named bb_something, exactly what the Blackbox installs under +-- that name? A trigger's name is all the recorder's skip list and the data +-- guards go by, and CREATE OR REPLACE TRIGGER keeps the name while changing what +-- it does, so the definition is compared instead: the function, when it fires +-- (tgtype), no column list (tgattr), and the WHEN clause (tgqual). The WHEN is +-- compared as Postgres prints it, with parentheses, casts and spaces taken +-- away from both sides, since the same expression prints differently between +-- major versions. +CREATE OR REPLACE FUNCTION bb._normal_when(e text) RETURNS text +LANGUAGE sql IMMUTABLE AS $$ + SELECT regexp_replace(lower(e), '::[a-z_ ]+(\[\])?|[()[:space:]]', '', 'g'); +$$; + +CREATE OR REPLACE FUNCTION bb._is_own_trigger(t oid) RETURNS boolean +LANGUAGE plpgsql STABLE SET search_path = pg_catalog, public, bb AS $$ +DECLARE g pg_trigger; a text[]; +BEGIN + SELECT * INTO g FROM pg_trigger WHERE oid = t; + IF NOT FOUND THEN + RETURN false; + END IF; + -- UPDATE OF a column nobody writes, a transition table, or a constraint trigger. + IF g.tgattr::text <> '' OR g.tgconstraint <> 0 OR g.tgdeferrable + OR g.tgoldtable IS NOT NULL OR g.tgnewtable IS NOT NULL THEN + RETURN false; + END IF; + CASE g.tgname + WHEN 'bb_guard_truncate' THEN -- BEFORE TRUNCATE, per statement + RETURN g.tgfoid = to_regprocedure('bb.guard_truncate()') AND g.tgtype = 34 AND g.tgqual IS NULL AND g.tgnargs = 0; + WHEN 'bb_record_dml' THEN -- AFTER UPDATE OR DELETE, per statement + RETURN g.tgfoid = to_regprocedure('bb.record_dml()') AND g.tgtype = 24 AND g.tgqual IS NULL AND g.tgnargs = 0; + WHEN 'bb_guard_dml' THEN -- BEFORE UPDATE OR DELETE, per statement + RETURN g.tgfoid = to_regprocedure('bb.guard_dml()') AND g.tgtype = 26 AND g.tgqual IS NULL AND g.tgnargs = 0; + WHEN 'bb_guard_impact' THEN -- BEFORE UPDATE OR DELETE, per row, with bb._impact_when + RETURN g.tgfoid = to_regprocedure('bb.guard_impact()') AND g.tgtype = 27 AND g.tgnargs = 0 + AND bb._normal_when(substring(pg_get_triggerdef(t) FROM ' WHEN \((.*)\) EXECUTE ')) = bb._normal_when(bb._impact_when()); + WHEN 'bb_track' THEN -- BEFORE INSERT OR UPDATE, per row + IF NOT (g.tgfoid = to_regprocedure('bb.track_row()') AND g.tgtype = 23 AND g.tgqual IS NULL AND g.tgnargs IN (3, 4)) THEN + RETURN false; + END IF; + -- Which columns are the owner's choice, but they must be columns: + -- arguments naming nothing would leave a tracking trigger that writes nothing. + a := string_to_array(encode(g.tgargs, 'escape'), '\000'); + RETURN EXISTS (SELECT 1 FROM unnest(a) x WHERE x <> '') + AND NOT EXISTS (SELECT 1 FROM unnest(a) x WHERE x <> '' + AND NOT EXISTS (SELECT 1 FROM pg_attribute + WHERE attrelid = g.tgrelid AND attname = x AND attnum > 0 AND NOT attisdropped)); + ELSE + RETURN false; + END CASE; +END; +$$; + -- ddl_command_end: record CREATE/ALTER changes (drops are recorded in sql_drop). CREATE OR REPLACE FUNCTION bb.log_ddl_end() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, bb AS $$ @@ -626,8 +705,10 @@ BEGIN IF r.command_tag IN ('CREATE SEQUENCE','ALTER SEQUENCE') THEN CONTINUE; END IF; IF r.command_tag = 'CREATE INDEX' AND r.object_identity ~ '_pkey$' THEN CONTINUE; END IF; -- The Blackbox's own data-change triggers (datachanges.sql), attached to - -- every table: plumbing, like the objects in schema bb. - IF r.command_tag = 'CREATE TRIGGER' AND r.object_identity ~ '^bb_(guard_truncate|record_dml|guard_dml|guard_impact) on ' THEN CONTINUE; END IF; + -- every table: plumbing, like the objects in schema bb. Only as installed: + -- one replaced under the override is a change, and is recorded. + IF r.command_tag = 'CREATE TRIGGER' AND r.object_identity ~ '^bb_(guard_truncate|record_dml|guard_dml|guard_impact) on ' + AND bb._is_own_trigger(r.objid) THEN CONTINUE; END IF; st := 'APPLIED'; rk := NULL; IF r.command_tag = 'ALTER TABLE' AND EXISTS (SELECT 1 FROM unnest(t) x WHERE x ~* '\malter\M[^;]*\mtype\M') THEN st := 'FLAGGED'; rk := 'type-change'; @@ -696,20 +777,65 @@ LANGUAGE sql IMMUTABLE AS $$ coalesce(r.risk,''), 'UTF8')), 'hex'); $$; +-- The head of the chain, one row: the last id handed out and its row_hash. The +-- tail used to be read with SELECT … ORDER BY id DESC, under whatever snapshot +-- the transaction had, and in REPEATABLE READ or SERIALIZABLE that snapshot can +-- be older than the lock: two rows chained off one parent, and verify called it +-- tampering. An UPDATE of a row changed since the snapshot fails with a +-- serialization error instead, so a stale writer retries rather than forks. +-- +-- One transaction, under the lock appends take: an append by the previous +-- bb.chain_row must not fall between reading the tail here and the new one +-- going in. Seeded from the tail on an existing ledger; nothing written so far +-- changes. The head only ever moves forward here: a tail ahead of it was +-- written by an older engine, and a tail behind it has lost rows, which the +-- next entry's link should go on showing. +BEGIN; +DO $$ BEGIN PERFORM pg_advisory_xact_lock(hashtext('bb.schema_ledger')); END $$; +CREATE TABLE IF NOT EXISTS bb.ledger_head ( + id boolean PRIMARY KEY DEFAULT true CHECK (id), + last_id bigint NOT NULL, + last_hash text NOT NULL +); +INSERT INTO bb.ledger_head (last_id, last_hash) +SELECT coalesce(t.id, 0), coalesce(t.row_hash, '') +FROM (SELECT 1) one LEFT JOIN LATERAL + (SELECT id, row_hash FROM bb.schema_ledger ORDER BY id DESC LIMIT 1) t ON true +ON CONFLICT (id) DO UPDATE SET last_id = EXCLUDED.last_id, last_hash = EXCLUDED.last_hash + WHERE EXCLUDED.last_id > bb.ledger_head.last_id; + +-- The id comes from the head too, after the lock, so id order is chain order +-- (the column's default is taken before the lock, by whoever got there first) +-- and a rolled-back entry leaves no gap. The default stays: it costs a sequence +-- value and keeps the sequence ahead of every id, should an older engine ever +-- write here again. +-- +-- A ledger emptied by the engine (a masked branch starts with a fresh one) starts +-- a new chain, with ids carrying on. Reading it is safe at this point: the UPDATE +-- above has proved this snapshot saw the head's latest change, and with it the +-- last entry. CREATE OR REPLACE FUNCTION bb.chain_row() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE prev text; BEGIN -- Serialize appends so two concurrent inserts can't chain off the same row. PERFORM pg_advisory_xact_lock(hashtext('bb.schema_ledger')); - SELECT row_hash INTO prev FROM bb.schema_ledger ORDER BY id DESC LIMIT 1; + UPDATE bb.ledger_head SET last_id = last_id + 1 RETURNING last_id, last_hash INTO NEW.id, prev; + IF NOT FOUND THEN + RAISE EXCEPTION 'bb.ledger_head has no row: restart the engine to restore it'; + END IF; + IF NOT EXISTS (SELECT 1 FROM bb.schema_ledger) THEN + prev := ''; + END IF; NEW.prev_hash := coalesce(prev, ''); NEW.row_hash := bb._ledger_hash(NEW); + UPDATE bb.ledger_head SET last_hash = NEW.row_hash; RETURN NEW; END; $$; CREATE OR REPLACE TRIGGER bb_chain BEFORE INSERT ON bb.schema_ledger FOR EACH ROW EXECUTE FUNCTION bb.chain_row(); +COMMIT; -- Append-only: the ledger records history, so history cannot be rewritten. CREATE OR REPLACE FUNCTION bb.deny_change() RETURNS trigger @@ -748,6 +874,33 @@ BEGIN END $$; ALTER ROLE db_agent SET role = db_client; +-- Defaults for client logins, for what the append lock does to everyone else. A +-- transaction that has recorded something holds that lock until it ends, and +-- every other session's DDL (and every agent write) queues behind it, so one +-- left open in a client stopped the database's history without an error. +-- idle_in_transaction_session_timeout = 10min the value the hardening check +-- recommends (conn.idle); it ends only a session that sat idle mid-transaction +-- lock_timeout = 60s a queued client gets an error, not an endless wait +-- No statement_timeout: long queries and migrations are legitimate work. These +-- are defaults a session can SET for itself. Set once for each role, so a value +-- an administrator chose stays; a role made since the last start has it from the +-- next one. The engine's own superuser logins are left without them. +DO $$ +DECLARE r record; s text; +BEGIN + FOR r IN SELECT oid, rolname FROM pg_roles + WHERE rolcanlogin AND NOT rolsuper + AND (rolname IN ('db_client', 'db_agent') OR pg_has_role(oid, 'db_client', 'MEMBER')) LOOP + FOREACH s IN ARRAY ARRAY['lock_timeout=60s', 'idle_in_transaction_session_timeout=10min'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_db_role_setting d, unnest(d.setconfig) c + WHERE d.setrole = r.oid AND d.setdatabase = 0 + AND split_part(c, '=', 1) = split_part(s, '=', 1)) THEN + EXECUTE format('ALTER ROLE %I SET %I = %L', r.rolname, split_part(s, '=', 1), split_part(s, '=', 2)); + END IF; + END LOOP; + END LOOP; +END $$; + -- Full access to application data (it owns what it creates; these cover what the -- superuser created, e.g. imported tables, now and in future). GRANT USAGE, CREATE ON SCHEMA public TO db_client; @@ -761,11 +914,79 @@ ALTER DEFAULT PRIVILEGES FOR ROLE dbadmin IN SCHEMA public GRANT ALL ON SEQUENCE -- INSERT/UPDATE/DELETE on the table at all — it cannot forge or rewrite entries. GRANT USAGE ON SCHEMA bb TO db_client; GRANT SELECT ON bb.schema_ledger, bb.policy, bb.policy_history TO db_client; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA bb TO db_client; --- …except the one that advances a run's statement position: a client calling it --- could line a blocked statement up with a harmless one's text. -REVOKE EXECUTE ON FUNCTION bb._statement_texts(text, text, text) FROM PUBLIC, db_client; --- …and dblink, wherever it lives. Only the Blackbox's SECURITY DEFINER triggers + +-- Functions. A SECURITY DEFINER function runs as the Blackbox's owner, so being +-- able to call one is being able to do what it does: bb._record_impact takes its +-- actor as arguments and writes a chained entry, bb._statement_texts moves a +-- run's statement position so a blocked statement matches a harmless one's text. +-- Postgres gives every new function to PUBLIC, and everything in bb used to be +-- granted to db_client on top, so all of them were callable. +-- +-- So: no client may call a SECURITY DEFINER function here unless it is listed. +-- The list is every function a client, the gateway or the console calls itself, +-- SECURITY DEFINER or not (bb.act_for is the only one that is today); nothing +-- else a client reaches needs its own grant. Trigger and event-trigger functions +-- are not on it: Postgres checks EXECUTE only when a trigger is made, and they +-- are made here, by the owner, so they fire for any client. So are the nested +-- calls among these functions: the inner call is checked against the owner. +-- SECURITY INVOKER helpers keep what they have; they can do no more than their caller. +CREATE OR REPLACE FUNCTION bb._client_callable() RETURNS name[] +LANGUAGE sql IMMUTABLE AS $$ + SELECT ARRAY[ + 'act_for', 'app_actor', 'app_actor_label', 'app_identity_mode', -- application identity + '_ledger_hash', '_ext_hash', -- verifying the record + 'track_table', 'untrack_table', 'tracked_tables', 'track_row', -- row tracking, by a table's owner + 'policy_check', 'blast_radius', -- previews in the console and MCP + 'ledger_v2_version', 'blackbox_provenance_version', + 'blackbox_policy_version', 'blackbox_impact_version' + ]::name[]; +$$; + +-- Take EXECUTE from clients on one function, unless it is listed above. +-- session_replication_role: the REVOKE is the Blackbox's own plumbing, not a +-- change anybody made. +CREATE OR REPLACE FUNCTION bb._seal_function(f oid) RETURNS void +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog SET session_replication_role = replica AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_proc p WHERE p.oid = f AND p.prosecdef AND NOT p.proname = ANY (bb._client_callable())) THEN + EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC, db_client', f::regprocedure); + END IF; +END; +$$; + +-- A function added later, by any of the schema files that run after this one +-- or by a future release, is sealed as it is made. The files that come after +-- are run with session_replication_role = replica or must not REVOKE at all +-- (see datachanges.sql), so the event trigger below is what covers a new +-- install, where those functions do not exist yet at this point in the file. +CREATE OR REPLACE FUNCTION bb.seal_new_functions() RETURNS event_trigger +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $$ +DECLARE r record; +BEGIN + FOR r IN SELECT objid FROM pg_event_trigger_ddl_commands() + WHERE object_type IN ('function', 'procedure') AND schema_name = 'bb' LOOP + PERFORM bb._seal_function(r.objid); + END LOOP; +END; +$$; + +-- And every function already here, on every start: this is what closes an +-- existing install. A CREATE OR REPLACE keeps a function's privileges, so what +-- is sealed stays sealed. +DO $seal$ +DECLARE p record; +BEGIN + FOR p IN SELECT f.oid, f.proname FROM pg_proc f JOIN pg_namespace n ON n.oid = f.pronamespace + WHERE n.nspname = 'bb' AND f.prokind IN ('f', 'p') LOOP + IF p.proname = ANY (bb._client_callable()) THEN + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO db_client', p.oid::regprocedure); + ELSE + PERFORM bb._seal_function(p.oid); + END IF; + END LOOP; +END $seal$; + +-- dblink, too, wherever it lives. Only the Blackbox's SECURITY DEFINER triggers -- call it, and they run as the owner. A client holding it could open -- connections from inside the database to anywhere the container can reach, -- with any credentials it has or has learned (audit GW-1). @@ -795,3 +1016,9 @@ CREATE EVENT TRIGGER bb_log_end ON ddl_command_end EXECUTE FUNCTION bb.log_ddl_end(); CREATE EVENT TRIGGER bb_log_drop ON sql_drop EXECUTE FUNCTION bb.log_ddl_drop(); +-- ALWAYS: most of the schema files that follow run with session_replication_role +-- = replica, and a function they make must be sealed all the same. +CREATE EVENT TRIGGER bb_seal_functions ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION', 'ALTER FUNCTION', 'CREATE PROCEDURE', 'ALTER PROCEDURE', 'ALTER ROUTINE') + EXECUTE FUNCTION bb.seal_new_functions(); +ALTER EVENT TRIGGER bb_seal_functions ENABLE ALWAYS; diff --git a/internal/ledger/sealing_test.go b/internal/ledger/sealing_test.go new file mode 100644 index 00000000..2f23a927 --- /dev/null +++ b/internal/ledger/sealing_test.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ledger + +import ( + "regexp" + "strings" + "testing" +) + +// allowedCallable reads ledger.sql's list of what a client may call. +func allowedCallable(t *testing.T) map[string]bool { + t.Helper() + i := strings.Index(Schema, "CREATE OR REPLACE FUNCTION bb._client_callable()") + if i < 0 { + t.Fatal("bb._client_callable is gone") + } + body := Schema[i:] + body = body[:strings.Index(body, "$$;")] + body = body[strings.Index(body, "SELECT ARRAY["):] + allowed := map[string]bool{} + for _, m := range regexp.MustCompile(`'(_?[a-z0-9_]+)'`).FindAllStringSubmatch(body, -1) { + allowed[m[1]] = true + } + return allowed +} + +// The ledger's entries are forged through what a client can call, so nothing a +// client can call may be a helper that writes one. Nothing grants the whole +// schema any more, and the seal is applied both to what exists and to what is +// made later (which, for most schema files, is made under a replica session). +func TestClientsCannotCallTheHelpers(t *testing.T) { + if strings.Contains(Schema, "ON ALL FUNCTIONS IN SCHEMA bb") { + t.Error("ledger.sql grants the whole of schema bb again") + } + for _, want := range []string{ + "format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC, db_client', f::regprocedure)", + "AND p.prosecdef AND NOT p.proname = ANY (bb._client_callable())", + "WHEN TAG IN ('CREATE FUNCTION', 'ALTER FUNCTION', 'CREATE PROCEDURE', 'ALTER PROCEDURE', 'ALTER ROUTINE')", + "ALTER EVENT TRIGGER bb_seal_functions ENABLE ALWAYS;", + "SET search_path = pg_catalog SET session_replication_role = replica", + } { + if !strings.Contains(Schema, want) { + t.Errorf("ledger.sql no longer contains %q", want) + } + } + // On every start, before capture is switched on, like the other privileges. + sweep := strings.Index(Schema, "DO $seal$") + on := strings.Index(Schema, "CREATE EVENT TRIGGER bb_guard_start") + if sweep < 0 || on < 0 || sweep > on { + t.Errorf("the sweep over existing functions must run before capture is on (sweep %d, capture %d)", sweep, on) + } + + allowed := allowedCallable(t) + for _, forbidden := range []string{"_record_impact", "_attach_data_triggers", "_statement_texts", + "guard_ddl_start", "log_ddl_end", "guard_truncate", "capture_ext", "_capture_disabled"} { + if allowed[forbidden] { + t.Errorf("bb.%s is client-callable: a client could use the owner's rights through it", forbidden) + } + } +} + +// Everything the client-side code calls as a client is on the list. What the +// engine calls as its superuser needs no grant, and neither does a function +// the docs name only for operators. +func TestClientCalledFunctionsAreOnTheList(t *testing.T) { + allowed := allowedCallable(t) + for _, name := range []string{"act_for", "app_actor", "app_actor_label", "app_identity_mode", + "_ledger_hash", "_ext_hash", "policy_check", "blast_radius"} { + if !allowed[name] { + t.Errorf("bb.%s is called by clients and is not on the list", name) + } + } +} + +// bb.act_for is the one SECURITY DEFINER function a client may call. The list +// must not grow, by accident, to include another that writes the ledger. +func TestOnlyActForIsAClientCallableDefiner(t *testing.T) { + allowed := allowedCallable(t) + def := regexp.MustCompile(`(?is)FUNCTION bb\.([a-z0-9_]+)\([^)]*\)[^;$]*?SECURITY DEFINER`) + for _, sql := range []string{Schema, SchemaData, SchemaExt, SchemaTracking, SchemaIdentity} { + for _, m := range def.FindAllStringSubmatch(sql, -1) { + if allowed[m[1]] && m[1] != "act_for" { + t.Errorf("bb.%s is SECURITY DEFINER and client-callable: review it, then allow it here", m[1]) + } + } + } +} diff --git a/internal/ledger/statement_test.go b/internal/ledger/statement_test.go index 67dc4e33..f8205e53 100644 --- a/internal/ledger/statement_test.go +++ b/internal/ledger/statement_test.go @@ -32,11 +32,10 @@ func TestSchemaStatementMatching(t *testing.T) { } // A client that could call the position function could line a blocked statement - // up with a harmless one's text. The revoke must come after the blanket grant. - grant := strings.Index(ledgerSQL, "GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA bb TO db_client;") - revoke := strings.Index(ledgerSQL, "REVOKE EXECUTE ON FUNCTION bb._statement_texts(text, text, text) FROM PUBLIC, db_client;") - if grant < 0 || revoke < 0 || revoke < grant { - t.Errorf("_statement_texts must be revoked from clients after the blanket grant (grant at %d, revoke at %d)", grant, revoke) + // up with a harmless one's text. It is SECURITY DEFINER and not on the list of + // what clients may call, which is what seals it (sealing_test.go). + if strings.Contains(ledgerSQL, "'_statement_texts'") { + t.Error("_statement_texts must not be client-callable") } if regexp.MustCompile(`(?i)grant[^;]*bb\.statement_cursor[^;]*to`).MatchString(ledgerSQL) { t.Error("clients must not be granted anything on bb.statement_cursor") diff --git a/internal/proxy/auth.go b/internal/proxy/auth.go index 8516c01a..194ea95e 100644 --- a/internal/proxy/auth.go +++ b/internal/proxy/auth.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/binary" + "errors" "fmt" "io" "net" @@ -26,14 +27,35 @@ import ( // --- low-level message framing (post-startup Postgres protocol) --- +const ( + // maxBackendMessage bounds a message from a branch's own Postgres. + maxBackendMessage = 1 << 24 + + // maxPasswordMessage bounds the one message read from a client before it has + // authenticated, length field included. It is sent in the clear-text form + // only, and holds an API key of well under 100 bytes; SCRAM is spoken to the + // backend, never to a client. Without this, each connection anyone opened + // could have the gateway allocate 16 MiB by claiming to send that much. + maxPasswordMessage = 1024 +) + +// errBadMsgLength is a message whose length field is out of bounds. +var errBadMsgLength = errors.New("bad message length") + func readMsg(c net.Conn) (byte, []byte, error) { + return readMsgMax(c, maxBackendMessage) +} + +// readMsgMax is readMsg for a peer held to a smaller limit, checked before +// anything is allocated for the body. +func readMsgMax(c net.Conn, limit uint32) (byte, []byte, error) { h := make([]byte, 5) if _, err := io.ReadFull(c, h); err != nil { return 0, nil, err } n := binary.BigEndian.Uint32(h[1:5]) - if n < 4 || n > 1<<24 { - return 0, nil, fmt.Errorf("bad message length %d", n) + if n < 4 || n > limit { + return 0, nil, fmt.Errorf("%w: %d", errBadMsgLength, n) } body := make([]byte, n-4) if _, err := io.ReadFull(c, body); err != nil { @@ -61,8 +83,11 @@ func requestClientPassword(client net.Conn) (string, error) { if err := writeMsg(client, 'R', req); err != nil { return "", err } - typ, body, err := readMsg(client) + typ, body, err := readMsgMax(client, maxPasswordMessage) if err != nil { + if errors.Is(err, errBadMsgLength) { + sendError(client, "08P01", "invalid password message length") // protocol_violation + } return "", err } if typ != 'p' { @@ -172,7 +197,7 @@ func scramSHA256(backend net.Conn, mechanisms []byte, password string) error { if err != nil { return err } - if typ != 'R' || binary.BigEndian.Uint32(body[:4]) != 11 { + if typ != 'R' || len(body) < 4 || binary.BigEndian.Uint32(body[:4]) != 11 { return fmt.Errorf("expected SASLContinue") } serverFirst := string(body[4:]) diff --git a/internal/proxy/gateway_throttle_test.go b/internal/proxy/gateway_throttle_test.go new file mode 100644 index 00000000..d55fb5bc --- /dev/null +++ b/internal/proxy/gateway_throttle_test.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxy + +import ( + "bytes" + "crypto/tls" + "encoding/binary" + "io" + "net" + "path/filepath" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" +) + +// connectFrom runs one connection through handle from a public address, over +// TLS as any such client must be, and returns what the Gateway answered first +// and the SQLSTATE it finally refused with. +func connectFrom(t *testing.T, ip, password string) (first byte, code string) { + t.Helper() + client, server := net.Pipe() + defer client.Close() + done := make(chan struct{}) + go func() { + defer close(done) + remote, _ := net.ResolveTCPAddr("tcp", ip+":50000") + handle(addrConn{Conn: server, remote: remote}) + }() + _ = client.SetDeadline(time.Now().Add(10 * time.Second)) + ssl := make([]byte, 8) + binary.BigEndian.PutUint32(ssl[0:], 8) + binary.BigEndian.PutUint32(ssl[4:], codeSSL) + if _, err := client.Write(ssl); err != nil { + t.Fatal(err) + } + if _, err := io.ReadFull(client, ssl[:1]); err != nil || ssl[0] != 'S' { + t.Fatalf("SSLRequest: %q %v", ssl[0], err) + } + tc := tls.Client(client, &tls.Config{InsecureSkipVerify: true}) + if _, err := tc.Write(buildStartup(map[string]string{"user": "u", "database": "main"})); err != nil { + t.Fatal(err) + } + typ, body, err := readMsg(tc) + if err != nil { + t.Fatal(err) + } + first = typ + if typ == 'R' { // the password asked for + if err := writeMsg(tc, 'p', append([]byte(password), 0)); err != nil { + t.Fatal(err) + } + if typ, body, err = readMsg(tc); err != nil { + t.Fatal(err) + } + } + if typ == 'E' { + for _, f := range bytes.Split(body, []byte{0}) { + if len(f) > 1 && f[0] == 'C' { + code = string(f[1:]) + } + } + } + <-done + return first, code +} + +// The whole refusal path through handle: invalid keys coalesce into one event, +// an address that has used up its failures is turned away before it is even +// asked for a key, and another address is still asked. +func TestGatewayThrottlesInvalidKeys(t *testing.T) { + store, err := auth.Open(auth.Config{DBPath: filepath.Join(t.TempDir(), "a.db"), WebOrigin: "http://x"}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + prevStore, prevTLS := authStore, tlsConfig + authStore, tlsConfig = store, testTLSConfig(t) + defer func() { authStore, tlsConfig = prevStore, prevTLS }() + + const attacker = "203.0.113.7" + attempts := 0 + for ; attempts < 200; attempts++ { + first, code := connectFrom(t, attacker, "key_nonsense") + if first != 'R' { + if first != 'E' || code != "28000" { + t.Fatalf("attempt %d: first answer %q, code %q", attempts, first, code) + } + break + } + if code != "28P01" { + t.Fatalf("attempt %d: code %q, want 28P01", attempts, code) + } + } + if attempts == 200 { + t.Fatal("never throttled") + } + if first, code := connectFrom(t, "203.0.113.8", "key_nonsense"); first != 'R' || code != "28P01" { + t.Fatalf("another address: first %q, code %q", first, code) + } + evs, err := store.SecurityEvents(0, 0) + if err != nil { + t.Fatal(err) + } + n := 0 + for _, e := range evs { + if e.Kind == auth.EvGatewayRefused && e.IP == attacker { + n++ + } + } + if n != 1 || len(evs) != 2 { + t.Fatalf("after %d refusals from one address: %d events for it, %d in all, want 1 and 2", attempts, n, len(evs)) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index c3e44f51..aac723fa 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -85,6 +85,11 @@ const ( codeSSL = 80877103 // SSLRequest codeGSS = 80877104 // GSSENCRequest codeCancel = 80877102 // CancelRequest + + // maxStartupPacket is the longest startup packet accepted, length field + // included: Postgres's own MAX_STARTUP_PACKET_LENGTH, so no client that + // works with Postgres is refused here. + maxStartupPacket = 10000 ) // errHandledCancel signals that readStartup already handled a CancelRequest, so @@ -174,6 +179,7 @@ func relayStartupCaptureKey(client, backend net.Conn, addr string) (string, erro // auto-resuming suspended branches on connect and auto-suspending branches idle // for longer than idle (0 disables suspension). func Serve(addr string, idle time.Duration) error { + branch.OnWake = func(name, state string) { log.Printf("auto-resume: waking %s (was %s)", name, state) } if gatewayNoAuth() { log.Printf("gateway authentication DISABLED (FOX_GATEWAY_NOAUTH) — trusted/local mode") } else { @@ -293,6 +299,12 @@ func handle(client net.Conn) { var agentKey bool var account auth.User if authStore != nil { + // Before the password is even asked for: an address that has used up + // its invalid keys costs no key lookup and no row in the security log. + if held, wait := authStore.GatewayThrottled(params["database"], remoteHost(client)); held { + sendError(client, "28000", fmt.Sprintf("too many invalid API keys from this address — try again in %s", wait.Round(time.Second))) + return + } key, err := requestClientPassword(client) if err != nil { log.Printf("gateway auth: %v", err) @@ -300,7 +312,7 @@ func handle(client net.Conn) { } u, scope, kind, ok := authStore.VerifyKeyKind(key) if !ok { - authStore.Audit(auth.EvGatewayRefused, "", params["database"], remoteHost(client), "invalid API key") + authStore.GatewayInvalidKey(params["database"], remoteHost(client)) sendError(client, "28P01", "invalid API key — use a key_ key as the password") return } @@ -311,7 +323,7 @@ func handle(client net.Conn) { // Refused before the branch is resolved, so such a key cannot even // wake one. if kind == auth.KindRealtime { - authStore.Audit(auth.EvGatewayRefused, u.Email, params["database"], remoteHost(client), "realtime key at the gateway") + authStore.GatewayRefused(auth.EvGatewayRefused, u.Email, params["database"], remoteHost(client), "realtime key at the gateway") sendError(client, "28P01", "this is a realtime key: it subscribes to the change feed and cannot run SQL — use a key_ key as the password") return } @@ -342,7 +354,7 @@ func handle(client net.Conn) { } actor = keyScope } else if gatewayACL != nil && !gatewayACL.Can(account, target, access.Use) { - authStore.Audit(auth.EvDenied, account.Email, target, remoteHost(client), "gateway") + authStore.GatewayRefused(auth.EvDenied, account.Email, target, remoteHost(client), "gateway") // Answered like a branch that does not exist, so a name is not // confirmed to someone who may not open it — and checked before the // branch is woken. @@ -350,43 +362,27 @@ func handle(client net.Conn) { return } touch(target) - var addr string - if standbyRead { - // Never EnsureRunning: the standby is not a branch, nothing may start - // it on a client's behalf, and a stepped-down primary must not be - // revived by somebody opening a read-only session. - a, err := branch.StandbyReadAddr(target) - if err != nil { - log.Printf("route dbname=%q: %v", branch.StandbyReadName(target), err) - // A refusal that retrying cannot change is a name problem; one - // that waiting can — a standby mid-rebuild, or promoted and not - // yet failed back — tells the client to come back instead. - code := "57P03" // cannot_connect_now - if errors.Is(err, branch.ErrNoStandbyRead) { - code = "3D000" // invalid_catalog_name - } - sendError(client, code, err.Error()) - return - } - addr = a - } else { - if st := branch.ContainerState(target); st != "running" && st != "absent" { - log.Printf("auto-resume: waking %s (was %s)", target, st) - } - a, err := branch.EnsureRunning(target) // resumes the branch if suspended - if err != nil { - log.Printf("route dbname=%q: %v", target, err) - if errors.Is(err, branch.ErrBranchClosed) { - sendError(client, "57P03", err.Error()) // cannot_connect_now: a masked branch being made - return - } - sendError(client, "3D000", err.Error()) // invalid_catalog_name - return + route := func() (string, error) { + if standbyRead { + // Never EnsureRunning: the standby is not a branch, nothing may + // start it on a client's behalf, and a stepped-down primary must + // not be revived by somebody opening a read-only session. + return standbyAddr(target) } - addr = a + return ensureRunning(target) // resumes the branch if suspended + } + addr, err := route() + if err != nil { + sendRouteError(client, target, standbyRead, err) + return } - backend, err := net.Dial("tcp", addr) + backend, addr, err := connectBackend(addr, route) if err != nil { + var refused *routeFailure + if errors.As(err, &refused) { + sendRouteError(client, target, standbyRead, refused.err) + return + } log.Printf("dial backend %s: %v", addr, err) sendError(client, "08006", fmt.Sprintf("could not reach branch %q", target)) return @@ -502,6 +498,84 @@ func handle(client net.Conn) { <-done } +// The dials are bounded, unlike the rest of a session. The first is short +// because a backend on the same docker bridge answers in a millisecond, and the +// one that does not is an address the container has left. The second has come +// through a fresh look at docker and is allowed more. +const ( + backendDialFirst = 3 * time.Second + backendDialRetry = 10 * time.Second +) + +// The gateway's ways out to the engine, variables so a test can stand in for a +// branch that moved. +var ( + ensureRunning = branch.EnsureRunning + standbyAddr = branch.StandbyReadAddr + forgetBackends = branch.ForgetBackends + dialBackend = net.DialTimeout +) + +// routeFailure is a retry that could not find the branch at all, which the +// client is told about as a routing problem rather than a dial failure. +type routeFailure struct{ err error } + +func (r *routeFailure) Error() string { return r.err.Error() } + +// connectBackend dials addr, and when that fails, routes again and dials once +// more. +// +// The address may be seconds old (branch/statecache.go): a suspend or a restart +// run by another process leaves it behind and nothing can tell this process. So +// a failed dial says the memory was wrong, and it is dropped before routing +// again, which then finds a suspended branch and wakes it. The client sees a +// slightly slower connect instead of an error. Once only: a backend that fails +// twice is down, and more tries would only make the client wait to hear it. +func connectBackend(addr string, route func() (string, error)) (net.Conn, string, error) { + backend, err := dialBackend("tcp", addr, backendDialFirst) + if err == nil { + return backend, addr, nil + } + log.Printf("dial backend %s: %v (looking the branch up again)", addr, err) + forgetBackends() + again, rerr := route() + if rerr != nil { + return nil, addr, &routeFailure{err: rerr} + } + backend, err = dialBackend("tcp", again, backendDialRetry) + if err != nil { + return nil, again, err + } + return backend, again, nil +} + +// sendRouteError tells the client why its branch could not be routed to. +func sendRouteError(client net.Conn, target string, standbyRead bool, err error) { + name := target + if standbyRead { + name = branch.StandbyReadName(target) + } + log.Printf("route dbname=%q: %v", name, err) + sendError(client, routeErrorCode(standbyRead, err), err.Error()) +} + +// routeErrorCode separates a refusal that retrying cannot change, which is a +// name problem, from one that waiting can — a standby mid-rebuild or promoted +// and not yet failed back, a masked branch being made, a docker that did not +// answer in time — which tells the client to come back instead. +func routeErrorCode(standbyRead bool, err error) string { + const cannotConnectNow, invalidCatalog = "57P03", "3D000" + switch { + case errors.Is(err, branch.ErrStateUnknown): + return cannotConnectNow + case standbyRead && !errors.Is(err, branch.ErrNoStandbyRead): + return cannotConnectNow + case errors.Is(err, branch.ErrBranchClosed): + return cannotConnectNow + } + return invalidCatalog +} + // readStartup reads the startup phase and returns the startup parameters (user, // database, ...). When TLS is configured it accepts an SSLRequest, upgrades the // connection, and returns the wrapped conn — the caller must use the returned @@ -512,8 +586,10 @@ func readStartup(client net.Conn) (net.Conn, map[string]string, error) { if _, err := io.ReadFull(client, header); err != nil { return client, nil, err } - msgLen := int(binary.BigEndian.Uint32(header)) - if msgLen < 8 || msgLen > 1<<20 { + // A uint32 kept whole: on a 32-bit build int() could turn a huge length negative. + msgLen := binary.BigEndian.Uint32(header) + if msgLen < 8 || msgLen > maxStartupPacket { + sendError(client, "08P01", "invalid length of startup packet") // protocol_violation return client, nil, fmt.Errorf("bad startup length %d", msgLen) } body := make([]byte, msgLen-4) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 2d8c5e9b..e0ffcbc1 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -16,6 +16,8 @@ import ( "io" "math/big" "net" + "os" + "strings" "testing" "time" ) @@ -193,3 +195,155 @@ func TestSendError(t *testing.T) { t.Errorf("ErrorResponse missing SQLSTATE code: %q", got) } } + +// readErrorResponse reads one ErrorResponse and returns its fields as text. +func readErrorResponse(t *testing.T, c net.Conn) string { + t.Helper() + h := make([]byte, 5) + if _, err := io.ReadFull(c, h); err != nil { + t.Fatalf("reading the error: %v", err) + } + if h[0] != 'E' { + t.Fatalf("expected an ErrorResponse, got %q", h[0]) + } + body := make([]byte, binary.BigEndian.Uint32(h[1:])-4) + if _, err := io.ReadFull(c, body); err != nil { + t.Fatalf("reading the error body: %v", err) + } + return string(body) +} + +// A startup packet that claims too much is refused on its header alone: the +// test never sends the body, so a reader that waited for it would time out. +func TestReadStartupRefusesOversizedPacket(t *testing.T) { + for _, n := range []uint32{maxStartupPacket + 1, 1 << 20, 1<<32 - 1, 4} { + client, server := net.Pipe() + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + _ = client.SetDeadline(time.Now().Add(3 * time.Second)) + go func() { + h := make([]byte, 4) + binary.BigEndian.PutUint32(h, n) + _, _ = client.Write(h) + }() + errc := make(chan error, 1) + go func() { _, _, err := readStartup(server); errc <- err }() + if got := readErrorResponse(t, client); !bytes.Contains([]byte(got), []byte("08P01")) { + t.Errorf("length %d: error %q, want SQLSTATE 08P01", n, got) + } + if err := <-errc; err == nil || errors.Is(err, os.ErrDeadlineExceeded) { + t.Errorf("length %d: readStartup = %v", n, err) + } + client.Close() + server.Close() + } +} + +// The largest packet Postgres itself would take still gets through. +func TestReadStartupAcceptsLargestPacket(t *testing.T) { + params := map[string]string{"user": "u", "database": "main"} + params["options"] = strings.Repeat("x", maxStartupPacket-len(buildStartup(params))-len("options")-2) + pkt := buildStartup(params) + if len(pkt) != maxStartupPacket { + t.Fatalf("test packet is %d bytes, want %d", len(pkt), maxStartupPacket) + } + client, server := net.Pipe() + defer client.Close() + defer server.Close() + go func() { _, _ = client.Write(pkt) }() + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + _, got, err := readStartup(server) + if err != nil { + t.Fatalf("readStartup: %v", err) + } + if got["database"] != "main" { + t.Errorf("database = %q", got["database"]) + } +} + +// startPasswordRead runs requestClientPassword against a pipe and consumes the +// AuthenticationCleartextPassword request it sends first. +func startPasswordRead(t *testing.T) (net.Conn, chan error, *string) { + t.Helper() + client, server := net.Pipe() + t.Cleanup(func() { client.Close(); server.Close() }) + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + _ = client.SetDeadline(time.Now().Add(3 * time.Second)) + errc := make(chan error, 1) + got := new(string) + go func() { + var err error + *got, err = requestClientPassword(server) + errc <- err + }() + if _, err := io.ReadFull(client, make([]byte, 9)); err != nil { + t.Fatalf("reading the auth request: %v", err) + } + return client, errc, got +} + +// A client that has not authenticated is held to a small password message, on +// the header alone (no body is ever sent here) and with a real protocol error. +func TestPasswordMessageIsCapped(t *testing.T) { + for _, n := range []uint32{maxPasswordMessage + 1, 1 << 24, 3} { + client, errc, _ := startPasswordRead(t) + h := []byte{'p', 0, 0, 0, 0} + binary.BigEndian.PutUint32(h[1:], n) + if _, err := client.Write(h); err != nil { + t.Fatal(err) + } + if got := readErrorResponse(t, client); !bytes.Contains([]byte(got), []byte("08P01")) { + t.Errorf("length %d: error %q, want SQLSTATE 08P01", n, got) + } + if err := <-errc; !errors.Is(err, errBadMsgLength) { + t.Errorf("length %d: %v, want errBadMsgLength", n, err) + } + } +} + +// An API key, and anything up to the limit, is still read. +func TestPasswordMessageWithinLimit(t *testing.T) { + for _, secret := range []string{"key_" + strings.Repeat("a", 64), strings.Repeat("k", maxPasswordMessage-5)} { + client, errc, got := startPasswordRead(t) + if err := writeMsg(client, 'p', append([]byte(secret), 0)); err != nil { + t.Fatal(err) + } + if err := <-errc; err != nil { + t.Fatalf("%d-byte secret: %v", len(secret), err) + } + if *got != secret { + t.Errorf("read %q, want %q", *got, secret) + } + } +} + +// The limit is the client's alone: a backend may send much larger messages. +func TestBackendMessagesKeepTheirLimit(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + defer server.Close() + body := make([]byte, maxStartupPacket*4) + go func() { _ = writeMsg(client, 'S', body) }() + _ = server.SetDeadline(time.Now().Add(3 * time.Second)) + typ, got, err := readMsg(server) + if err != nil || typ != 'S' || len(got) != len(body) { + t.Fatalf("readMsg: %q, %d bytes, %v", typ, len(got), err) + } +} + +// A SASLContinue too short to hold its code used to index out of range. +func TestScramRefusesShortSASLContinue(t *testing.T) { + for _, short := range [][]byte{{}, {0, 0, 11}} { + gateway, backend := net.Pipe() + _ = gateway.SetDeadline(time.Now().Add(3 * time.Second)) + _ = backend.SetDeadline(time.Now().Add(3 * time.Second)) + go func() { + _, _, _ = readMsg(backend) // SASLInitialResponse + _ = writeMsg(backend, 'R', short) + }() + if err := scramSHA256(gateway, []byte("SCRAM-SHA-256\x00\x00"), "pw"); err == nil { + t.Errorf("a %d-byte SASLContinue was accepted", len(short)) + } + gateway.Close() + backend.Close() + } +} diff --git a/internal/proxy/stale_test.go b/internal/proxy/stale_test.go new file mode 100644 index 00000000..9f605697 --- /dev/null +++ b/internal/proxy/stale_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxy + +import ( + "errors" + "net" + "testing" + "time" + + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// stubDial records the addresses dialled and fails the ones in down. +func stubDial(t *testing.T, down ...string) *[]string { + t.Helper() + dialled := &[]string{} + old, oldForget := dialBackend, forgetBackends + dialBackend = func(_, addr string, _ time.Duration) (net.Conn, error) { + *dialled = append(*dialled, addr) + for _, d := range down { + if d == addr { + return nil, errors.New("connect: no route to host") + } + } + a, b := net.Pipe() + t.Cleanup(func() { a.Close(); b.Close() }) + return a, nil + } + t.Cleanup(func() { dialBackend, forgetBackends = old, oldForget }) + return dialled +} + +// A branch another process suspended or moved leaves a remembered address that +// no longer answers. The client must not hear about that: the address is +// forgotten, the branch routed again, and the connection made. +func TestAStaleAddressIsForgottenAndRoutedAgain(t *testing.T) { + dialled := stubDial(t, "10.0.0.1:5432") + forgot := 0 + forgetBackends = func() { forgot++ } + routes := 0 + conn, addr, err := connectBackend("10.0.0.1:5432", func() (string, error) { + routes++ + return "10.0.0.2:5432", nil + }) + if err != nil { + t.Fatalf("a stale address reached the client: %v", err) + } + defer conn.Close() + if addr != "10.0.0.2:5432" || routes != 1 || forgot != 1 { + t.Fatalf("addr %s after %d routes and %d forgets, want the new address after one of each", addr, routes, forgot) + } + if len(*dialled) != 2 { + t.Fatalf("dialled %v, want the old address and then the new one", *dialled) + } +} + +// A working address costs nothing but the dial. +func TestAGoodAddressIsNotRoutedAgain(t *testing.T) { + stubDial(t) + forgetBackends = func() { t.Fatal("forgot a working address") } + conn, _, err := connectBackend("10.0.0.1:5432", func() (string, error) { + t.Fatal("routed again after a good dial") + return "", nil + }) + if err != nil { + t.Fatal(err) + } + conn.Close() +} + +// One retry only: a backend that is really down is reported, not chased. +func TestABackendDownTwiceIsReported(t *testing.T) { + dialled := stubDial(t, "10.0.0.1:5432") + forgetBackends = func() {} + _, addr, err := connectBackend("10.0.0.1:5432", func() (string, error) { return "10.0.0.1:5432", nil }) + if err == nil { + t.Fatal("two failed dials were reported as a connection") + } + var refused *routeFailure + if errors.As(err, &refused) { + t.Fatalf("a dial failure was reported as a routing problem: %v", err) + } + if addr != "10.0.0.1:5432" || len(*dialled) != 2 { + t.Fatalf("addr %q, dialled %v: want exactly two tries", addr, *dialled) + } +} + +// A retry can find the branch gone, which is a routing answer. +func TestARouteFailureOnRetryIsARoutingAnswer(t *testing.T) { + stubDial(t, "10.0.0.1:5432") + forgetBackends = func() {} + gone := errors.New("branch does not exist") + _, _, err := connectBackend("10.0.0.1:5432", func() (string, error) { return "", gone }) + var refused *routeFailure + if !errors.As(err, &refused) || !errors.Is(refused.err, gone) { + t.Fatalf("err = %v, want the routing failure", err) + } +} + +func TestRouteErrorCodes(t *testing.T) { + cases := []struct { + name string + standby bool + err error + want string + }{ + {"no such branch", false, errors.New("branch does not exist"), "3D000"}, + {"masked branch being made", false, branch.ErrBranchClosed, "57P03"}, + // A docker that did not answer is not a branch that does not exist: + // the client should come back. + {"docker did not answer", false, branch.ErrStateUnknown, "57P03"}, + {"no standby here", true, branch.ErrNoStandbyRead, "3D000"}, + {"standby mid-rebuild", true, errors.New("rebuilding"), "57P03"}, + } + for _, c := range cases { + if got := routeErrorCode(c.standby, c.err); got != c.want { + t.Errorf("%s: %s, want %s", c.name, got, c.want) + } + } +} diff --git a/internal/update/checksums.go b/internal/update/checksums.go index 369af8ca..5102f08c 100644 --- a/internal/update/checksums.go +++ b/internal/update/checksums.go @@ -14,7 +14,9 @@ import ( ) // SumsAsset is the checksum file every release publishes -// (scripts/release-checksums.sh writes it). +// (scripts/release-checksums.sh writes it, in release.yml's job). It lists every +// file an update needs; the Windows distro image has a file of its own, which +// only install.ps1 reads. const SumsAsset = "SHA256SUMS" // Checksums maps a file name to its lowercase hex SHA-256. diff --git a/internal/update/select.go b/internal/update/select.go index ec7af9d7..8a17e31f 100644 --- a/internal/update/select.go +++ b/internal/update/select.go @@ -5,6 +5,7 @@ package update import ( "bytes" "context" + "errors" "fmt" "sort" "strings" @@ -173,7 +174,8 @@ func Candidates(rels []Release, current Version, t Target, pin string) ([]Releas // // tag is a release tag, or "" / "latest" for the newest published release that // has every file this target needs. The returned Offer carries the release's -// SHA256SUMS, so Download verifies what it fetches. +// SHA256SUMS, checked against the release signature, so Download verifies what +// it fetches. There is no unsigned path: a missing or bad signature is an error. func (c *Client) Release(ctx context.Context, tag string, t Target) (*Offer, error) { rels, err := c.ListReleases(ctx) if err != nil { @@ -222,14 +224,9 @@ func (c *Client) Release(ctx context.Context, tag string, t Target) (*Offer, err } r := usable[0] - sa, _ := r.Asset(SumsAsset) - body, err := c.fetchSmall(ctx, sa.URL, 1<<20) - if err != nil { - return nil, fmt.Errorf("%s: fetching %s: %w", r.Tag, SumsAsset, err) - } - sums, err := ParseChecksums(bytes.NewReader(body)) + sums, err := c.verifiedSums(ctx, r) if err != nil { - return nil, fmt.Errorf("%s: %w", r.Tag, err) + return nil, err } for _, name := range required { if _, ok := sums[name]; !ok { @@ -240,6 +237,42 @@ func (c *Client) Release(ctx context.Context, tag string, t Target) (*Offer, err return &Offer{Release: r, Version: v, Sums: sums, Required: required}, nil } +// signatureError is a release whose SHA256SUMS failed its signature check, as +// opposed to one that could not be fetched. +type signatureError struct{ error } + +func (e signatureError) Unwrap() error { return e.error } + +// verifiedSums is the only way a release's checksums are read: it fetches +// SHA256SUMS and its signature, and parses the list only once the signature +// checks out, because nothing in SHA256SUMS is trusted before that. +func (c *Client) verifiedSums(ctx context.Context, r Release) (Checksums, error) { + sa, ok := r.Asset(SumsAsset) + if !ok { + return nil, fmt.Errorf("release %s has no %s", r.Tag, SumsAsset) + } + sigAsset, ok := r.Asset(SigAsset) + if !ok { + return nil, fmt.Errorf("release %s has no %s, so it cannot be verified", r.Tag, SigAsset) + } + body, err := c.fetchSmall(ctx, sa.URL, 1<<20) + if err != nil { + return nil, fmt.Errorf("%s: fetching %s: %w", r.Tag, SumsAsset, err) + } + sig, err := c.fetchSmall(ctx, sigAsset.URL, 1024) + if err != nil { + return nil, fmt.Errorf("%s: fetching %s: %w", r.Tag, SigAsset, err) + } + if err := VerifySums(body, sig); err != nil { + return nil, signatureError{fmt.Errorf("release %s: %w", r.Tag, err)} + } + sums, err := ParseChecksums(bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("%s: %w", r.Tag, err) + } + return sums, nil +} + // Available is the check `fox start` makes: the newest release worth telling the // user about, or nil when this install is up to date. // @@ -293,29 +326,14 @@ func (c *Client) Resolve(ctx context.Context, current Version, t Target, pin str if ctx.Err() != nil { return nil, ctx.Err() } - sa, _ := r.Asset(SumsAsset) - body, err := c.fetchSmall(ctx, sa.URL, 1<<20) - if err != nil { - lastErr = err - continue - } - // Nothing in SHA256SUMS is trusted until its signature checks out. - sigAsset, _ := r.Asset(SigAsset) - sig, err := c.fetchSmall(ctx, sigAsset.URL, 1024) + sums, err := c.verifiedSums(ctx, r) if err != nil { - lastErr = err - continue - } - if err := VerifySums(body, sig); err != nil { // Not "nothing new": a release that exists and does not verify is // either forged or broken, and saying "up to date" would hide it. - sigErr = fmt.Errorf("release %s: %w", r.Tag, err) - lastErr = sigErr - continue - } - sums, err := ParseChecksums(bytes.NewReader(body)) - if err != nil { - lastErr = fmt.Errorf("%s: %w", r.Tag, err) + if errors.As(err, new(signatureError)) { + sigErr = err + } + lastErr = err continue } var unlisted []string diff --git a/internal/update/update_test.go b/internal/update/update_test.go index aacda2f9..570dcd66 100644 --- a/internal/update/update_test.go +++ b/internal/update/update_test.go @@ -11,6 +11,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -503,6 +504,70 @@ func TestReleaseForSetup(t *testing.T) { } } +// `fox setup` installs an engine that runs as root, so Release holds its +// checksums to the same signature as Resolve: nothing comes back from a release +// that doesn't verify, whichever way it fails. +func TestReleaseRequiresTheSignature(t *testing.T) { + linux := Target{GOOS: "linux", HostArch: "amd64"} + ctx := context.Background() + _, other, _ := ed25519.GenerateKey(nil) + + t.Run("good", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "good"}) + o, err := f.client("").Release(ctx, "v0.9.1", linux) + if err != nil || o.Sums[engLinuxAMD64] == "" { + t.Fatalf("release = %+v, %v", o, err) + } + }) + t.Run("signed by another key", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "forged"}) + f.files["/dl/v0.9.1/SHA256SUMS.sig"] = ed25519.Sign(other, f.files["/dl/v0.9.1/SHA256SUMS"]) + for _, tag := range []string{"latest", "v0.9.1"} { + if o, err := f.client("").Release(ctx, tag, linux); o != nil || err == nil || + !strings.Contains(err.Error(), "not signed with the FoxByte release key") { + t.Errorf("%s: release = %+v, %v; want the signature error", tag, o, err) + } + } + }) + t.Run("altered after signing", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "good"}) + f.files["/dl/v0.9.1/SHA256SUMS"] = append(f.files["/dl/v0.9.1/SHA256SUMS"], '\n') + if o, err := f.client("").Release(ctx, "latest", linux); o != nil || err == nil { + t.Errorf("release = %+v, %v; want refused", o, err) + } + }) + t.Run("signature not attached", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "unsigned"}) + f.releases[0].Assets = f.releases[0].Assets[:len(f.releases[0].Assets)-1] + for _, tag := range []string{"latest", "v0.9.1"} { + if o, err := f.client("").Release(ctx, tag, linux); o != nil || err == nil { + t.Errorf("%s: release = %+v, %v; want refused", tag, o, err) + } + } + }) + t.Run("signature attached but unavailable", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "unsigned"}) + delete(f.files, "/dl/v0.9.1/SHA256SUMS.sig") + if o, err := f.client("").Release(ctx, "latest", linux); o != nil || err == nil || + !strings.Contains(err.Error(), SigAsset) { + t.Errorf("release = %+v, %v; want an error naming %s", o, err, SigAsset) + } + }) + t.Run("build with no release key", func(t *testing.T) { + f := newFakeGitHub(t) + f.publish("v0.9.1", map[string]string{engLinuxAMD64: "good"}) + releasePublicKey = "" // newFakeGitHub's cleanup restores it + if o, err := f.client("").Release(ctx, "latest", linux); o != nil || !errors.Is(err, ErrNoReleaseKey) { + t.Errorf("release = %+v, %v; want ErrNoReleaseKey", o, err) + } + }) +} + func TestShouldCheck(t *testing.T) { env := map[string]string{} get := func(k string) string { return env[k] } diff --git a/scripts/release-checksums.sh b/scripts/release-checksums.sh index 618b6f73..efc1a4b8 100755 --- a/scripts/release-checksums.sh +++ b/scripts/release-checksums.sh @@ -1,41 +1,42 @@ #!/usr/bin/env bash # SPDX-License-Identifier: AGPL-3.0-or-later # -# Publish this job's checksums into a GitHub release's SHA256SUMS without dropping -# anyone else's. +# Sign and publish the checksums of the files this job built, under a name only +# this job writes. # -# release-checksums.sh ... +# release-checksums.sh ... # -# Two workflows attach assets to the same release: release.yml (the fox and -# fox-verify binaries, the Docker context) and wsl-distro.yml (the Windows distro -# image). Both start from the same tag push and either can finish first. The -# installers verify every download against SHA256SUMS and silently skip any file -# that isn't listed, so a job that replaces the whole file with only its own -# entries quietly turns verification off for the other job's assets. That -# happened with v0.8.1. +# Two workflows attach assets to the same release, on their own schedules: +# release.yml (the fox and fox-verify binaries, the Docker context) publishes +# SHA256SUMS, and wsl-distro.yml (the Windows distro image) publishes +# SHA256SUMS.distro. Each file has its own signature, .sig. # -# So each job: -# 1. downloads the published SHA256SUMS (if any), -# 2. keeps every entry except those for the files it is publishing (matched by -# exact file name, not substring), -# 3. adds its own entries and uploads the result, -# 4. waits a moment, downloads again and checks its entries are still there — -# if the other job uploaded in between, it merges again (up to 3 times). +# This used to merge every job's entries into one SHA256SUMS: download the +# published file, add this job's lines, sign the result. That made the job a +# signing oracle, because anyone able to upload a release asset could put a line +# in the published file and have the release key sign it. So nothing read back +# from a release is ever signed here: the entries are hashed from the local +# files, which are this job's own build outputs, and a re-run simply replaces +# them. # -# Signing (audit v2 G22): with RELEASE_SIGN set to the releasesign tool -# (cmd/releasesign, which reads FOX_RELEASE_SIGNING_KEY), each upload of the -# merged SHA256SUMS goes with its signature, SHA256SUMS.sig, and step 4 also -# checks the published signature matches the published file. The two jobs can -# interleave (A's file, B's file, B's signature, A's signature), so a mismatch -# there is merged and signed again like any other race. +# The read-back afterwards only confirms what was uploaded is what is published +# (the release API can lag just behind an upload); it feeds nothing back in. +# +# RELEASE_SIGN is the releasesign tool (cmd/releasesign, which reads +# FOX_RELEASE_SIGNING_KEY). It is required: clients refuse an unsigned +# SHA256SUMS, so publishing one would only break the release. # # Needs gh (GH_TOKEN set) and sha256sum or shasum. Files may be given with a # directory; entries use the base name, as the installers expect. set -euo pipefail tag="${1:-}" -[ -n "$tag" ] && [ "$#" -ge 2 ] || { echo "usage: $0 ..." >&2; exit 2; } -shift +asset="${2:-}" +[ -n "$tag" ] && [ -n "$asset" ] && [ "$#" -ge 3 ] || { echo "usage: $0 ..." >&2; exit 2; } +shift 2 + +sign="${RELEASE_SIGN:-}" +[ -n "$sign" ] || { echo "error: RELEASE_SIGN is not set; a release's checksums must be signed" >&2; exit 2; } if command -v sha256sum >/dev/null 2>&1; then checksum() { sha256sum "$@"; } @@ -46,48 +47,26 @@ fi work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -# Our entries: " ". -checksum "$@" | awk '{ name = $2; sub(/.*\//, "", name); print $1 " " name }' > "$work/mine" +# Our entries, and nothing else: " ". +checksum "$@" | awk '{ name = $2; sub(/^\*/, "", name); sub(/.*\//, "", name); print $1 " " name }' | sort -k2 > "$work/$asset" + +"$sign" sign "$work/$asset" "$work/$asset.sig" +gh release upload "$tag" "$work/$asset" "$work/$asset.sig" --clobber -sign="${RELEASE_SIGN:-}" attempts="${CHECKSUM_ATTEMPTS:-3}" settle="${CHECKSUM_SETTLE_SECONDS:-10}" - +mkdir "$work/published" for attempt in $(seq 1 "$attempts"); do - # A release without SHA256SUMS yet starts from empty. - if ! gh release download "$tag" -p SHA256SUMS -O "$work/existing" --clobber >/dev/null 2>&1; then - : > "$work/existing" - fi - # Every published entry whose file we aren't publishing, then ours. - awk 'NR == FNR { mine[$2] = 1; next } NF >= 2 && !($2 in mine)' "$work/mine" "$work/existing" > "$work/SHA256SUMS" - cat "$work/mine" >> "$work/SHA256SUMS" - sort -k2 -o "$work/SHA256SUMS" "$work/SHA256SUMS" - - if [ -n "$sign" ]; then - "$sign" sign "$work/SHA256SUMS" "$work/SHA256SUMS.sig" - gh release upload "$tag" "$work/SHA256SUMS" "$work/SHA256SUMS.sig" --clobber - else - gh release upload "$tag" "$work/SHA256SUMS" --clobber - fi - sleep "$settle" - gh release download "$tag" -p SHA256SUMS -O "$work/published" --clobber - signed_ok=1 - if [ -n "$sign" ]; then - signed_ok=0 - if gh release download "$tag" -p SHA256SUMS.sig -O "$work/published.sig" --clobber >/dev/null 2>&1 && - "$sign" verify "$work/published" "$work/published.sig" >/dev/null 2>&1; then - signed_ok=1 - fi - fi - if [ "$signed_ok" = 1 ] && awk 'NR == FNR { want[$1 " " $2] = 1; total++; next } - (($1 " " $2) in want) && !seen[$1 " " $2]++ { found++ } - END { exit (found == total) ? 0 : 1 }' "$work/mine" "$work/published"; then - cat "$work/published" + if gh release download "$tag" -p "$asset" -O "$work/published/$asset" --clobber >/dev/null 2>&1 && + gh release download "$tag" -p "$asset.sig" -O "$work/published/$asset.sig" --clobber >/dev/null 2>&1 && + cmp -s "$work/$asset" "$work/published/$asset" && + cmp -s "$work/$asset.sig" "$work/published/$asset.sig"; then + cat "$work/$asset" exit 0 fi - echo "SHA256SUMS changed while publishing (another job uploaded at the same time); merging again ($attempt/$attempts)" >&2 + echo "$asset of $tag is not what was uploaded yet ($attempt/$attempts)" >&2 done -echo "error: could not publish checksums for: $*" >&2 +echo "error: $asset of $tag does not match what this job uploaded: another writer, or the upload failed" >&2 exit 1 diff --git a/scripts/test_release_checksums.sh b/scripts/test_release_checksums.sh index 6f0a0819..10c16a9e 100755 --- a/scripts/test_release_checksums.sh +++ b/scripts/test_release_checksums.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # # Tests scripts/release-checksums.sh against a fake `gh` that keeps the release's -# SHA256SUMS in a temporary directory. Runs anywhere (no GitHub access): +# assets in a temporary directory. Runs anywhere (no GitHub access): # bash scripts/test_release_checksums.sh set -uo pipefail @@ -20,14 +20,16 @@ assert_eq() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1"; printf ' got:\ # Supports exactly what the script uses: # gh release download -p -O --clobber # gh release upload ... --clobber -# STUB_RELEASE directory holding the release's SHA256SUMS -# STUB_OVERWRITE if set to a file, the first upload is immediately replaced by -# that file's content, as if another job uploaded at that moment +# STUB_RELEASE directory holding the release's assets +# STUB_LOG every call is appended here +# STUB_OVERWRITE if set to a file, each upload is immediately replaced by that +# file's content, as if somebody else wrote the asset # STUB_FAIL_UPLOAD make every upload fail mkdir -p "$tmp/bin" cat > "$tmp/bin/gh" <<'EOF' #!/usr/bin/env bash set -eu +echo "gh $*" >> "$STUB_LOG" [ "$1" = release ] || { echo "fake gh: unsupported: $*" >&2; exit 2; } case "$2" in download) @@ -39,108 +41,128 @@ download) upload) [ -z "${STUB_FAIL_UPLOAD:-}" ] || { echo "upload failed" >&2; exit 1; } shift 3 - for f in "$@"; do [ "$f" = --clobber ] || cp "$f" "$STUB_RELEASE/$(basename "$f")"; done - if [ -n "${STUB_OVERWRITE:-}" ] && [ ! -f "$STUB_RELEASE/.overwritten" ]; then - cp "$STUB_OVERWRITE" "$STUB_RELEASE/SHA256SUMS" - touch "$STUB_RELEASE/.overwritten" - fi + first="" + for f in "$@"; do + [ "$f" = --clobber ] && continue + cp "$f" "$STUB_RELEASE/$(basename "$f")" + [ -n "$first" ] || first="$(basename "$f")" + done + [ -z "${STUB_OVERWRITE:-}" ] || cp "$STUB_OVERWRITE" "$STUB_RELEASE/$first" ;; *) echo "fake gh: unsupported: $*" >&2; exit 2 ;; esac EOF chmod +x "$tmp/bin/gh" -export PATH="$tmp/bin:$PATH" CHECKSUM_SETTLE_SECONDS=0 + +# A stand-in for cmd/releasesign: the "signature" is the file's SHA-256, so the +# test can see exactly which bytes were signed. Every signed file is kept. +cat > "$tmp/bin/fakesign" <<'SIGNER' +#!/usr/bin/env bash +set -eu +h() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi; } +case "$1" in +sign) h "$2" > "$3"; cat "$2" >> "$STUB_SIGNED"; echo "signed" >> "$STUB_LOG" ;; +verify) [ "$(h "$2")" = "$(cat "$3")" ] ;; +esac +SIGNER +chmod +x "$tmp/bin/fakesign" +export PATH="$tmp/bin:$PATH" CHECKSUM_SETTLE_SECONDS=0 RELEASE_SIGN="$tmp/bin/fakesign" if command -v sha256sum >/dev/null 2>&1; then sum() { sha256sum "$1" | awk '{print $1}'; }; else sum() { shasum -a 256 "$1" | awk '{print $1}'; }; fi # Assets of the two jobs, in their own dist directories. mkdir -p "$tmp/rel" "$tmp/distro" for f in fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz; do echo "content of $f" > "$tmp/rel/$f"; done -echo "content of the distro image" > "$tmp/distro/foxbyte-distro.tar.gz" - -publish_release() { (cd "$tmp/rel" && bash "$script" v9.9.9 fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz) >/dev/null; } -publish_distro() { (cd "$tmp/distro" && bash "$script" v9.9.9 foxbyte-distro.tar.gz) >/dev/null; } -names() { awk '{print $2}' "$STUB_RELEASE/SHA256SUMS" | tr '\n' ' ' | sed 's/ $//'; } -new_release() { export STUB_RELEASE="$tmp/release-$1"; rm -rf "$STUB_RELEASE"; mkdir -p "$STUB_RELEASE"; unset STUB_OVERWRITE STUB_FAIL_UPLOAD; } +echo "content of the distro image" > "$tmp/distro/foxbyte-distro.tar.zst" + +publish_release() { (cd "$tmp/rel" && bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz) >/dev/null; } +publish_distro() { (cd "$tmp/distro" && bash "$script" v9.9.9 SHA256SUMS.distro foxbyte-distro.tar.zst) >/dev/null; } +names() { awk '{print $2}' "$STUB_RELEASE/$1" | tr '\n' ' ' | sed 's/ $//'; } +new_release() { + export STUB_RELEASE="$tmp/release-$1" STUB_LOG="$tmp/log-$1" STUB_SIGNED="$tmp/signed-$1" + rm -rf "$STUB_RELEASE" "$STUB_LOG" "$STUB_SIGNED" + mkdir -p "$STUB_RELEASE" + : > "$STUB_LOG" + : > "$STUB_SIGNED" + unset STUB_OVERWRITE STUB_FAIL_UPLOAD +} +sig_matches() { [ -f "$STUB_RELEASE/$1.sig" ] && [ "$(sum "$STUB_RELEASE/$1")" = "$(cat "$STUB_RELEASE/$1.sig")" ] && echo yes || echo no; } + +evil="$(printf 'e%.0s' $(seq 1 64))" echo "### release-checksums.sh" new_release first publish_release -assert_eq "a release without SHA256SUMS gets this job's entries" "$(names)" \ +assert_eq "the release job's file lists exactly what it built" "$(names SHA256SUMS)" \ "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz" +assert_eq "…and it is signed" "$(sig_matches SHA256SUMS)" "yes" publish_distro -assert_eq "the second job adds its entry and keeps the first job's" "$(names)" \ - "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-distro.tar.gz foxbyte-docker-context.tar.gz" -release_then_distro="$(cat "$STUB_RELEASE/SHA256SUMS")" +assert_eq "the distro job writes its own file" "$(names SHA256SUMS.distro)" "foxbyte-distro.tar.zst" +assert_eq "…signed on its own" "$(sig_matches SHA256SUMS.distro)" "yes" +assert_eq "…and leaves SHA256SUMS and its signature alone" "$(names SHA256SUMS)|$(sig_matches SHA256SUMS)" \ + "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz|yes" +release_then_distro="$(cat "$STUB_RELEASE/SHA256SUMS" "$STUB_RELEASE/SHA256SUMS.distro")" new_release reversed publish_distro publish_release -assert_eq "the result doesn't depend on which job finishes first" "$(cat "$STUB_RELEASE/SHA256SUMS")" "$release_then_distro" +assert_eq "the result doesn't depend on which job finishes first" \ + "$(cat "$STUB_RELEASE/SHA256SUMS" "$STUB_RELEASE/SHA256SUMS.distro")" "$release_then_distro" -echo "changed binary" > "$tmp/rel/fox-linux-amd64" +# The finding itself: whatever is already on the release must never be signed. +new_release oracle +printf '%s %s\n' "$evil" "fox-backdoor" > "$STUB_RELEASE/SHA256SUMS" +printf '%s %s\n' "$evil" "fox-linux-amd64" >> "$STUB_RELEASE/SHA256SUMS" publish_release -assert_eq "a re-run replaces its own entry without duplicating it" \ - "$(grep -c ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS")|$(grep ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS" | awk '{print $1}')|$(wc -l < "$STUB_RELEASE/SHA256SUMS" | tr -d ' ')" \ - "1|$(sum "$tmp/rel/fox-linux-amd64")|4" -assert_eq "…and leaves the other job's entry untouched" \ - "$(grep ' foxbyte-distro.tar.gz$' "$STUB_RELEASE/SHA256SUMS" | awk '{print $1}')" "$(sum "$tmp/distro/foxbyte-distro.tar.gz")" - -new_release exact -printf '%s %s\n' "$(printf 'a%.0s' $(seq 1 64))" "fox-linux-amd64.sig" > "$STUB_RELEASE/SHA256SUMS" +assert_eq "a line planted on the release is not signed" "$(grep -c "$evil" "$STUB_SIGNED")" "0" +assert_eq "…nor published" "$(grep -c "$evil" "$STUB_RELEASE/SHA256SUMS")" "0" +assert_eq "…and the planted file is replaced by the signed one" "$(names SHA256SUMS)|$(sig_matches SHA256SUMS)" \ + "fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz|yes" +assert_eq "signing happens before anything is read from the release" \ + "$(grep -n 'download\|signed' "$STUB_LOG" | head -1 | sed 's/^[0-9]*://')" "signed" + +new_release rerun publish_release -assert_eq "entries are matched by exact name (fox-linux-amd64.sig survives)" "$(names)" \ - "fox-linux-amd64 fox-linux-amd64.sig fox-verify-linux-amd64 foxbyte-docker-context.tar.gz" - -new_release race -printf '%s %s\n' "$(printf 'b%.0s' $(seq 1 64))" "foxbyte-distro.tar.gz" > "$tmp/other-job.txt" -export STUB_OVERWRITE="$tmp/other-job.txt" -out="$( (cd "$tmp/rel" && bash "$script" v9.9.9 fox-linux-amd64 fox-verify-linux-amd64 foxbyte-docker-context.tar.gz) 2>&1 >/dev/null)" -assert_eq "another job uploading at the same moment is detected and merged again" \ - "$(echo "$out" | grep -c 'merging again')|$(names)" \ - "1|fox-linux-amd64 fox-verify-linux-amd64 foxbyte-distro.tar.gz foxbyte-docker-context.tar.gz" +echo "changed binary" > "$tmp/rel/fox-linux-amd64" +publish_release +assert_eq "a re-run replaces its own entries" \ + "$(grep -c ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS")|$(grep ' fox-linux-amd64$' "$STUB_RELEASE/SHA256SUMS" | awk '{print $1}')|$(sig_matches SHA256SUMS)" \ + "1|$(sum "$tmp/rel/fox-linux-amd64")|yes" + +new_release overwritten +printf '%s %s\n' "$evil" "fox-linux-amd64" > "$tmp/other-writer.txt" +export STUB_OVERWRITE="$tmp/other-writer.txt" +(cd "$tmp/rel" && CHECKSUM_ATTEMPTS=2 bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64) >/dev/null 2>&1 +assert_eq "somebody else's file in its place fails the job" "$?" "1" +assert_eq "…and is still not signed" "$(grep -c "$evil" "$STUB_SIGNED")" "0" +unset STUB_OVERWRITE new_release failing export STUB_FAIL_UPLOAD=1 -(cd "$tmp/rel" && bash "$script" v9.9.9 fox-linux-amd64) >/dev/null 2>&1 +(cd "$tmp/rel" && bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64) >/dev/null 2>&1 assert_eq "a failed upload fails the job" "$?" "1" unset STUB_FAIL_UPLOAD -assert_eq "missing arguments are a usage error" "$(bash "$script" v9.9.9 >/dev/null 2>&1; echo $?)" "2" +new_release nofile +(cd "$tmp/rel" && bash "$script" v9.9.9 SHA256SUMS no-such-file) >/dev/null 2>&1 +assert_eq "a file this job did not build fails the job" "$?:$(ls "$STUB_RELEASE" | wc -l | tr -d ' ')" "1:0" + +assert_eq "missing arguments are a usage error" "$(bash "$script" v9.9.9 SHA256SUMS >/dev/null 2>&1; echo $?)" "2" +assert_eq "no signer is an error, not an unsigned release" \ + "$(cd "$tmp/rel" && RELEASE_SIGN='' bash "$script" v9.9.9 SHA256SUMS fox-linux-amd64 >/dev/null 2>&1; echo $?)" "2" echo -# --- signing --------------------------------------------------------------- -# A stand-in for cmd/releasesign: the "signature" is the file's SHA-256, so the -# test can see whether the published signature belongs to the published file. -cat > "$tmp/bin/fakesign" <<'SIGNER' -#!/usr/bin/env bash -set -eu -h() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi; } -case "$1" in -sign) h "$2" > "$3" ;; -verify) [ "$(h "$2")" = "$(cat "$3")" ] ;; -esac -SIGNER -chmod +x "$tmp/bin/fakesign" -sig_matches() { [ -f "$STUB_RELEASE/SHA256SUMS.sig" ] && [ "$(sum "$STUB_RELEASE/SHA256SUMS")" = "$(cat "$STUB_RELEASE/SHA256SUMS.sig")" ] && echo yes || echo no; } - -new_release signed -RELEASE_SIGN="$tmp/bin/fakesign" publish_release -RELEASE_SIGN="$tmp/bin/fakesign" publish_distro -assert_eq "signed: the published signature belongs to the final, merged SHA256SUMS" "$(sig_matches)" "yes" - -# Another job's upload lands between this job's file and its signature, leaving -# a signature for a file that is no longer there: merged and signed again. -new_release signed-race -RELEASE_SIGN="$tmp/bin/fakesign" publish_distro -printf '%s %s\n' "$(sum "$tmp/distro/foxbyte-distro.tar.gz")" foxbyte-distro.tar.gz > "$tmp/theirs" -export STUB_OVERWRITE="$tmp/theirs" -RELEASE_SIGN="$tmp/bin/fakesign" publish_release -unset STUB_OVERWRITE -assert_eq "signed: a signature left stale by another upload is redone" "$(sig_matches)|$(names)" \ - "yes|fox-linux-amd64 fox-verify-linux-amd64 foxbyte-distro.tar.gz foxbyte-docker-context.tar.gz" +# The workflows must stay on this path: nothing may read a release's checksums +# back in order to rewrite them. +wf="$here/../.github/workflows" +assert_eq "no workflow reads SHA256SUMS back from a release" \ + "$(grep -lE 'gh release download.*SHA256SUMS' "$wf"/*.yml | wc -l | tr -d ' ')" "0" +assert_eq "only release.yml publishes SHA256SUMS" \ + "$(grep -lE 'release-checksums\.sh "\$TAG" SHA256SUMS |gh release upload.* SHA256SUMS' "$wf"/*.yml | xargs -n1 basename | tr '\n' ' ' | sed 's/ $//')" "release.yml" +assert_eq "wsl-distro.yml publishes its own file" \ + "$(grep -c 'release-checksums\.sh "\$TAG" SHA256SUMS\.distro ' "$wf/wsl-distro.yml")" "1" echo "==== ${PASS} passed, ${FAIL} failed ====" [ "$FAIL" -eq 0 ] diff --git a/web/src/pages/Dashboard.tsx b/web/src/pages/Dashboard.tsx index bdd19c77..699064eb 100644 --- a/web/src/pages/Dashboard.tsx +++ b/web/src/pages/Dashboard.tsx @@ -1,10 +1,11 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { getStatus, getBranches, createBranch, deleteBranch, suspendBranch, resumeBranch, listBackups, API, type Status, type Branch, type Backup, } from '../api' import { useConfirm } from '../confirm' import { useFeatures } from '../features' +import { usePolling } from '../usePolling' function Dot({ up }: { up: boolean }) { return @@ -131,19 +132,19 @@ export default function Dashboard() { const [err, setErr] = useState('') const [offline, setOffline] = useState(false) + // The newest refresh wins: a poll already on its way when the user suspends a + // branch can come back after the refresh that follows, carrying the old list. + const latest = useRef(0) const refresh = async () => { + const mine = ++latest.current try { const [s, b] = await Promise.all([getStatus(), getBranches()]) - setStatus(s); setBranches(b); setOffline(false) + if (mine === latest.current) { setStatus(s); setBranches(b); setOffline(false) } } catch { - setOffline(true) + if (mine === latest.current) setOffline(true) } } - useEffect(() => { - refresh() - const t = setInterval(refresh, 3000) - return () => clearInterval(t) - }, []) + usePolling(refresh, 10000) const act = async (fn: () => Promise) => { setErr('') @@ -182,7 +183,7 @@ export default function Dashboard() { return (

Dashboard

-

Live control plane · auto-refreshing every 3s.

+

Live control plane · auto-refreshing every 10s.

{tiles.map(([k, v], i) => ( diff --git a/web/src/pages/Import.tsx b/web/src/pages/Import.tsx index ecf33109..7b65536e 100644 --- a/web/src/pages/Import.tsx +++ b/web/src/pages/Import.tsx @@ -5,6 +5,7 @@ import { type ImportEvent, type ImportResult, type Replication, } from '../api' import { useConfirm } from '../confirm' +import { usePolling } from '../usePolling' type Line = { text: string; kind?: 'err' | 'ok' } type Prog = { done: number; total: number; label: string } @@ -186,14 +187,10 @@ function ContinuousImports() { const [done, setDone] = useState('') const [busy, setBusy] = useState('') - useEffect(() => { - let live = true - const load = () => listReplication().then(l => { if (live) { setList(l); setErr('') } }) - .catch(e => { if (live) setErr((e as Error).message) }) - load() - const t = setInterval(load, 4000) - return () => { live = false; clearInterval(t) } - }, []) + const live = useRef(true) + useEffect(() => { live.current = true; return () => { live.current = false } }, []) + usePolling(() => listReplication().then(l => { if (live.current) { setList(l); setErr('') } }) + .catch(e => { if (live.current) setErr((e as Error).message) }), 10000) const cutover = async (r: Replication) => { const ok = await confirm({ diff --git a/web/src/usePolling.ts b/web/src/usePolling.ts new file mode 100644 index 00000000..13f51175 --- /dev/null +++ b/web/src/usePolling.ts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import { useEffect, useRef } from 'react' + +// Run fn now and then every ms, for a page showing something that changes. +// +// setInterval does not know whether the last request came back: with a slow +// server the requests stack up, which makes the server slower, and every open +// tab did this whether anyone was looking at it or not. So the next poll is +// timed from the end of the last one, and a hidden tab does not poll at all and +// catches up the moment it is shown again. +// +// fn reports its own errors; one that throws anyway does not stop the polling. +export function usePolling(fn: () => unknown, ms: number) { + const latest = useRef(fn) + latest.current = fn + + useEffect(() => { + let stopped = false + let running = false + let timer: ReturnType | undefined + + const tick = async () => { + if (stopped || running || document.hidden) return + running = true + try { + await latest.current() + } catch { + // fn's to report + } + running = false + if (!stopped && !document.hidden) timer = setTimeout(tick, ms) + } + const onVisibility = () => { + if (document.hidden) return + clearTimeout(timer) + tick() + } + + document.addEventListener('visibilitychange', onVisibility) + tick() + return () => { + stopped = true + clearTimeout(timer) + document.removeEventListener('visibilitychange', onVisibility) + } + }, [ms]) +}