diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index e18b7fa9aa..7d48399daa 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -15,7 +15,7 @@ jobs: benchmarks: name: "Run Full Go benchmarks on Main" runs-on: "depot-ubuntu-24.04-arm-8" # same as build-test.yaml - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: "actions/checkout@v7" with: diff --git a/internal/datastore/benchmark/driver_bench_test.go b/internal/datastore/benchmark/driver_bench_test.go index 0255c68fe4..8bead1aaa1 100644 --- a/internal/datastore/benchmark/driver_bench_test.go +++ b/internal/datastore/benchmark/driver_bench_test.go @@ -57,6 +57,10 @@ var sortOrders = map[string]options.SortOrder{ } func BenchmarkDatastoreDriver(b *testing.B) { + if testing.Short() { + b.Skip("skipping datastore driver benchmarks in -short mode") + } + for _, driver := range drivers { b.Run(driver.name+driver.suffix, func(b *testing.B) { b.StopTimer() diff --git a/internal/fdw/README.md b/internal/fdw/README.md index af2465bff7..28e07aa326 100644 --- a/internal/fdw/README.md +++ b/internal/fdw/README.md @@ -226,8 +226,8 @@ SPICEDB_SHUTDOWN_GRACE_PERIOD | `subject_type` | text | Subject type | | `subject_id` | text | Subject ID | | `optional_subject_relation` | text | Optional subject relation | -| `optional_caveat_name` | text | Optional caveat name | -| `optional_caveat_context` | jsonb | Optional caveat context | +| `caveat_name` | text | Optional caveat name | +| `caveat_context` | text | Optional caveat context | | `consistency` | text | Consistency token (ZedToken) | **Supported Operations:** diff --git a/internal/services/integrationtesting/benchmark_executors_test.go b/internal/services/integrationtesting/benchmark_executors_test.go index 1a6738c20d..9c9aa2cbf8 100644 --- a/internal/services/integrationtesting/benchmark_executors_test.go +++ b/internal/services/integrationtesting/benchmark_executors_test.go @@ -55,7 +55,7 @@ const classicDispatchDepthRemaining = 50 func BenchmarkExecutors(b *testing.B) { for _, engineID := range enginesToBenchmark { if testing.Short() && engineID != "memory" { - continue + b.Skip("skipping non-memory driver benchmarks in -short mode") } b.Run(engineID, func(b *testing.B) { for _, scenarioName := range executorScenarios { diff --git a/internal/services/integrationtesting/benchmark_test.go b/internal/services/integrationtesting/benchmark_test.go index 2ff1277619..7bc480a6a0 100644 --- a/internal/services/integrationtesting/benchmark_test.go +++ b/internal/services/integrationtesting/benchmark_test.go @@ -299,10 +299,8 @@ var allScenarios = []benchmarkScenario{ // Hierarchy: engine → scenario (test data) → operation → {dispatch, queryplan} func BenchmarkServices(b *testing.B) { for _, engineID := range enginesToBenchmark { - if testing.Short() { - if engineID != "memory" { - continue - } + if testing.Short() && engineID != "memory" { + b.Skip("skipping non-memory driver benchmarks in -short mode") } b.Run(engineID, func(b *testing.B) { for _, scenario := range allScenarios { diff --git a/magefiles/benchmark.go b/magefiles/benchmark.go index 4bb0c6664e..e224cd437f 100644 --- a/magefiles/benchmark.go +++ b/magefiles/benchmark.go @@ -3,6 +3,8 @@ package main import ( + "fmt" + "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" ) @@ -11,10 +13,29 @@ type Benchmark mg.Namespace // All Runs all benchmarks func (b Benchmark) All() error { - return sh.RunV("go", "test", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "5s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem") + if err := prebuildTestBinaries(); err != nil { + return err + } + fmt.Println("running all benchmarks") + // -p 1 runs one package's test binary at a time so no other package runs during a measurement window. + // -cpu 1 sets GOMAXPROCS=1 inside each benchmark, measuring single-threaded per-op latency + return sh.RunV("go", "test", "-p", "1", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "5s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem") } // Short runs the short benchmarks for individual PRs on CI -- less time, only the short ones func (b Benchmark) Short() error { - return sh.RunV("go", "test", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "2s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem", "-short") + if err := prebuildTestBinaries(); err != nil { + return err + } + fmt.Println("running short benchmarks") + // -p 1 runs one package's test binary at a time so no other package runs during a measurement window. + // -cpu 1 sets GOMAXPROCS=1 inside each benchmark, measuring single-threaded per-op latency + return sh.RunV("go", "test", "-p", "1", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "2s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem", "-short") +} + +// prebuildTestBinaries compiles and links every test binary in parallel (-run=XXX runs nothing), +// with the same tags/ldflags as the benchmark runs so they hit the build cache. +func prebuildTestBinaries() error { + fmt.Println("prebuilding all test binaries in parallel") + return sh.RunV("go", "test", "./...", "-ldflags=-checklinkname=0", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX") }