ci: parallelize and shard the test suite, restore Go caches

.github/workflows/ci.yml

@@ -18,8 +18,83 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # The make test suite: every top-level test of every package runs in
+  # exactly one shard, packed by recorded duration (scripts/test_shards.py).
+  # macOS: four shards plus checks fit GitHub's five concurrent macOS jobs.
   test:
-    name: test (${{ matrix.platform }})
+    name: test (${{ matrix.platform }}, ${{ matrix.shard }}/${{ matrix.shards }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu-24.04, platform: linux-amd64, shards: 6, shard: 1}
+          - {os: ubuntu-24.04, platform: linux-amd64, shards: 6, shard: 2}
+          - {os: ubuntu-24.04, platform: linux-amd64, shards: 6, shard: 3}
+          - {os: ubuntu-24.04, platform: linux-amd64, shards: 6, shard: 4}
+          - {os: ubuntu-24.04, platform: linux-amd64, shards: 6, shard: 5}
+          - {os: ubuntu-24.04, platform: linux-amd64, shards: 6, shard: 6}
+          - {os: macos-14, platform: darwin-arm64, shards: 4, shard: 1}
+          - {os: macos-14, platform: darwin-arm64, shards: 4, shard: 2}
+          - {os: macos-14, platform: darwin-arm64, shards: 4, shard: 3}
+          - {os: macos-14, platform: darwin-arm64, shards: 4, shard: 4}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache locations
+        run: |
+          echo "GOSVM_CACHE=$RUNNER_TEMP/gosvm-cache" >> "$GITHUB_ENV"
+          echo "GOCACHE=$RUNNER_TEMP/go-build" >> "$GITHUB_ENV"
+          echo "GOMODCACHE=$RUNNER_TEMP/go-mod" >> "$GITHUB_ENV"
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.22.0'
+          cache: false
+
+      # Module and build caches under one key per commit's Go sources, so a
+      # run restores the latest earlier build of the same platform.
+      - name: Go caches
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ runner.temp }}/go-build
+            ${{ runner.temp }}/go-mod
+          key: go-${{ matrix.platform }}-${{ hashFiles('go.mod', '**/go.sum', '**/*.go') }}
+          restore-keys: go-${{ matrix.platform }}-
+
+      # The official archive is 430-530 MB; only the extracted Clang/LLD are
+      # cached, keyed by the files that pin them.
+      - name: Cache platform-tools backend
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/gosvm-cache/toolchains
+          key: platform-tools-v1.57-${{ matrix.platform }}-${{ hashFiles('internal/toolchain/archives.go', 'internal/targets/profile.go') }}
+
+      - name: Build CLI and install backend and runners
+        run: |
+          bash scripts/build-cli.sh build/gosvm
+          build/gosvm toolchain install -runners=false
+          # Runner releases exist only after the maintainer publishes them;
+          # until then the SBF executions in the tests skip, saying so.
+          if ! build/gosvm runner install; then
+            echo "::warning::SVM runners not installed for ${{ matrix.platform }}; SBF executions will be skipped"
+          fi
+
+      # -count=1: with the build cache restored, `go test` would otherwise
+      # report cached results instead of running the tests.
+      - name: Root tests
+        env:
+          GOFLAGS: -count=1
+        run: |
+          # The real-libraries tests read these modules from the module cache.
+          (cd examples/real-libraries && go mod download)
+          python3 scripts/test_shards.py --shard ${{ matrix.shard }}/${{ matrix.shards }}
+
+  # Everything else, alongside the test shards.
+  checks:
+    name: checks (${{ matrix.platform }})
     strategy:
       fail-fast: false
       matrix:
@@ -29,23 +104,31 @@ jobs:
           - os: macos-14
             platform: darwin-arm64
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 90
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
 
+      - name: Cache locations
+        run: |
+          echo "GOSVM_CACHE=$RUNNER_TEMP/gosvm-cache" >> "$GITHUB_ENV"
+          echo "GOCACHE=$RUNNER_TEMP/go-build" >> "$GITHUB_ENV"
+          echo "GOMODCACHE=$RUNNER_TEMP/go-mod" >> "$GITHUB_ENV"
+
       - uses: actions/setup-go@v5
         with:
           go-version: '1.22.0'
-          cache: true
-
-      - name: Isolated gosvm cache
-        run: echo "GOSVM_CACHE=$RUNNER_TEMP/gosvm-cache" >> "$GITHUB_ENV"
-
-      - name: Build CLI
-        run: bash scripts/build-cli.sh build/gosvm
+          cache: false
 
-      - name: Vet
-        run: go vet ./...
+      # Module and build caches under one key per commit's Go sources, so a
+      # run restores the latest earlier build of the same platform.
+      - name: Go caches
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ runner.temp }}/go-build
+            ${{ runner.temp }}/go-mod
+          key: go-${{ matrix.platform }}-${{ hashFiles('go.mod', '**/go.sum', '**/*.go') }}
+          restore-keys: go-${{ matrix.platform }}-
 
       # The official archive is 430-530 MB; only the extracted Clang/LLD are
       # cached, keyed by the files that pin them.
@@ -55,28 +138,25 @@ jobs:
           path: ${{ runner.temp }}/gosvm-cache/toolchains
           key: platform-tools-v1.57-${{ matrix.platform }}-${{ hashFiles('internal/toolchain/archives.go', 'internal/targets/profile.go') }}
 
-      - name: Install backend
-        run: build/gosvm toolchain install -runners=false
-
-      # Runner releases exist only after the maintainer publishes them (and
-      # Linux pins only after the first runner release); until then the SBF
-      # executions in the tests skip, saying so.
-      - name: Install SVM runners
+      - name: Build CLI and install backend and runners
         run: |
+          bash scripts/build-cli.sh build/gosvm
+          build/gosvm toolchain install -runners=false
+          # Runner releases exist only after the maintainer publishes them;
+          # until then the SBF executions in the tests skip, saying so.
           if ! build/gosvm runner install; then
             echo "::warning::SVM runners not installed for ${{ matrix.platform }}; SBF executions will be skipped"
           fi
 
+      - name: Vet
+        run: go vet ./...
+
       - name: Doctor
         run: build/gosvm doctor
 
-      - name: Root tests
-        run: |
-          # The real-libraries tests read these modules from the module cache.
-          (cd examples/real-libraries && go mod download)
-          make test
-
       - name: Example and benchmark module tests
+        env:
+          GOFLAGS: -count=1
         run: |
           flags=""
           if [ "$(uname -s)" = Darwin ]; then flags="-ldflags=-linkmode=external"; fi
@@ -143,6 +223,18 @@ jobs:
           bash -n scripts/install.sh scripts/build-cli.sh scripts/build_runner_linux.sh
           sh -n scripts/install-release.sh
       - name: Python syntax
-        run: python3 -m py_compile scripts/package_runner_release.py scripts/update_runner_pins.py scripts/package_cli_release.py scripts/check_rebuild_reference.py
+        run: python3 -m py_compile scripts/package_runner_release.py scripts/update_runner_pins.py scripts/package_cli_release.py scripts/check_rebuild_reference.py scripts/test_shards.py
       - name: Committed release pins are well formed
         run: python3 -c "import json,glob; [json.load(open(p)) for p in glob.glob('internal/runner/pins/**/*.json', recursive=True)]"
+
+  # The single required check for branch protection.
+  ci-ok:
+    name: ci-ok
+    if: always()
+    needs: [test, checks, scripts]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: All jobs passed
+        run: |
+          echo '${{ toJSON(needs) }}'
+          [ "${{ needs.test.result }}" = success ] && [ "${{ needs.checks.result }}" = success ] && [ "${{ needs.scripts.result }}" = success ]

internal/compiler/accounts64_sbf_test.go

@@ -221,6 +221,7 @@ func TestAccounts64SBF(t *testing.T) {
 // 3,584-byte table overflows 4 KiB in the tutorial (4,224 bytes) and escrow
 // (4,672); with -max-accounts 64 both build for both targets.
 func TestAccounts64Frames(t *testing.T) {
+	t.Parallel()
 	llvm := compiler.ModernLLVMForTest()
 	if llvm == "" {
 		t.Skip("pinned platform-tools v1.57 backend required")

internal/compiler/accounts_test.go

@@ -14,6 +14,7 @@ import (
 // directly. Duplicate descriptors must retain data/flags/key aliasing across
 // both the old boundary and the new final slot.
 func TestSDKAccountDecoderCapacityAndAliases(t *testing.T) {
+	t.Parallel()
 	source := `package app;import "gosvm/solana"
 func Process(c solana.Context)uint64 {
  d:=solana.Instruction(c)
@@ -144,6 +145,7 @@ while(scanf("%llu %u %u %u",&n,&pattern,&bad,&strict)==4){
 // Only programs that opt in to 64 accounts get the wide table and Process's
 // own frame; SDK 1 and other limits are rejected.
 func TestMaxAccountsOptIn(t *testing.T) {
+	t.Parallel()
 	source := `package app;import "gosvm/solana";func Process(c solana.Context)uint64{return solana.Count(c)}`
 	for _, tc := range []struct {
 		sdk, max   int

internal/compiler/alloc_growth_test.go

@@ -90,6 +90,7 @@ func hasPointers(t reflect.Type) bool {
 // growthTypes and append sequences of 1, 2, 3, 7 and 100 elements, until the
 // backing array exceeds 300 KiB.
 func TestGrowthModelMatchesGoRuntime(t *testing.T) {
+	t.Parallel()
 	if v := strings.TrimPrefix(runtime.Version(), "go"); !strings.HasPrefix(v, "1.22") {
 		t.Skipf("the oracle is Go 1.22's runtime; running %s", v)
 	}
@@ -127,6 +128,7 @@ func TestGrowthModelMatchesGoRuntime(t *testing.T) {
 // the host over a grid of lengths, capacities, counts, sizes and both
 // pointer modes, and compares every resulting capacity with the model.
 func TestGrowthRuntimeMatchesModel(t *testing.T) {
+	t.Parallel()
 	cc, err := exec.LookPath("clang")
 	if err != nil {
 		t.Skip("host Clang is needed")
@@ -202,6 +204,7 @@ while(scanf("%llu %llu %llu %llu %llu",&l,&c,&n,&s,&scan)==5){
 // TestGrowthModelMatchesCompiledAppend checks the model against append on
 // concrete compiled types (no reflect), which is what native fixtures run.
 func TestGrowthModelMatchesCompiledAppend(t *testing.T) {
+	t.Parallel()
 	type big struct {
 		P *uint64
 		A [64]uint64

internal/compiler/alloc_moves_test.go

@@ -10,6 +10,7 @@ import (
 // lifetime tables): each now compiles, moves the escaping local to the heap
 // and returns the value native Go would.
 func TestFormerEscapesMoveToHeap(t *testing.T) {
+	t.Parallel()
 	for name, c := range map[string]struct {
 		source string
 		want   string

internal/compiler/alloc_sbf_test.go

@@ -160,6 +160,7 @@ func allocLargeHeapCases() []allocCase {
 // runners. Skips without the v1.57 backend or a runner (make test).
 // GOSVM_ALLOC_RESULTS=<dir> keeps fixtures, ELFs, reports and a summary.
 func TestAllocationSBF(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("SBF runs are skipped in -short mode")
 	}

internal/compiler/alloc_test.go

@@ -71,6 +71,7 @@ func Process(s, i []byte) uint64 {
 `
 
 func TestAllocationAcceptedForms(t *testing.T) {
+	t.Parallel()
 	for name, body := range map[string]string{
 		"new scalar":           `p := new(uint64); *p = 1; s[0] = byte(*p)`,
 		"new struct":           `v := new(V); v.N = 2; s[0] = byte(v.N)`,
@@ -123,6 +124,7 @@ func TestAllocationAcceptedForms(t *testing.T) {
 }
 
 func TestAllocationRejections(t *testing.T) {
+	t.Parallel()
 	for name, c := range map[string]struct{ body, want string }{
 		"make channel":        {`ch := make(chan uint64); _ = ch`, "make supports only slices and maps"},
 		"new int":             {`p := new(int); _ = p`, ""},
@@ -179,6 +181,7 @@ func TestAllocationRejections(t *testing.T) {
 // Heap moves and parameter escapes are part of the checked program, so a
 // backend allocates exactly the same storage.
 func TestHeapMovesExported(t *testing.T) {
+	t.Parallel()
 	checked, err := Check(&Program{Entry: "program", Packages: []PackageSources{{"program", []Source{{Name: "alloc.go", Data: []byte(`package p
 type N struct{ Next *N; V uint64 }
 func keep(h *N, n *N) { h.Next = n }
@@ -229,6 +232,7 @@ func Process(s, i []byte) uint64 {
 // Programs that do not allocate keep their exact C; the runtime and the
 // configured budget appear only when used.
 func TestAllocationRuntimeOnlyWhenUsed(t *testing.T) {
+	t.Parallel()
 	without, err := Compile("a.go", []byte("package p\ntype B struct{ K [4]byte }\nfunc Process(s, i []byte) uint64 { var b B; v := b.K[:]; return uint64(len(v)) }\n"))
 	if err != nil {
 		t.Fatal(err)
@@ -339,6 +343,7 @@ func AllocVectorsForTest() [][]byte { return allocVectors(20261015, 3, 3000, 150
 // for 18 element kinds, make/new zero values and make traps, copy overlap,
 // slicing bounds, linked structures, escaping locals, literals and aliasing.
 func TestAllocationDifferential(t *testing.T) {
+	t.Parallel()
 	source, err := os.ReadFile("../../examples/alloc-values/program.go")
 	if err != nil {
 		t.Fatal(err)
@@ -411,6 +416,7 @@ func runOOM(t *testing.T, bin string, n uint64) string {
 // first allocation that does not fit aborts. Native Go never runs out of
 // memory at these sizes, so the expectations are computed, not observed.
 func TestOutOfMemoryBoundaries(t *testing.T) {
+	t.Parallel()
 	build := func(heap int, body string) string {
 		source := "package p\nfunc load(i []byte) uint64 { n := uint64(0); for j := uint64(0); j < 8; j++ { n = n | uint64(i[j])<<(8*j) }; return n }\nfunc save(s []byte, v uint64) { for j := uint64(0); j < 8; j++ { s[j] = byte(v >> (8 * j)) } }\nfunc Process(s, i []byte) uint64 {\nn := load(i)\n" + body + "\n}\n"
 		c, err := CompileProgram(&Program{Entry: "program", HeapBytes: heap, Packages: []PackageSources{{"program", []Source{{Name: "oom.go", Data: []byte(source)}}}}})

internal/compiler/arrays_test.go

@@ -43,6 +43,7 @@ while(scanf("%32s %48s",ih,sh)==2){
 }return 0;}`
 
 func TestScalarArraysDifferential(t *testing.T) {
+	t.Parallel()
 	source, err := os.ReadFile("../../examples/array-values/program.go")
 	if err != nil {
 		t.Fatal(err)
@@ -96,6 +97,7 @@ func TestScalarArraysDifferential(t *testing.T) {
 }
 
 func TestArrayDeclarationOrder(t *testing.T) {
+	t.Parallel()
 	sources := []Source{
 		{"a.go", []byte(`package p; type Box struct { K Key };func Process(s,i []byte)uint64 {b:=Box{K:Key{1}};return uint64(b.K[0])}`)},
 		{"b.go", []byte(`package p;type Key [32]byte;func echo(k [32]byte)[32]byte{return k}`)},
@@ -115,6 +117,7 @@ func TestArrayDeclarationOrder(t *testing.T) {
 }
 
 func TestArrayRejections(t *testing.T) {
+	t.Parallel()
 	for name, source := range map[string]string{
 		"oversize byte": `type K [1025]byte;func Process(s,i []byte)uint64{return 0}`,
 		"oversize word": `type K [129]uint64;func Process(s,i []byte)uint64{return 0}`,
@@ -139,6 +142,7 @@ func TestArrayRejections(t *testing.T) {
 }
 
 func TestArrayStorageLimit(t *testing.T) {
+	t.Parallel()
 	for _, typ := range []string{"[1024]byte", "[128]uint64", "[256]uint32", "[1024]bool", "[0]byte"} {
 		_, err := Compile("limit.go", []byte("package p;type K "+typ+`;func Process(s,i []byte)uint64{var a K;return uint64(len(a))}`))
 		if err != nil {
@@ -148,6 +152,7 @@ func TestArrayStorageLimit(t *testing.T) {
 }
 
 func TestArrayCapacityMemory(t *testing.T) {
+	t.Parallel()
 	source, err := os.ReadFile("../../examples/array-capacity/program.go")
 	if err != nil {
 		t.Fatal(err)

internal/compiler/bits_diff_test.go

@@ -238,6 +238,7 @@ func bitsEdges(width int) []uint64 {
 }
 
 func TestBitsDifferentialAgainstGo(t *testing.T) {
+	t.Parallel()
 	ops := bitsOps()
 	program := bitsDifferentialProgram(ops)
 	bin := hostProgram(t, program, bitsHostDriver)

internal/compiler/bits_sbf_test.go

@@ -298,6 +298,7 @@ func bitsProbeCases() (probe, cu []probeCase) {
 // backend or a runner. GOSVM_BITS_RESULTS=<dir> keeps the fixtures, ELFs,
 // runner reports and a summary per target.
 func TestBitsSBF(t *testing.T) {
+	t.Parallel()
 	llvm := modernLLVM()
 	if llvm == "" {
 		t.Skip("pinned platform-tools v1.57 backend required")