Narya Ed25519 integration + FD/Solana Conformance Suite refresh

Makefile

@@ -8,7 +8,7 @@ LDFLAGS := -X github.com/Overclock-Validator/mithril/pkg/version.Version=$(VERSI
            -X github.com/Overclock-Validator/mithril/pkg/version.GitBranch=$(GIT_BRANCH) \
            -X github.com/Overclock-Validator/mithril/pkg/version.BuildDate=$(BUILD_DATE)
 
-.PHONY: build release clean server-setup disk-setup tune test-conformance-elf test-conformance-vm-programs test-conformance-sbpf
+.PHONY: build release clean server-setup disk-setup tune conformance-vectors test-conformance-elf test-conformance-vm-programs test-conformance-sbpf test-conformance-precompiles
 
 build:
 	go build -ldflags "$(LDFLAGS)" -o mithril ./cmd/mithril
@@ -29,6 +29,24 @@ disk-setup:
 tune:
 	./scripts/performance-tune.sh $(ARGS)
 
+# Firedancer's fixture corpus is ~7 GB and gitignored, so it is fetched rather
+# than vendored. The revision is pinned: the corpus moves, and both its schema
+# and its per-suite result counts move with it, so tracking main would make a
+# passing run unreproducible and a regression indistinguishable from an upstream
+# edit. Bump this deliberately and re-record the counts when you do.
+CONFORMANCE_VECTORS_REV ?= a87fc430
+conformance-vectors:
+	@if [ ! -d conformance/test-vectors/.git ]; then \
+		git clone --filter=blob:none --no-checkout \
+			https://github.com/firedancer-io/test-vectors.git conformance/test-vectors; \
+	fi
+	@git -C conformance/test-vectors fetch --depth 1 origin $(CONFORMANCE_VECTORS_REV)
+	@git -C conformance/test-vectors checkout --force --detach $(CONFORMANCE_VECTORS_REV)
+	@echo "conformance corpus pinned at $(CONFORMANCE_VECTORS_REV)"
+
+test-conformance-precompiles:
+	go test ./conformance/ -run 'TestConformance_Precompile_' -timeout 90m -v
+
 test-conformance-elf:
 	go test ./conformance/ -run TestConformance_ElfLoader_Firedancer -v
 

cmd/mithril/configcmd/configcmd.go

@@ -260,6 +260,7 @@ max_rps = 8               # Verifier's own RPC budget (never shares the block-fe
 # ── Replay tuning ────────────────────────────────────────────────────────
 [tuning]
 txpar = 24                # Validator auto-defaults to 2x CPU cores only when unset; explicit 0 = sequential
+sigverify_backend = "auto" # auto|r51|generic|stdlib; stdlib uses Go's crypto/ed25519 impl after strict checks.
 
 # ── Mithril's RPC server ─────────────────────────────────────────────────
 [rpc]

cmd/mithril/node/node.go

@@ -44,6 +44,7 @@ import (
 	"github.com/Overclock-Validator/mithril/pkg/rpcserver"
 	"github.com/Overclock-Validator/mithril/pkg/sbpf"
 	"github.com/Overclock-Validator/mithril/pkg/sealevel"
+	"github.com/Overclock-Validator/mithril/pkg/sigverify"
 	"github.com/Overclock-Validator/mithril/pkg/snapshot"
 	"github.com/Overclock-Validator/mithril/pkg/snapshotdl"
 	"github.com/Overclock-Validator/mithril/pkg/state"
@@ -124,6 +125,10 @@ var (
 	debugDumpEpochVotingRewardDiff bool
 	cpuprofPath                    string
 
+	// resolvedSigverifyBackend is what the backend selector actually chose. It
+	// differs from the configured value under "auto", which is exactly when an
+	// operator needs to be told, so it is reported at startup.
+	resolvedSigverifyBackend string
 	paramArenaSizeMB         uint64
 	borrowedAccountArenaSize uint64
 
@@ -417,6 +422,8 @@ func init() {
 	Run.Flags().IntVar(&snapshot.SnapshotIndexEntryCommitterWorkers, "snapshot-index-committer-workers", snapshot.DefaultSnapshotIndexEntryCommitterWorkers, "Snapshot bootstrap account-index shard enqueue workers")
 	Run.Flags().IntVar(&snapshot.SnapshotIndexShards, "snapshot-index-shards", snapshot.DefaultSnapshotIndexShards, "Snapshot bootstrap account-index shard count")
 	Run.Flags().StringVar(&snapshot.SnapshotIndexTempDir, "snapshot-index-temp-dir", "", "Optional directory for snapshot index shard logs/SST staging")
+	Run.Flags().StringVar(&sigverify.Cfg.Backend, "sigverify-backend", sigverify.Defaults().Backend,
+		"ed25519 verification backend: auto|r51|generic|stdlib")
 	Run.Flags().BoolVar(&sbpf.UsePool, "use-pool", true, "Disable to allocate fresh slices")
 	Run.Flags().IntVar(&accountsdb.StoreAccountsWorkers, "store-accounts-workers", 128, "Number of workers to write account updates")
 	Run.Flags().IntVar(&accountsdb.ProgramCacheMaxMB, "program-cache-max-mb", accountsdb.DefaultProgramCacheMaxMB, "Maximum approximate SBPF program cache size in MiB")
@@ -974,6 +981,16 @@ func initConfigAndBindFlags(cmd *cobra.Command) error {
 	if snapshot.SnapshotIndexShards <= 0 || snapshot.SnapshotIndexShards > 1000 {
 		return fmt.Errorf("tuning.snapshot_index_shards must be between 1 and 1000")
 	}
+	// Resolve and install the signature-verification backend here rather than
+	// later: narya pins its backend on first use, and selecting it explicitly
+	// doubles as a startup health check, so a machine that cannot run the
+	// requested backend fails now instead of at the first block.
+	sigverify.Cfg.Backend = getString("sigverify-backend", "tuning.sigverify_backend")
+	resolved, err := sigverify.Configure(sigverify.Cfg)
+	if err != nil {
+		return fmt.Errorf("tuning.sigverify_backend: %w", err)
+	}
+	resolvedSigverifyBackend = resolved
 	sbpf.UsePool = getBool("use-pool", "tuning.use_pool")
 	accountsdb.StoreAccountsWorkers = getInt("store-accounts-workers", "tuning.store_accounts_workers")
 	accountsdb.ProgramCacheMaxMB = getInt("program-cache-max-mb", "tuning.program_cache_max_mb")
@@ -2632,6 +2649,8 @@ postBootstrap:
 		}()
 	}
 
+	startSigverifyReporter(ctx)
+
 	turbineAlpenglowAddr := ""
 	if alpenglowMode {
 		turbineAlpenglowAddr = alpenglowAddrForGossip(alpenglowObserverBindAddr)
@@ -2931,6 +2950,18 @@ func printStartupInfo(commandName string) {
 		fmt.Printf("  Bootstrap:    %s%s%s\n", green, bootstrapMode, reset)
 	}
 
+	if resolvedSigverifyBackend != "" {
+		sigverifyDesc := "AVX-512 accelerated"
+		switch resolvedSigverifyBackend {
+		case sigverify.BackendGeneric:
+			sigverifyDesc = "portable; no AVX512-IFMA on this CPU"
+		case sigverify.BackendStdlib:
+			sigverifyDesc = "Go's crypto/ed25519, but also uses Narya's strictness checks"
+		}
+		fmt.Printf("  Sigverify:    %s%s%s %s(%s)%s\n",
+			green, resolvedSigverifyBackend, reset, dim, sigverifyDesc, reset)
+	}
+
 	// Load state file for detailed info (only show for modes that use existing AccountsDB)
 	// In snapshot/new-snapshot modes, we're rebuilding so state info is not relevant
 	willUseExistingAccountsDB := bootstrapMode == "auto" || bootstrapMode == "accountsdb"

cmd/mithril/node/sigverify_reporter.go

@@ -0,0 +1,104 @@
+package node
+
+import (
+	"context"
+	"time"
+
+	"github.com/Overclock-Validator/mithril/pkg/mlog"
+	"github.com/Overclock-Validator/mithril/pkg/sigverify"
+)
+
+// sigverifyReportInterval is deliberately coarse. Batch width is a property of
+// the workload's shape, not of any one block, so a short interval would report
+// noise; the question this answers is "are we filling groups over a stretch of
+// replay", and that changes on the scale of minutes.
+const sigverifyReportInterval = 5 * time.Minute
+
+// startSigverifyReporter periodically records verification behaviour to the
+// per-run log directory.
+//
+// It writes only to <run-dir>/sigverify.log via NamedFilef, never to the
+// terminal. Operator-facing stderr already carries replay progress, and batch
+// width is diagnostic rather than something to watch live; adding it to the
+// console would cost attention from the output that matters and change the
+// startup display that people read at a glance.
+//
+// Three things go in, and each answers a question the others cannot:
+//
+//   - the resolved backend, because with backend=auto the name is the only way
+//     to learn whether this machine got the accelerated path;
+//   - InternalFaultFallbacks, which should be zero forever -- a nonzero value
+//     is a bug in the accelerated backend rather than an input-dependent
+//     condition, so it is worth alerting on rather than merely recording;
+//   - the batch-width distribution, because a signature total cannot show
+//     whether the drain policy is filling groups, and filling them is worth a
+//     ~3.7x factor per signature.
+func startSigverifyReporter(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(sigverifyReportInterval)
+		defer ticker.Stop()
+
+		// Baseline at startup so the first interval has something to difference
+		// against, and so the resolved backend is recorded even on a node that
+		// exits before the first tick.
+		previous := sigverify.Stats()
+		mlog.NamedFilef("sigverify", "startup: %s", previous)
+
+		for {
+			select {
+			case <-ctx.Done():
+				// A final snapshot: a short run would otherwise leave no record
+				// of what it did, and a run that ends because of a verification
+				// problem is exactly when this is worth having.
+				mlog.NamedFilef("sigverify", "shutdown: %s", sigverify.Stats())
+				return
+
+			case <-ticker.C:
+				current := sigverify.Stats()
+
+				// Counters are monotonic for the process lifetime, so an
+				// interval line needs the delta. Reporting only cumulative
+				// values would let a long-healthy run hide a recent collapse in
+				// batch width.
+				interval := sigverify.Snapshot{
+					Backend:                current.Backend,
+					InternalFaultFallbacks: current.InternalFaultFallbacks,
+					Batches:                current.Batches - previous.Batches,
+					Signatures:             current.Signatures - previous.Signatures,
+					EmptyBatches:           current.EmptyBatches - previous.EmptyBatches,
+					Widths:                 diffWidths(previous.Widths, current.Widths),
+				}
+				mlog.NamedFilef("sigverify", "interval: %s", interval)
+
+				if current.InternalFaultFallbacks > previous.InternalFaultFallbacks {
+					// This one does reach the operator log, because it means the
+					// accelerated backend produced a result it could not trust
+					// and recomputed on the portable path. It should never
+					// happen; if it does, silence is the wrong default.
+					mlog.Log.Warnf("sigverify: accelerated backend fell back on an internal fault %d time(s) total; this is a backend bug, not an input condition",
+						current.InternalFaultFallbacks)
+				}
+
+				previous = current
+			}
+		}
+	}()
+}
+
+// diffWidths subtracts an earlier histogram from a later one. Buckets are
+// matched by upper bound rather than by position, because Stats omits empty
+// buckets and the two snapshots need not have the same shape.
+func diffWidths(before, after []sigverify.WidthBucket) []sigverify.WidthBucket {
+	earlier := make(map[int]uint64, len(before))
+	for _, b := range before {
+		earlier[b.Upper] = b.Batches
+	}
+
+	delta := make([]sigverify.WidthBucket, 0, len(after))
+	for _, b := range after {
+		if n := b.Batches - earlier[b.Upper]; n > 0 {
+			delta = append(delta, sigverify.WidthBucket{Upper: b.Upper, Batches: n})
+		}
+	}
+	return delta
+}

cmd/mithril/node/sigverify_reporter_test.go

@@ -0,0 +1,45 @@
+package node
+
+import (
+	"testing"
+
+	"github.com/Overclock-Validator/mithril/pkg/sigverify"
+	"github.com/stretchr/testify/assert"
+)
+
+// Interval lines are differences between monotonic snapshots. Getting this
+// wrong is not loud: the reporter keeps writing plausible-looking lines, and a
+// collapse in batch width during one interval stays hidden behind a healthy
+// cumulative total.
+func TestDiffWidthsSubtractsByBucketNotPosition(t *testing.T) {
+	// Stats omits empty buckets, so the two snapshots need not share a shape.
+	// Matching by position would misattribute counts across bucket boundaries.
+	before := []sigverify.WidthBucket{
+		{Upper: 1, Batches: 10},
+		{Upper: 8, Batches: 4},
+	}
+	after := []sigverify.WidthBucket{
+		{Upper: 1, Batches: 12},
+		{Upper: 4, Batches: 7}, // a bucket absent from `before` entirely
+		{Upper: 8, Batches: 9},
+	}
+
+	assert.ElementsMatch(t, []sigverify.WidthBucket{
+		{Upper: 1, Batches: 2},
+		{Upper: 4, Batches: 7},
+		{Upper: 8, Batches: 5},
+	}, diffWidths(before, after))
+}
+
+// A bucket that saw no traffic during the interval must be dropped rather than
+// reported as zero, or every line carries every bucket forever and the shape of
+// the distribution stops being readable at a glance.
+func TestDiffWidthsOmitsUnchangedBuckets(t *testing.T) {
+	same := []sigverify.WidthBucket{{Upper: 8, Batches: 5}}
+	assert.Empty(t, diffWidths(same, same))
+}
+
+func TestDiffWidthsHandlesAnEmptyBaseline(t *testing.T) {
+	after := []sigverify.WidthBucket{{Upper: 2, Batches: 3}}
+	assert.Equal(t, after, diffWidths(nil, after))
+}

config.example.toml

@@ -519,6 +519,14 @@ name = "mithril"
     # Number of borrowed accounts to preallocate in arena (0 to disable)
     borrowed_account_arena_size = 1024
 
+    # ed25519 signature verification backend.
+    #   auto    - use the AVX-512 accelerated backend when the CPU has
+    #             AVX512-IFMA (Zen 4/5, Ice Lake and newer), else portable
+    #   r51     - force the accelerated backend; startup fails without AVX512-IFMA
+    #   generic - force the portable pure-Go backend
+    #   stdlib  - use Go’s crypto/ed25519 implementation after the mandatory strict rejection checks.
+    sigverify_backend = "auto"
+
     # Enable/disable pool allocator for slices
     use_pool = true
 

conformance/bpf_loader_program_test.go

@@ -38,9 +38,7 @@ func bpfLoaderTestAccountStateChangesMatch(t *testing.T, execCtx *sealevel.Execu
 				if fixtureModifiedAcct.Executable != mithrilModifiedAcct.Executable {
 					return false
 				}
-				if fixtureModifiedAcct.RentEpoch != mithrilModifiedAcct.RentEpoch {
-					return false
-				}
+				// AcctState dropped rent_epoch in protosol v5.4.0.
 				if solana.PublicKeyFromBytes(fixtureModifiedAcct.Owner[:]) != solana.PublicKeyFromBytes(mithrilModifiedAcct.Owner[:]) {
 					return false
 				}