From eb8b21c5afa9e944f9bcd8c6efcc518e60999970 Mon Sep 17 00:00:00 2001 From: smcio Date: Fri, 24 Jul 2026 01:11:15 +0200 Subject: [PATCH 1/5] minimise duplication of message identity work --- pkg/block/block.go | 115 +++++++++++- pkg/block/message_identity.go | 36 ++++ .../message_identity_concurrency_test.go | 50 ++++++ pkg/block/message_identity_lookup_test.go | 41 +++++ pkg/block/message_identity_mutation_test.go | 37 ++++ pkg/block/message_identity_test.go | 163 ++++++++++++++++++ pkg/replay/block.go | 47 +++-- pkg/replay/transaction_status.go | 19 +- .../transaction_status_benchmark_test.go | 88 ++++++++++ pkg/replay/transaction_status_cache.go | 70 +++++--- .../transaction_status_plan_binding_test.go | 30 ++++ .../transaction_status_prepared_test.go | 129 ++++++++++++++ pkg/replay/transaction_status_test.go | 14 +- pkg/txstatus/agave_snapshot.go | 4 +- pkg/txstatus/message_identity.go | 48 ++++++ pkg/txstatus/message_identity_test.go | 70 ++++++++ 16 files changed, 881 insertions(+), 80 deletions(-) create mode 100644 pkg/block/message_identity.go create mode 100644 pkg/block/message_identity_concurrency_test.go create mode 100644 pkg/block/message_identity_lookup_test.go create mode 100644 pkg/block/message_identity_mutation_test.go create mode 100644 pkg/block/message_identity_test.go create mode 100644 pkg/replay/transaction_status_benchmark_test.go create mode 100644 pkg/replay/transaction_status_plan_binding_test.go create mode 100644 pkg/replay/transaction_status_prepared_test.go create mode 100644 pkg/txstatus/message_identity.go create mode 100644 pkg/txstatus/message_identity_test.go diff --git a/pkg/block/block.go b/pkg/block/block.go index 0abb835b..a09cc2c8 100644 --- a/pkg/block/block.go +++ b/pkg/block/block.go @@ -1,12 +1,15 @@ package block import ( + "fmt" + "sync" "time" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/lthash" "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/txstatus" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" ) @@ -22,6 +25,25 @@ type TurbineIngressTimings struct { ReplayAdmission time.Duration } +var transactionDerivedStateInitMu sync.Mutex + +// transactionDerivedState is shared by shallow in-memory Block copies and is +// never serialized. Its mutex makes repeated preparation safe when status-cache +// validation of the same immutable block overlaps. +type transactionDerivedState struct { + mu sync.Mutex + signaturesVerified bool + messageIdentities *PreparedTransactionMessageIdentities +} + +// PreparedTransactionMessageIdentities is an opaque, immutable set of message +// identities bound to one ordered transaction slice. +type PreparedTransactionMessageIdentities struct { + transactions []*solana.Transaction + versions []solana.MessageVersion + identities []txstatus.TransactionMessageIdentity +} + type Block struct { Slot uint64 ParentSlot uint64 @@ -69,10 +91,9 @@ type Block struct { FromLiveStream bool FromLocalProduction bool IsSkipped bool // True for slots that were skipped by the leader - // transactionSignaturesVerified is deliberately private: it is a trust - // marker set only by an in-process source after it has verified every - // transaction signature. It must not be accepted from JSON/file input. - transactionSignaturesVerified bool + // transactionDerivedState contains only trusted, nonserialized data derived + // from this exact in-memory block (and is shared by its shallow copies). + transactionDerivedState *transactionDerivedState // turbineReplayAdmissionStart is deliberately private and monotonic. It // exists only on the exact in-memory block handed from Turbine to replay, // so serialized/RPC blocks cannot inject or preserve an admission clock. @@ -101,16 +122,93 @@ type Block struct { // transaction data. Signature-preserving transformations such as resolving // address-table lookups remain safe. func (b *Block) MarkTransactionSignaturesVerified() { - if b != nil { - b.transactionSignaturesVerified = true + if b == nil { + return } + state := b.transactionState() + state.mu.Lock() + state.signaturesVerified = true + state.mu.Unlock() } // TransactionSignaturesVerified reports whether this exact in-memory block // crossed a trusted transaction-signature verification boundary. The marker is // intentionally not serialized; replay re-verifies after any serialization. func (b *Block) TransactionSignaturesVerified() bool { - return b != nil && b.transactionSignaturesVerified + if b == nil { + return false + } + state := b.transactionState() + state.mu.Lock() + defer state.mu.Unlock() + return state.signaturesVerified +} + +// PrepareTransactionMessageIdentities serializes and hashes every transaction +// message at most once for an immutable in-memory block. The returned opaque +// handle exposes identities only by value, so callers cannot mutate the cache. +// +// As with MarkTransactionSignaturesVerified, callers must not mutate signed +// message contents after preparation. Address-table resolution is safe because +// it does not change the canonical serialized message. +func (b *Block) PrepareTransactionMessageIdentities() (*PreparedTransactionMessageIdentities, error) { + if b == nil { + return nil, fmt.Errorf("nil block") + } + state := b.transactionState() + state.mu.Lock() + defer state.mu.Unlock() + if state.messageIdentities.matches(b.Transactions) { + return state.messageIdentities, nil + } + if state.messageIdentities != nil { + state.messageIdentities = nil + state.signaturesVerified = false + } + + prepared := &PreparedTransactionMessageIdentities{ + transactions: append([]*solana.Transaction(nil), b.Transactions...), + versions: make([]solana.MessageVersion, len(b.Transactions)), + identities: make([]txstatus.TransactionMessageIdentity, len(b.Transactions)), + } + for index, tx := range b.Transactions { + if tx == nil { + state.signaturesVerified = false + return nil, fmt.Errorf("transaction %d is nil", index) + } + identity, err := txstatus.IdentityForTransaction(tx) + if err != nil { + state.signaturesVerified = false + return nil, fmt.Errorf("transaction %d message identity: %w", index, err) + } + prepared.versions[index] = tx.Message.GetVersion() + prepared.identities[index] = identity + } + state.messageIdentities = prepared + return prepared, nil +} + +func (cache *PreparedTransactionMessageIdentities) matches(transactions []*solana.Transaction) bool { + if cache == nil || len(cache.transactions) != len(transactions) || + len(cache.versions) != len(transactions) || len(cache.identities) != len(transactions) { + return false + } + for index, tx := range transactions { + if tx == nil || cache.transactions[index] != tx || + cache.versions[index] != tx.Message.GetVersion() || + cache.identities[index].RecentBlockhash != tx.Message.RecentBlockhash { + return false + } + } + return true +} + +func (b *Block) invalidateTransactionDerivedState() { + state := b.transactionState() + state.mu.Lock() + state.messageIdentities = nil + state.signaturesVerified = false + state.mu.Unlock() } // MarkTurbineReplayAdmissionStart starts the interval from successful @@ -163,9 +261,10 @@ func (b *Block) CompleteTurbineReplayAdmission(at time.Time) (TurbineIngressTimi return b.turbineIngressTimings, true } func (b *Block) FixupTxVersions() { - if len(b.Versions) == 0 { + if b == nil || len(b.Versions) == 0 { return } + b.invalidateTransactionDerivedState() for idx, tx := range b.Transactions { tx.Message.SetVersion(solana.MessageVersion(b.Versions[idx])) } diff --git a/pkg/block/message_identity.go b/pkg/block/message_identity.go new file mode 100644 index 00000000..067ac4de --- /dev/null +++ b/pkg/block/message_identity.go @@ -0,0 +1,36 @@ +package block + +import "github.com/Overclock-Validator/mithril/pkg/txstatus" + +// transactionState initializes the nonserialized holder under a short global +// lock. Message hashing itself is protected only by the per-block state lock, +// so unrelated blocks can prepare concurrently. +func (b *Block) transactionState() *transactionDerivedState { + transactionDerivedStateInitMu.Lock() + defer transactionDerivedStateInitMu.Unlock() + if b.transactionDerivedState == nil { + b.transactionDerivedState = &transactionDerivedState{} + } + return b.transactionDerivedState +} + +// Len returns the number of identities in this immutable prepared set. +func (prepared *PreparedTransactionMessageIdentities) Len() int { + if prepared == nil { + return 0 + } + return len(prepared.identities) +} + +// Identity returns one prepared identity by transaction index. +func (prepared *PreparedTransactionMessageIdentities) Identity(index int) txstatus.TransactionMessageIdentity { + return prepared.identities[index] +} + +// MatchesBlock reports whether this prepared set is still bound to the +// block's ordered transaction pointers, message versions, and blockhashes. +// Signed message contents otherwise remain subject to Block's immutability +// contract; detecting arbitrary in-place edits would require hashing again. +func (prepared *PreparedTransactionMessageIdentities) MatchesBlock(block *Block) bool { + return block != nil && prepared.matches(block.Transactions) +} diff --git a/pkg/block/message_identity_concurrency_test.go b/pkg/block/message_identity_concurrency_test.go new file mode 100644 index 00000000..dcf4be04 --- /dev/null +++ b/pkg/block/message_identity_concurrency_test.go @@ -0,0 +1,50 @@ +package block + +import ( + "sync" + "testing" + + "github.com/gagliardetto/solana-go" +) + +func TestConcurrentTransactionMessageIdentityPreparationSharesOneResult(t *testing.T) { + block := &Block{Transactions: []*solana.Transaction{ + identityTestTransaction(1), + identityTestTransaction(2), + }} + const workers = 16 + start := make(chan struct{}) + results := make(chan *PreparedTransactionMessageIdentities, workers) + errors := make(chan error, workers) + var wait sync.WaitGroup + wait.Add(workers) + for range workers { + go func() { + defer wait.Done() + <-start + prepared, err := block.PrepareTransactionMessageIdentities() + results <- prepared + errors <- err + }() + } + close(start) + wait.Wait() + close(results) + close(errors) + + for err := range errors { + if err != nil { + t.Fatalf("concurrent preparation: %v", err) + } + } + var first *PreparedTransactionMessageIdentities + for prepared := range results { + if first == nil { + first = prepared + continue + } + if prepared != first { + t.Fatal("concurrent preparation published more than one identity set") + } + } +} diff --git a/pkg/block/message_identity_lookup_test.go b/pkg/block/message_identity_lookup_test.go new file mode 100644 index 00000000..d99d1563 --- /dev/null +++ b/pkg/block/message_identity_lookup_test.go @@ -0,0 +1,41 @@ +package block + +import ( + "testing" + + "github.com/gagliardetto/solana-go" +) + +func TestPreparedTransactionMessageIdentitySurvivesV0LookupResolution(t *testing.T) { + tableID := solana.PublicKey{0x70} + tx := identityTestTransaction(1) + tx.Message.SetAddressTableLookups([]solana.MessageAddressTableLookup{{ + AccountKey: tableID, + WritableIndexes: []byte{0}, + ReadonlyIndexes: []byte{1}, + }}) + block := &Block{Transactions: []*solana.Transaction{tx}} + before, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("prepare unresolved identity: %v", err) + } + + if err := tx.Message.SetAddressTables(map[solana.PublicKey]solana.PublicKeySlice{ + tableID: {{0x71}, {0x72}}, + }); err != nil { + t.Fatalf("set address table: %v", err) + } + if err := tx.Message.ResolveLookups(); err != nil { + t.Fatalf("resolve address-table lookups: %v", err) + } + after, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("reuse resolved identity: %v", err) + } + if before != after { + t.Fatal("address-table resolution rebuilt an identity whose canonical message is unchanged") + } + if before.Identity(0) != after.Identity(0) { + t.Fatalf("identity changed across lookup resolution: before %+v, after %+v", before.Identity(0), after.Identity(0)) + } +} diff --git a/pkg/block/message_identity_mutation_test.go b/pkg/block/message_identity_mutation_test.go new file mode 100644 index 00000000..44ce4d49 --- /dev/null +++ b/pkg/block/message_identity_mutation_test.go @@ -0,0 +1,37 @@ +package block + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/gagliardetto/solana-go" +) + +func TestTransactionMessageIdentityInvalidatesOnRecentBlockhashChange(t *testing.T) { + tx := identityTestTransaction(1) + block := &Block{Transactions: []*solana.Transaction{tx}} + if _, err := block.PrepareTransactionMessageIdentities(); err != nil { + t.Fatalf("prepare original identity: %v", err) + } + originalCache := block.transactionDerivedState.messageIdentities + block.MarkTransactionSignaturesVerified() + + tx.Message.RecentBlockhash = solana.Hash{0x99} + got, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("rebuild identity: %v", err) + } + if block.transactionDerivedState.messageIdentities == originalCache { + t.Fatal("recent-blockhash mutation reused stale identity cache") + } + if block.TransactionSignaturesVerified() { + t.Fatal("recent-blockhash mutation retained signature-verification trust") + } + want, err := txstatus.IdentityForTransaction(tx) + if err != nil { + t.Fatalf("hash mutated transaction: %v", err) + } + if got.Identity(0) != want { + t.Fatalf("rebuilt identity = %+v, want %+v", got.Identity(0), want) + } +} diff --git a/pkg/block/message_identity_test.go b/pkg/block/message_identity_test.go new file mode 100644 index 00000000..155857db --- /dev/null +++ b/pkg/block/message_identity_test.go @@ -0,0 +1,163 @@ +package block + +import ( + "encoding/json" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/gagliardetto/solana-go" +) + +func identityTestTransaction(messageByte byte) *solana.Transaction { + return &solana.Transaction{ + Signatures: []solana.Signature{{messageByte}}, + Message: solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1}, + AccountKeys: []solana.PublicKey{{messageByte}, {0x44}}, + RecentBlockhash: solana.Hash{0x55}, + Instructions: []solana.CompiledInstruction{{ + ProgramIDIndex: 1, + Accounts: []uint16{0}, + Data: []byte{messageByte}, + }}, + }, + } +} + +func TestTransactionMessageIdentitiesReuseAcrossShallowBlockCopy(t *testing.T) { + block := &Block{Transactions: []*solana.Transaction{ + identityTestTransaction(1), + identityTestTransaction(2), + }} + first, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("prepare identities: %v", err) + } + second, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("reuse identities: %v", err) + } + if first != second { + t.Fatal("same block did not reuse prepared identity storage") + } + + candidate := *block + copied, err := candidate.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("reuse identities through shallow copy: %v", err) + } + if first != copied { + t.Fatal("shallow pre-consensus block copy did not share immutable identities") + } +} + +func TestTransactionMessageIdentitiesInvalidateOnTransactionSliceChange(t *testing.T) { + tests := map[string]func([]*solana.Transaction) []*solana.Transaction{ + "replace": func(transactions []*solana.Transaction) []*solana.Transaction { + transactions[0] = identityTestTransaction(3) + return transactions + }, + "reorder": func(transactions []*solana.Transaction) []*solana.Transaction { + transactions[0], transactions[1] = transactions[1], transactions[0] + return transactions + }, + "append": func(transactions []*solana.Transaction) []*solana.Transaction { + return append(transactions, identityTestTransaction(3)) + }, + "truncate": func(transactions []*solana.Transaction) []*solana.Transaction { + return transactions[:1] + }, + } + + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + block := &Block{Transactions: []*solana.Transaction{ + identityTestTransaction(1), + identityTestTransaction(2), + }} + if _, err := block.PrepareTransactionMessageIdentities(); err != nil { + t.Fatalf("prepare original identities: %v", err) + } + originalCache := block.transactionDerivedState.messageIdentities + block.MarkTransactionSignaturesVerified() + block.Transactions = mutate(block.Transactions) + + identities, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("rebuild identities: %v", err) + } + if block.transactionDerivedState.messageIdentities == originalCache { + t.Fatal("transaction slice change reused stale identity cache") + } + if block.TransactionSignaturesVerified() { + t.Fatal("transaction slice change retained signature-verification trust") + } + if identities.Len() != len(block.Transactions) { + t.Fatalf("identity count = %d, want %d", identities.Len(), len(block.Transactions)) + } + for index, tx := range block.Transactions { + want, err := txstatus.IdentityForTransaction(tx) + if err != nil { + t.Fatalf("hash transaction %d: %v", index, err) + } + if identities.Identity(index) != want { + t.Fatalf("identity %d = %+v, want %+v", index, identities.Identity(index), want) + } + } + }) + } +} + +func TestTransactionMessageIdentitiesAreNotSerialized(t *testing.T) { + original := &Block{Slot: 42, Transactions: []*solana.Transaction{identityTestTransaction(1)}} + if _, err := original.PrepareTransactionMessageIdentities(); err != nil { + t.Fatalf("prepare identities: %v", err) + } + + encoded, err := json.Marshal(original) + if err != nil { + t.Fatalf("marshal block: %v", err) + } + var decoded Block + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal block: %v", err) + } + if decoded.transactionDerivedState != nil { + t.Fatal("prepared identities crossed a serialization boundary") + } + if _, err := decoded.PrepareTransactionMessageIdentities(); err != nil { + t.Fatalf("prepare identities after serialization: %v", err) + } +} + +func TestFixupTxVersionsInvalidatesTransactionDerivedState(t *testing.T) { + tx := identityTestTransaction(1) + block := &Block{ + Transactions: []*solana.Transaction{tx}, + Versions: []uint8{uint8(solana.MessageVersionV0)}, + } + before, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("prepare legacy identity: %v", err) + } + block.MarkTransactionSignaturesVerified() + + block.FixupTxVersions() + + if tx.Message.GetVersion() != solana.MessageVersionV0 { + t.Fatalf("message version = %d, want v0", tx.Message.GetVersion()) + } + if block.transactionDerivedState.messageIdentities != nil { + t.Fatal("version fixup retained prepared identities") + } + if block.TransactionSignaturesVerified() { + t.Fatal("version fixup retained signature-verification trust") + } + after, err := block.PrepareTransactionMessageIdentities() + if err != nil { + t.Fatalf("prepare v0 identity: %v", err) + } + if before.Identity(0) == after.Identity(0) { + t.Fatal("legacy and v0 canonical messages produced the same identity") + } +} diff --git a/pkg/replay/block.go b/pkg/replay/block.go index e02692ab..1a14673d 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -3250,6 +3250,7 @@ func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Ac } type blockTransactionExecutionPlan struct { + messageIdentities *b.PreparedTransactionMessageIdentities execute []bool processedTxCount uint64 processedSignatures uint64 @@ -3259,21 +3260,25 @@ type blockTransactionExecutionPlan struct { // transactions presented to one bank. A duplicate message makes the whole // block invalid; replay must never silently filter it and compute a bank hash // over different contents than the producer committed. -func planBlockTransactionExecution(slot uint64, transactions []*solana.Transaction) (blockTransactionExecutionPlan, error) { - plan := blockTransactionExecutionPlan{execute: make([]bool, len(transactions))} - seen := make(map[[32]byte]int, len(transactions)) +func planBlockTransactionExecution(block *b.Block) (blockTransactionExecutionPlan, error) { + if block == nil { + return blockTransactionExecutionPlan{}, errors.New("nil block") + } + identities, err := block.PrepareTransactionMessageIdentities() + if err != nil { + return blockTransactionExecutionPlan{}, err + } + plan := blockTransactionExecutionPlan{ + messageIdentities: identities, + execute: make([]bool, identities.Len()), + } + seen := make(map[[32]byte]int, len(block.Transactions)) var duplicates *DuplicateTransactionMessagesError - for idx, tx := range transactions { - if tx == nil { - return blockTransactionExecutionPlan{}, fmt.Errorf("transaction %d is nil", idx) - } - messageHash, err := TransactionMessageHash(tx) - if err != nil { - return blockTransactionExecutionPlan{}, fmt.Errorf("hash transaction %d message: %w", idx, err) - } + for idx, tx := range block.Transactions { + messageHash := identities.Identity(idx).MessageHash if firstIndex, duplicate := seen[messageHash]; duplicate { if duplicates == nil { - duplicates = &DuplicateTransactionMessagesError{Slot: slot} + duplicates = &DuplicateTransactionMessagesError{Slot: block.Slot} } duplicates.DuplicateCount++ if len(duplicates.Occurrences) < maxDuplicateTransactionOccurrences { @@ -3298,7 +3303,7 @@ func validateBlockTransactionMessages(block *b.Block) error { if block == nil { return errors.New("nil block") } - _, err := planBlockTransactionExecution(block.Slot, block.Transactions) + _, err := planBlockTransactionExecution(block) return err } @@ -3316,6 +3321,10 @@ func validatePreConsensusTransactionStatuses( if block == nil { return errors.New("nil block") } + plan, err := planBlockTransactionExecution(block) + if err != nil { + return err + } candidate := *block switch { case block.SourceParentSlot != 0: @@ -3325,7 +3334,7 @@ func validatePreConsensusTransactionStatuses( default: candidate.ParentSlot = selectedParentSlot } - return statuses.ValidateBlock(&candidate) + return statuses.validateBlockWithPlan(&candidate, plan) } func sequentialTxLoop(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, block *b.Block, executionPlan blockTransactionExecutionPlan, dbgOpts *DebugOptions, shouldVerifySignatures bool) (fees.TxFeeInfoAccumulator, uint64) { @@ -3637,13 +3646,13 @@ func ProcessBlock( if block == nil { return nil, errors.New("validate transaction messages: nil block") } - if err := transactionStatuses.ValidateBlock(block); err != nil { - return nil, fmt.Errorf("validate transaction statuses for slot %d: %w", block.Slot, err) - } - executionPlan, err := planBlockTransactionExecution(block.Slot, block.Transactions) + executionPlan, err := planBlockTransactionExecution(block) if err != nil { return nil, fmt.Errorf("validate transaction messages for slot %d: %w", block.Slot, err) } + if err := transactionStatuses.validateBlockWithPlan(block, executionPlan); err != nil { + return nil, fmt.Errorf("validate transaction statuses for slot %d: %w", block.Slot, err) + } ctx, task := trace.NewTask(context.Background(), "ProcessBlock") defer task.End() trace.Log(ctx, "slot", fmt.Sprintf("%d", block.Slot)) @@ -3850,7 +3859,7 @@ func ProcessBlock( if err != nil { return slotCtx, err } - if statusErr := transactionStatuses.CommitBlock(block); statusErr != nil { + if statusErr := transactionStatuses.commitBlockWithPlan(block, executionPlan); statusErr != nil { return nil, fmt.Errorf("commit transaction statuses for slot %d after bank state commit: %w", block.Slot, statusErr) } diff --git a/pkg/replay/transaction_status.go b/pkg/replay/transaction_status.go index 394f935c..dfc3f21e 100644 --- a/pkg/replay/transaction_status.go +++ b/pkg/replay/transaction_status.go @@ -4,12 +4,10 @@ import ( "fmt" "strings" + "github.com/Overclock-Validator/mithril/pkg/txstatus" "github.com/gagliardetto/solana-go" - "github.com/zeebo/blake3" ) -const transactionMessageHashDomain = "solana-tx-message-v1" - const maxDuplicateTransactionOccurrences = 16 // DuplicateTransactionOccurrence identifies a repeated transaction message @@ -57,18 +55,5 @@ func (e *DuplicateTransactionMessagesError) Error() string { // TransactionMessageHash returns the message identity Agave uses for // AlreadyProcessed checks. Signatures are deliberately excluded. func TransactionMessageHash(tx *solana.Transaction) ([32]byte, error) { - var messageHash [32]byte - if tx == nil { - return messageHash, fmt.Errorf("transaction is nil") - } - message, err := tx.Message.MarshalBinary() - if err != nil { - return messageHash, fmt.Errorf("serialize transaction message: %w", err) - } - - hasher := blake3.New() - _, _ = hasher.Write([]byte(transactionMessageHashDomain)) - _, _ = hasher.Write(message) - copy(messageHash[:], hasher.Sum(nil)) - return messageHash, nil + return txstatus.TransactionMessageHash(tx) } diff --git a/pkg/replay/transaction_status_benchmark_test.go b/pkg/replay/transaction_status_benchmark_test.go new file mode 100644 index 00000000..7690d6e0 --- /dev/null +++ b/pkg/replay/transaction_status_benchmark_test.go @@ -0,0 +1,88 @@ +package replay + +import ( + "encoding/binary" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" +) + +var ( + benchmarkTransactionMessageHash [32]byte + benchmarkExecutionPlan blockTransactionExecutionPlan +) + +func benchmarkUniqueTransactions(count int) []*solana.Transaction { + transactions := make([]*solana.Transaction, count) + for index := range transactions { + data := make([]byte, 4) + binary.LittleEndian.PutUint32(data, uint32(index)) + transactions[index] = &solana.Transaction{ + Message: solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1}, + AccountKeys: []solana.PublicKey{{1}, {2}}, + RecentBlockhash: solana.Hash{3}, + Instructions: []solana.CompiledInstruction{{ + ProgramIDIndex: 1, + Accounts: []uint16{0}, + Data: data, + }}, + }, + } + } + return transactions +} + +func BenchmarkTransactionMessageIdentityLifecycle32K(benchmark *testing.B) { + transactions := benchmarkUniqueTransactions(32_000) + + benchmark.Run("six_hash_passes", func(benchmark *testing.B) { + benchmark.ReportAllocs() + benchmark.ResetTimer() + for range benchmark.N { + for range 6 { + for _, tx := range transactions { + messageHash, err := TransactionMessageHash(tx) + if err != nil { + benchmark.Fatal(err) + } + benchmarkTransactionMessageHash = messageHash + } + } + } + }) + + benchmark.Run("prepared_process_path", func(benchmark *testing.B) { + benchmark.ReportAllocs() + benchmark.ResetTimer() + for iteration := range benchmark.N { + block := &b.Block{Slot: uint64(iteration + 1), Transactions: transactions} + plan, err := planBlockTransactionExecution(block) + if err != nil { + benchmark.Fatal(err) + } + if !plan.messageIdentities.MatchesBlock(block) || + !plan.messageIdentities.MatchesBlock(block) { + benchmark.Fatal("prepared plan lost its block binding") + } + benchmarkExecutionPlan = plan + } + }) + + preparedBlock := &b.Block{Slot: 1, Transactions: transactions} + if _, err := preparedBlock.PrepareTransactionMessageIdentities(); err != nil { + benchmark.Fatal(err) + } + benchmark.Run("cached_duplicate_plan", func(benchmark *testing.B) { + benchmark.ReportAllocs() + benchmark.ResetTimer() + for range benchmark.N { + plan, err := planBlockTransactionExecution(preparedBlock) + if err != nil { + benchmark.Fatal(err) + } + benchmarkExecutionPlan = plan + } + }) +} diff --git a/pkg/replay/transaction_status_cache.go b/pkg/replay/transaction_status_cache.go index 0587d1a5..8320df05 100644 --- a/pkg/replay/transaction_status_cache.go +++ b/pkg/replay/transaction_status_cache.go @@ -505,9 +505,22 @@ func (c *TransactionStatusCache) ValidateBlock(block *b.Block) error { if block == nil { return errors.New("nil block") } - if _, err := planBlockTransactionExecution(block.Slot, block.Transactions); err != nil { + plan, err := planBlockTransactionExecution(block) + if err != nil { return err } + return c.validateBlockWithPlan(block, plan) +} + +// validateBlockWithPlan preserves the status-cache checks while letting +// replay reuse the exact immutable identities used for execution planning. +func (c *TransactionStatusCache) validateBlockWithPlan(block *b.Block, plan blockTransactionExecutionPlan) error { + if block == nil { + return errors.New("nil block") + } + if plan.messageIdentities == nil || !plan.messageIdentities.MatchesBlock(block) { + return errors.New("prepared transaction message identities do not match block") + } if c == nil { return &IncompleteTransactionStatusCoverageError{} } @@ -520,21 +533,18 @@ func (c *TransactionStatusCache) ValidateBlock(block *b.Block) error { if err := c.validateParentLocked(block); err != nil { return err } - return c.validateAncestorTransactionsLocked(block.Slot, block.Transactions) + return c.validateAncestorTransactionsLocked(block.Slot, plan.messageIdentities) } -func (c *TransactionStatusCache) validateAncestorTransactionsLocked(slot uint64, transactions []*solana.Transaction) error { +func (c *TransactionStatusCache) validateAncestorTransactionsLocked(slot uint64, identities *b.PreparedTransactionMessageIdentities) error { var already *AncestorAlreadyProcessedTransactionMessagesError - for index, tx := range transactions { - messageHash, err := TransactionMessageHash(tx) - if err != nil { - return fmt.Errorf("hash transaction %d message: %w", index, err) - } - group := c.visible[tx.Message.RecentBlockhash] + for index := 0; index < identities.Len(); index++ { + identity := identities.Identity(index) + group := c.visible[identity.RecentBlockhash] if group == nil { continue } - key := sliceTransactionStatusKey(messageHash, group.keyIndex) + key := sliceTransactionStatusKey(identity.MessageHash, group.keyIndex) if group.keys[key] == 0 { continue } @@ -545,7 +555,7 @@ func (c *TransactionStatusCache) validateAncestorTransactionsLocked(slot uint64, if len(already.Occurrences) < maxAncestorAlreadyProcessedOccurrences { already.Occurrences = append(already.Occurrences, AncestorAlreadyProcessedOccurrence{ Index: index, - ProcessedSlot: c.processedSlotLocked(tx.Message.RecentBlockhash, key), + ProcessedSlot: c.processedSlotLocked(identity.RecentBlockhash, key), }) } } @@ -566,34 +576,38 @@ func (c *TransactionStatusCache) CommitBlock(block *b.Block) error { if block == nil { return errors.New("commit transaction statuses: nil block") } + plan, err := planBlockTransactionExecution(block) + if err != nil { + return err + } + return c.commitBlockWithPlan(block, plan) +} + +// commitBlockWithPlan atomically rechecks the mutable lineage/status state and +// publishes the already-prepared immutable transaction identities. +func (c *TransactionStatusCache) commitBlockWithPlan(block *b.Block, plan blockTransactionExecutionPlan) error { + if block == nil || plan.messageIdentities == nil || !plan.messageIdentities.MatchesBlock(block) { + return errors.New("prepared transaction message identities do not match block") + } c.mu.Lock() defer c.mu.Unlock() if !c.coverageComplete { return &IncompleteTransactionStatusCoverageError{CachedRoot: c.rootedThrough} } - // Recheck both same-bank duplicates and ancestor status atomically with - // publication. This keeps CommitBlock safe even if a caller's earlier - // ValidateBlock result raced a branch transition. - if _, err := planBlockTransactionExecution(block.Slot, block.Transactions); err != nil { - return err - } + // Parent lineage and ancestor status are mutable, so both remain under the + // publication lock even when hashing and same-bank deduplication happened + // earlier. This keeps commit safe across a concurrent branch transition. if err := c.validateParentLocked(block); err != nil { return err } - if err := c.validateAncestorTransactionsLocked(block.Slot, block.Transactions); err != nil { + if err := c.validateAncestorTransactionsLocked(block.Slot, plan.messageIdentities); err != nil { return err } delta := make(transactionStatusDelta) - for index, tx := range block.Transactions { - if tx == nil { - return fmt.Errorf("commit transaction statuses: transaction %d is nil", index) - } - messageHash, err := TransactionMessageHash(tx) - if err != nil { - return fmt.Errorf("commit transaction statuses: hash transaction %d: %w", index, err) - } - blockhash := tx.Message.RecentBlockhash + for index := 0; index < plan.messageIdentities.Len(); index++ { + identity := plan.messageIdentities.Identity(index) + blockhash := identity.RecentBlockhash group := delta[blockhash] if group == nil { keyIndex := uint8(0) @@ -606,7 +620,7 @@ func (c *TransactionStatusCache) CommitBlock(block *b.Block) error { } delta[blockhash] = group } - group.keys[sliceTransactionStatusKey(messageHash, group.keyIndex)] = struct{}{} + group.keys[sliceTransactionStatusKey(identity.MessageHash, group.keyIndex)] = struct{}{} } if err := c.addDeltaVisibleLocked(delta); err != nil { diff --git a/pkg/replay/transaction_status_plan_binding_test.go b/pkg/replay/transaction_status_plan_binding_test.go new file mode 100644 index 00000000..30b456d3 --- /dev/null +++ b/pkg/replay/transaction_status_plan_binding_test.go @@ -0,0 +1,30 @@ +package replay + +import ( + "testing" +) + +func TestPreparedCommitRejectsTransactionReplacement(t *testing.T) { + cache := NewTransactionStatusCache() + if err := cache.CommitBlock(statusCacheTestBlock(10)); err != nil { + t.Fatal(err) + } + + candidate := statusCacheTestBlock(11, statusCacheTestTransaction(1, 2, 3)) + plan, err := planBlockTransactionExecution(candidate) + if err != nil { + t.Fatal(err) + } + candidate.Transactions[0] = statusCacheTestTransaction(4, 5, 6) + + err = cache.commitBlockWithPlan(candidate, plan) + if err == nil || err.Error() != "prepared transaction message identities do not match block" { + t.Fatalf("commit error = %v, want prepared-plan binding failure", err) + } + cache.mu.RLock() + tipSlot := cache.tip.slot + cache.mu.RUnlock() + if tipSlot != 10 { + t.Fatalf("status-cache tip = %d after stale-plan rejection, want 10", tipSlot) + } +} diff --git a/pkg/replay/transaction_status_prepared_test.go b/pkg/replay/transaction_status_prepared_test.go new file mode 100644 index 00000000..44a42ca8 --- /dev/null +++ b/pkg/replay/transaction_status_prepared_test.go @@ -0,0 +1,129 @@ +package replay + +import ( + "errors" + "testing" +) + +func TestPreparedCommitRechecksAncestorAfterForkSwitch(t *testing.T) { + cache := NewTransactionStatusCache() + requireNoError := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + requireNoError(cache.CommitBlock(statusCacheTestBlock(10))) + requireNoError(cache.CommitBlock(statusCacheTestBlock( + 11, + statusCacheTestTransaction(6, 6, 1), + ))) + + retried := statusCacheTestTransaction(7, 8, 2) + unique := statusCacheTestTransaction(7, 9, 3) + candidate := statusCacheTestBlock(12, retried, unique) + plan, err := planBlockTransactionExecution(candidate) + requireNoError(err) + requireNoError(cache.validateBlockWithPlan(candidate, plan)) + + requireNoError(cache.Unwind(11)) + replacement := statusCacheTestBlock( + 11, + statusCacheTestTransaction(7, 8, 4), + ) + requireNoError(cache.CommitBlock(replacement)) + + err = cache.commitBlockWithPlan(candidate, plan) + var ancestorErr *AncestorAlreadyProcessedTransactionMessagesError + if !errors.As(err, &ancestorErr) { + t.Fatalf("prepared commit error = %v, want ancestor AlreadyProcessed", err) + } + if ancestorErr.AlreadyProcessedCount != 1 || + len(ancestorErr.Occurrences) != 1 || + ancestorErr.Occurrences[0].Index != 0 || + ancestorErr.Occurrences[0].ProcessedSlot != 11 { + t.Fatalf("ancestor error = %+v, want candidate tx 0 processed at slot 11", ancestorErr) + } + + cache.mu.RLock() + tipSlot := cache.tip.slot + cache.mu.RUnlock() + if tipSlot != 11 { + t.Fatalf("status-cache tip = %d after rejected commit, want 11", tipSlot) + } + view := cache.View() + containsUnique, err := view.ContainsTransaction(unique) + requireNoError(err) + if containsUnique { + t.Fatal("rejected prepared commit partially published its unique transaction") + } +} + +func TestConcurrentPreparedSiblingCommitsPublishExactlyOne(t *testing.T) { + cache := NewTransactionStatusCache() + if err := cache.CommitBlock(statusCacheTestBlock(10)); err != nil { + t.Fatal(err) + } + + left := statusCacheTestBlock(11, statusCacheTestTransaction(1, 2, 3)) + right := statusCacheTestBlock(11, statusCacheTestTransaction(4, 5, 6)) + leftPlan, err := planBlockTransactionExecution(left) + if err != nil { + t.Fatal(err) + } + rightPlan, err := planBlockTransactionExecution(right) + if err != nil { + t.Fatal(err) + } + if err := cache.validateBlockWithPlan(left, leftPlan); err != nil { + t.Fatalf("prevalidate left sibling: %v", err) + } + if err := cache.validateBlockWithPlan(right, rightPlan); err != nil { + t.Fatalf("prevalidate right sibling: %v", err) + } + + start := make(chan struct{}) + results := make(chan error, 2) + go func() { + <-start + results <- cache.commitBlockWithPlan(left, leftPlan) + }() + go func() { + <-start + results <- cache.commitBlockWithPlan(right, rightPlan) + }() + close(start) + + successes := 0 + lineageFailures := 0 + for range 2 { + err := <-results + if err == nil { + successes++ + continue + } + var lineageErr *TransactionStatusLineageError + if errors.As(err, &lineageErr) { + lineageFailures++ + continue + } + t.Fatalf("unexpected sibling commit error: %v", err) + } + if successes != 1 || lineageFailures != 1 { + t.Fatalf("sibling results: %d successes, %d lineage failures; want 1 and 1", successes, lineageFailures) + } + + view := cache.View() + leftVisible, err := view.ContainsTransaction(left.Transactions[0]) + if err != nil { + t.Fatal(err) + } + rightVisible, err := view.ContainsTransaction(right.Transactions[0]) + if err != nil { + t.Fatal(err) + } + if leftVisible == rightVisible { + t.Fatalf("published sibling statuses: left=%t right=%t; want exactly one", leftVisible, rightVisible) + } +} diff --git a/pkg/replay/transaction_status_test.go b/pkg/replay/transaction_status_test.go index 06e476e1..bb915855 100644 --- a/pkg/replay/transaction_status_test.go +++ b/pkg/replay/transaction_status_test.go @@ -67,11 +67,13 @@ func TestPlanBlockTransactionExecutionRejectsDuplicateMessages(t *testing.T) { require.Equal(t, firstHash, duplicateHash) require.NotEqual(t, firstHash, differentHash) - plan, err := planBlockTransactionExecution(42, []*solana.Transaction{ - first, - duplicateMessage, - differentMessage, - }) + block := &b.Block{ + Slot: 42, + Transactions: []*solana.Transaction{ + first, duplicateMessage, differentMessage, + }, + } + plan, err := planBlockTransactionExecution(block) var duplicateErr *DuplicateTransactionMessagesError require.Error(t, err) require.True(t, errors.As(err, &duplicateErr)) @@ -85,7 +87,7 @@ func TestPlanBlockTransactionExecutionRejectsDuplicateMessages(t *testing.T) { } func TestPlanBlockTransactionExecutionRejectsNilTransaction(t *testing.T) { - _, err := planBlockTransactionExecution(42, []*solana.Transaction{nil}) + _, err := planBlockTransactionExecution(&b.Block{Slot: 42, Transactions: []*solana.Transaction{nil}}) require.ErrorContains(t, err, "transaction 0 is nil") } diff --git a/pkg/txstatus/agave_snapshot.go b/pkg/txstatus/agave_snapshot.go index e6df34ac..5354cf21 100644 --- a/pkg/txstatus/agave_snapshot.go +++ b/pkg/txstatus/agave_snapshot.go @@ -1,5 +1,5 @@ -// Package txstatus contains transaction-status cache interchange types that -// are shared by snapshot ingestion, replay, and block production. +// Package txstatus contains transaction-status identities and interchange +// types shared by snapshot ingestion, replay, and block production. package txstatus import ( diff --git a/pkg/txstatus/message_identity.go b/pkg/txstatus/message_identity.go new file mode 100644 index 00000000..4d41f545 --- /dev/null +++ b/pkg/txstatus/message_identity.go @@ -0,0 +1,48 @@ +package txstatus + +import ( + "fmt" + + "github.com/gagliardetto/solana-go" + "github.com/zeebo/blake3" +) + +const transactionMessageHashDomain = "solana-tx-message-v1" + +// TransactionMessageIdentity is the immutable identity used by Agave's +// AlreadyProcessed checks. Signatures are deliberately excluded. +type TransactionMessageIdentity struct { + MessageHash [32]byte + RecentBlockhash solana.Hash +} + +// TransactionMessageHash hashes a transaction's canonical message bytes. +func TransactionMessageHash(tx *solana.Transaction) ([32]byte, error) { + var messageHash [32]byte + if tx == nil { + return messageHash, fmt.Errorf("transaction is nil") + } + message, err := tx.Message.MarshalBinary() + if err != nil { + return messageHash, fmt.Errorf("serialize transaction message: %w", err) + } + + hasher := blake3.New() + _, _ = hasher.Write([]byte(transactionMessageHashDomain)) + _, _ = hasher.Write(message) + hasher.Sum(messageHash[:0]) + return messageHash, nil +} + +// IdentityForTransaction captures both components needed for a status-cache +// lookup so later phases never need to inspect or reserialize the message. +func IdentityForTransaction(tx *solana.Transaction) (TransactionMessageIdentity, error) { + messageHash, err := TransactionMessageHash(tx) + if err != nil { + return TransactionMessageIdentity{}, err + } + return TransactionMessageIdentity{ + MessageHash: messageHash, + RecentBlockhash: tx.Message.RecentBlockhash, + }, nil +} diff --git a/pkg/txstatus/message_identity_test.go b/pkg/txstatus/message_identity_test.go new file mode 100644 index 00000000..64cc0535 --- /dev/null +++ b/pkg/txstatus/message_identity_test.go @@ -0,0 +1,70 @@ +package txstatus + +import ( + "bytes" + "testing" + + "github.com/gagliardetto/solana-go" +) + +func TestTransactionMessageIdentityStableAcrossV0LookupResolution(t *testing.T) { + tableID := solana.PublicKey{0x70} + dynamicWritable := solana.PublicKey{0x71} + dynamicReadonly := solana.PublicKey{0x72} + tx := &solana.Transaction{ + Message: solana.Message{ + Header: solana.MessageHeader{ + NumRequiredSignatures: 1, + NumReadonlyUnsignedAccounts: 1, + }, + AccountKeys: []solana.PublicKey{{0x11}, {0x22}}, + RecentBlockhash: solana.Hash{0x33}, + Instructions: []solana.CompiledInstruction{{ + ProgramIDIndex: 1, + Accounts: []uint16{0, 2, 3}, + Data: []byte{0x44}, + }}, + }, + } + tx.Message.SetAddressTableLookups([]solana.MessageAddressTableLookup{{ + AccountKey: tableID, + WritableIndexes: []byte{0}, + ReadonlyIndexes: []byte{1}, + }}) + + beforeWire, err := tx.Message.MarshalBinary() + if err != nil { + t.Fatalf("serialize unresolved v0 message: %v", err) + } + before, err := IdentityForTransaction(tx) + if err != nil { + t.Fatalf("hash unresolved v0 message: %v", err) + } + + if err := tx.Message.SetAddressTables(map[solana.PublicKey]solana.PublicKeySlice{ + tableID: {dynamicWritable, dynamicReadonly}, + }); err != nil { + t.Fatalf("set address tables: %v", err) + } + if err := tx.Message.ResolveLookups(); err != nil { + t.Fatalf("resolve address-table lookups: %v", err) + } + if len(tx.Message.AccountKeys) != 4 { + t.Fatalf("resolved account-key count = %d, want 4", len(tx.Message.AccountKeys)) + } + + afterWire, err := tx.Message.MarshalBinary() + if err != nil { + t.Fatalf("serialize resolved v0 message: %v", err) + } + after, err := IdentityForTransaction(tx) + if err != nil { + t.Fatalf("hash resolved v0 message: %v", err) + } + if !bytes.Equal(beforeWire, afterWire) { + t.Fatalf("canonical v0 message changed across lookup resolution:\nbefore %x\nafter %x", beforeWire, afterWire) + } + if before != after { + t.Fatalf("message identity changed across lookup resolution: before %+v, after %+v", before, after) + } +} From a77c4dae2914991483a57f3d6c4236c8a1af1351 Mon Sep 17 00:00:00 2001 From: smcio Date: Fri, 24 Jul 2026 22:19:04 +0200 Subject: [PATCH 2/5] speed up replay hot paths --- cmd/mithril/node/node.go | 5 + config.example.toml | 4 + pkg/accountsdb/accountsdb.go | 190 +++++++-- pkg/accountsdb/batch.go | 352 +++++++++++----- pkg/accountsdb/batch_test.go | 287 ++++++++++++- pkg/accountsdb/cache_admission.go | 187 +++++++++ pkg/accountsdb/cache_admission_test.go | 89 ++++ pkg/accountsdb/compact.go | 11 +- pkg/accountsdb/fold.go | 20 +- pkg/accountsdb/index.go | 12 +- pkg/accountsdb/rewind.go | 28 +- pkg/bankhash/bankhash.go | 16 +- pkg/bankhash/lthash.go | 138 ++++++- pkg/bankhash/lthash_optimized_test.go | 382 ++++++++++++++++++ pkg/config/config.go | 3 +- pkg/epochstakes/epoch_stakes.go | 219 ++++++++-- .../epoch_stakes_generation_test.go | 256 ++++++++++++ pkg/global/global_ctx.go | 79 ++-- pkg/global/leader_vote_account_test.go | 89 ++++ pkg/leaderschedule/leader_schedule.go | 65 +-- .../leader_vote_account_test.go | 73 ++++ pkg/lthash/account_hasher_test.go | 141 +++++++ pkg/lthash/lthash.go | 85 ++-- pkg/metrics/metrics.go | 112 ++++- pkg/replay/block.go | 104 ++++- pkg/replay/commit.go | 49 ++- pkg/replay/commit_test.go | 217 ++++++++++ pkg/replay/epoch.go | 11 +- .../failed_publication_integration_test.go | 135 +++++++ pkg/replay/leader_vote_account_test.go | 98 +++++ pkg/replay/topsort_planner.go | 9 + pkg/replay/transaction.go | 58 ++- pkg/replay/vote_reward.go | 159 ++++++-- pkg/replay/vote_reward_verifier_cache.go | 253 ++++++++++++ .../vote_reward_verifier_cache_bench_test.go | 120 ++++++ pkg/replay/vote_reward_verifier_cache_test.go | 178 ++++++++ pkg/rewardcerts/final_cert.go | 25 ++ pkg/rewardcerts/validated.go | 109 ++++- pkg/rewardcerts/validated_verifier_test.go | 114 ++++++ pkg/statsd/replay_diagnostics_test.go | 155 +++++++ pkg/statsd/statsd.go | 172 ++++++-- scripts/replay_timings_viewer.py | 310 +++++++++++++- 42 files changed, 4714 insertions(+), 405 deletions(-) create mode 100644 pkg/accountsdb/cache_admission.go create mode 100644 pkg/accountsdb/cache_admission_test.go create mode 100644 pkg/bankhash/lthash_optimized_test.go create mode 100644 pkg/epochstakes/epoch_stakes_generation_test.go create mode 100644 pkg/global/leader_vote_account_test.go create mode 100644 pkg/leaderschedule/leader_vote_account_test.go create mode 100644 pkg/lthash/account_hasher_test.go create mode 100644 pkg/replay/failed_publication_integration_test.go create mode 100644 pkg/replay/leader_vote_account_test.go create mode 100644 pkg/replay/vote_reward_verifier_cache.go create mode 100644 pkg/replay/vote_reward_verifier_cache_bench_test.go create mode 100644 pkg/replay/vote_reward_verifier_cache_test.go create mode 100644 pkg/rewardcerts/validated_verifier_test.go create mode 100644 pkg/statsd/replay_diagnostics_test.go diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index f179e019..43d91e77 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -420,6 +420,7 @@ func init() { Run.Flags().BoolVar(&sbpf.UsePool, "use-pool", true, "Disable to allocate fresh slices") Run.Flags().IntVar(&accountsdb.StoreAccountsWorkers, "store-accounts-workers", 128, "Number of workers to write account updates") Run.Flags().IntVar(&accountsdb.ProgramCacheMaxMB, "program-cache-max-mb", accountsdb.DefaultProgramCacheMaxMB, "Maximum approximate SBPF program cache size in MiB") + Run.Flags().IntVar(&accountsdb.CommonAccountCacheMaxMB, "common-account-cache-max-mb", accountsdb.DefaultCommonAccountCacheMaxMB, "Approximate retained decoded account cache weight budget in MiB") Run.Flags().Int64Var(&rewindToSlot, "rewind-to-slot", 0, "Rewind durable account state to the fold batch boundary at this slot before replaying (must be a retained boundary; run once to list boundaries on mismatch)") // [tuning.pprof] section flags @@ -979,6 +980,10 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { if accountsdb.ProgramCacheMaxMB <= 0 { return fmt.Errorf("tuning.program_cache_max_mb must be > 0") } + accountsdb.CommonAccountCacheMaxMB = getInt("common-account-cache-max-mb", "tuning.common_account_cache_max_mb") + if accountsdb.CommonAccountCacheMaxMB <= 0 { + return fmt.Errorf("tuning.common_account_cache_max_mb must be > 0") + } return nil } diff --git a/config.example.toml b/config.example.toml index db715f53..c8119353 100644 --- a/config.example.toml +++ b/config.example.toml @@ -528,6 +528,10 @@ name = "mithril" # Approximate maximum retained SBPF program cache size in MiB. program_cache_max_mb = 1024 + # Approximate retained decoded account cache weight budget in MiB. Large block scans + # pass through a bounded two-hit admission filter before using this budget. + common_account_cache_max_mb = 256 + # [tuning.pprof] - CPU/Memory Profiling # # Usage (assuming port = 6060): diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index 0b744c82..47e76b90 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -33,6 +33,24 @@ type AccountsDb struct { VoteAcctCache otter.Cache[solana.PublicKey, *accounts.Account] CommonAcctsCache otter.Cache[solana.PublicKey, *accounts.Account] ProgramCache otter.Cache[solana.PublicKey, *ProgramCacheEntry] + // Otter permits concurrent ordinary operations but not Clear. Rewind takes + // the write side; hot-path program cache operations take the read side. + programCacheMu sync.RWMutex + + // readCacheEpochMu makes the authoritative index flip and its cache refresh + // one publication boundary. Batch readers retain the epoch captured with + // their Pebble snapshot and may only admit decoded values while it still + // matches, preventing an old appendvec read from overwriting a newer fold. + readCacheEpochMu sync.RWMutex + readCacheEpoch uint64 + commonAdmission *commonCacheAdmission + batchHooks batchReadTestHooks + + // appendVecReadMu pins appendvec paths from before an index snapshot until + // its file reads complete. Compaction/rewind take the write side across an + // index move plus source unlink, and the legacy mutable store takes it while + // writing, so a batch never falls through to a different logical epoch. + appendVecReadMu sync.RWMutex // RootedDurable keeps the canonical store rooted-only: replayed slots buffer // in an in-RAM working set (pkg/accounts) and fold to disk via CommitBatch @@ -86,16 +104,18 @@ func (silentLogger) Fatalf(format string, args ...any) { log.Fatalf(format, args var ( ErrNoAccount = errors.New("ErrNoAccount") - StoreAccountsWorkers = 128 - ProgramCacheMaxMB = DefaultProgramCacheMaxMB + StoreAccountsWorkers = 128 + ProgramCacheMaxMB = DefaultProgramCacheMaxMB + CommonAccountCacheMaxMB = DefaultCommonAccountCacheMaxMB ) const ( indexPebbleMemTableSize = 64 << 20 indexPebbleMemTableStopWritesThreshold = 4 - commonAccountCacheCapacity = 5000 + DefaultCommonAccountCacheMaxMB = 256 DefaultProgramCacheMaxMB = 1024 programCacheCostUnitBytes = 1 << 20 + commonAccountCacheEntryOverheadBytes = 256 ) // DisableIndexWAL (storage.index_wal=false) runs the account index without a @@ -196,6 +216,7 @@ func (accountsDb *AccountsDb) InitCaches() { if accountsDb.inProgressStoreRequests == nil { accountsDb.inProgressStoreRequests = list.New() } + accountsDb.commonAdmission = newCommonCacheAdmission() accountsDb.VoteAcctCache, err = otter.MustBuilder[solana.PublicKey, *accounts.Account](2500). Cost(func(key solana.PublicKey, acct *accounts.Account) uint32 { return 1 @@ -214,9 +235,9 @@ func (accountsDb *AccountsDb) InitCaches() { panic(err) } - accountsDb.CommonAcctsCache, err = otter.MustBuilder[solana.PublicKey, *accounts.Account](commonAccountCacheCapacity). + accountsDb.CommonAcctsCache, err = otter.MustBuilder[solana.PublicKey, *accounts.Account](commonAccountCacheCapacityBytes()). Cost(func(key solana.PublicKey, acct *accounts.Account) uint32 { - return 1 + return commonAccountCacheCost(acct) }). Build() if err != nil { @@ -236,6 +257,29 @@ func programCacheCapacityUnits() int { return ProgramCacheMaxMB } +func commonAccountCacheCapacityBytes() int { + maxMB := CommonAccountCacheMaxMB + if maxMB <= 0 { + maxMB = DefaultCommonAccountCacheMaxMB + } + maxInt := int(^uint(0) >> 1) + if maxMB > maxInt/(1<<20) { + return maxInt + } + return maxMB << 20 +} + +func commonAccountCacheCost(acct *accounts.Account) uint32 { + bytes := uint64(commonAccountCacheEntryOverheadBytes) + if acct != nil { + bytes += uint64(len(acct.Data)) + } + if bytes > uint64(^uint32(0)) { + return ^uint32(0) + } + return uint32(bytes) +} + func (entry *ProgramCacheEntry) CostUnits() uint32 { if entry == nil || entry.Program == nil { return 1 @@ -256,14 +300,20 @@ func (accountsDb *AccountsDb) MaybeGetProgramFromCache(pubkey solana.PublicKey) if accountsDb == nil { return nil, false } + accountsDb.programCacheMu.RLock() + defer accountsDb.programCacheMu.RUnlock() return accountsDb.ProgramCache.Get(pubkey) } func (accountsDb *AccountsDb) AddProgramToCache(pubkey solana.PublicKey, programEntry *ProgramCacheEntry) { + accountsDb.programCacheMu.RLock() + defer accountsDb.programCacheMu.RUnlock() accountsDb.ProgramCache.Set(pubkey, programEntry) } func (accountsDb *AccountsDb) RemoveProgramFromCache(pubkey solana.PublicKey) { + accountsDb.programCacheMu.RLock() + defer accountsDb.programCacheMu.RUnlock() accountsDb.ProgramCache.Delete(pubkey) } @@ -271,19 +321,28 @@ func (accountsDb *AccountsDb) GetAccount(slot uint64, pubkey solana.PublicKey) ( if accountsDb == nil { return nil, ErrNoAccount } + accountsDb.appendVecReadMu.RLock() + defer accountsDb.appendVecReadMu.RUnlock() accts := accountsDb.getStoreInProgressAccounts([]solana.PublicKey{pubkey}) if accts[0] != nil { return accts[0], nil } - return accountsDb.getStoredAccount(slot, pubkey) + return accountsDb.getStoredAccountPinned(slot, pubkey) } func (accountsDb *AccountsDb) getStoredAccount(slot uint64, pubkey solana.PublicKey) (*accounts.Account, error) { + accountsDb.appendVecReadMu.RLock() + defer accountsDb.appendVecReadMu.RUnlock() + return accountsDb.getStoredAccountPinned(slot, pubkey) +} + +// getStoredAccountPinned requires appendVecReadMu to be held for reading. +func (accountsDb *AccountsDb) getStoredAccountPinned(slot uint64, pubkey solana.PublicKey) (*accounts.Account, error) { if accountsDb.Index == nil { return nil, ErrNoAccount } r := trace.StartRegion(context.Background(), "GetStoredAccountCache") - cachedAcct, hasAcct := accountsDb.getCachedAccount(pubkey) + cachedAcct, hasAcct, cacheEpoch := accountsDb.getCachedAccountAndEpoch(pubkey) if hasAcct { r.End() return cachedAcct, nil @@ -292,11 +351,9 @@ func (accountsDb *AccountsDb) getStoredAccount(slot uint64, pubkey solana.Public defer trace.StartRegion(context.Background(), "GetStoredAccountDisk").End() - // One-shot retry: between fetching the index entry and reading the file, - // a compaction cycle may move the record and unlink its old file (ENOENT, - // or in pathological interleavings a wrong-pubkey/short read). Re-fetching - // the index entry observes the moved location. Failing twice means real - // corruption and surfaces as an error rather than a panic. + // One-shot retry preserves the single-account path's existing tolerance for + // an externally removed or stale index location. The appendvec reader pin + // prevents in-process compaction and legacy stores from racing this read. acct, err := accountsDb.readIndexedAccount(pubkey) if err == ErrNoAccount { return nil, ErrNoAccount @@ -310,19 +367,40 @@ func (accountsDb *AccountsDb) getStoredAccount(slot uint64, pubkey solana.Public } } - accountsDb.cacheReadAccount(pubkey, acct) + accountsDb.cacheReadAccount(pubkey, acct, cacheEpoch) return acct, nil } func (accountsDb *AccountsDb) getCachedAccount(pubkey solana.PublicKey) (*accounts.Account, bool) { + accountsDb.readCacheEpochMu.RLock() + defer accountsDb.readCacheEpochMu.RUnlock() + return accountsDb.getCachedAccountLocked(pubkey) +} + +func (accountsDb *AccountsDb) getCachedAccountAndEpoch(pubkey solana.PublicKey) (*accounts.Account, bool, uint64) { + accountsDb.readCacheEpochMu.RLock() + defer accountsDb.readCacheEpochMu.RUnlock() + acct, ok := accountsDb.getCachedAccountLocked(pubkey) + return acct, ok, accountsDb.readCacheEpoch +} + +// getCachedAccountLocked requires readCacheEpochMu to be held for reading or +// writing. Keeping a whole batch's cache probes under one epoch makes its +// cache results coherent with the Pebble snapshot created at that boundary. +func (accountsDb *AccountsDb) getCachedAccountLocked(pubkey solana.PublicKey) (*accounts.Account, bool) { if acct, ok := accountsDb.VoteAcctCache.Get(pubkey); ok { return acct, true } return accountsDb.CommonAcctsCache.Get(pubkey) } -func (accountsDb *AccountsDb) cacheReadAccount(pubkey solana.PublicKey, acct *accounts.Account) { +func (accountsDb *AccountsDb) cacheReadAccount(pubkey solana.PublicKey, acct *accounts.Account, expectedEpoch uint64) { + accountsDb.readCacheEpochMu.RLock() + defer accountsDb.readCacheEpochMu.RUnlock() + if expectedEpoch != accountsDb.readCacheEpoch { + return + } if solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr { accountsDb.VoteAcctCache.Set(pubkey, acct) } else { @@ -330,20 +408,45 @@ func (accountsDb *AccountsDb) cacheReadAccount(pubkey solana.PublicKey, acct *ac } } -// cacheBatchReadAccount keeps small batch reads useful to the read-through -// cache without allowing a large block scan to flood a 5k-entry cache with -// tens of thousands of one-shot accounts. Vote accounts retain their dedicated -// cache regardless of batch size. -func (accountsDb *AccountsDb) cacheBatchReadAccount(pubkey solana.PublicKey, acct *accounts.Account, admitCommon bool) (voteAdmitted, commonAdmitted bool) { +type batchCacheAdmission uint8 + +const ( + batchCacheNone batchCacheAdmission = iota + batchCacheCommonSkipped + batchCacheVoteSkipped + batchCacheEpochRejected + batchCacheCommon + batchCacheVote +) + +// cacheBatchReadAccount publishes a decoded value only if the authoritative +// index/cache epoch captured with its batch snapshot is still current. Large +// common-account scans are filtered by commonCacheAdmission before this call; +// vote accounts retain their dedicated unconditional policy. +func (accountsDb *AccountsDb) cacheBatchReadAccount( + pubkey solana.PublicKey, + acct *accounts.Account, + admitCommon bool, + expectedEpoch uint64, +) batchCacheAdmission { + if hook := accountsDb.batchHooks.beforeCacheAdmission; hook != nil { + hook(pubkey) + } + accountsDb.readCacheEpochMu.RLock() + defer accountsDb.readCacheEpochMu.RUnlock() + if expectedEpoch != accountsDb.readCacheEpoch { + return batchCacheEpochRejected + } if solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr { - accountsDb.VoteAcctCache.Set(pubkey, acct) - return true, false + if accountsDb.VoteAcctCache.Set(pubkey, acct) { + return batchCacheVote + } + return batchCacheVoteSkipped } - if admitCommon { - accountsDb.CommonAcctsCache.Set(pubkey, acct) - return false, true + if admitCommon && accountsDb.CommonAcctsCache.Set(pubkey, acct) { + return batchCacheCommon } - return false, false + return batchCacheCommonSkipped } // readIndexedAccount performs one index-fetch + file-read attempt. @@ -440,30 +543,43 @@ func (accountsDb *AccountsDb) StoreAccounts( m[a.Key] = a } // Must not hold lock during channel send to avoid deadlock with storeWorker. + accountsDb.appendVecReadMu.Lock() accountsDb.inProgressStoreRequestsMu.Lock() element := accountsDb.inProgressStoreRequests.PushBack(storeRequest{accts: accts, slot: slot, m: m, cb: cb}) accountsDb.inProgressStoreRequestsMu.Unlock() + accountsDb.appendVecReadMu.Unlock() accountsDb.storeRequestChan <- element return nil } func (accountsDb *AccountsDb) storeAccountsSync(accts []*accounts.Account, slot uint64) { defer trace.StartRegion(context.Background(), "StoreAccounts").End() + // The request is still present in the in-progress overlay, so publish its + // cache epoch before disk I/O. New readers see the overlay; older readers + // cannot publish stale values after this bump. + accountsDb.refreshReadCaches(accts) + accountsDb.appendVecReadMu.Lock() + defer accountsDb.appendVecReadMu.Unlock() if StoreAccountsWorkers == 1 { accountsDb.storeAccountsInternal(accts, slot) } else { accountsDb.parallelStoreAccounts(StoreAccountsWorkers, accts, slot) } - - accountsDb.refreshReadCaches(accts) } // refreshReadCaches keeps already-hot common entries coherent after a store, -// but does not admit every account in a large fold: doing so turns the 5k-entry -// cache into an expensive write-only churn loop. Vote accounts retain their +// but does not admit every account in a large fold: doing so turns the retained +// byte budget into an expensive write-only churn loop. Vote accounts retain their // dedicated cache. Deleted and owner-transitioned entries are evicted from the // cache that can no longer serve them. func (accountsDb *AccountsDb) refreshReadCaches(accts []*accounts.Account) { + accountsDb.readCacheEpochMu.Lock() + defer accountsDb.readCacheEpochMu.Unlock() + accountsDb.readCacheEpoch++ + accountsDb.refreshReadCachesLocked(accts) +} + +func (accountsDb *AccountsDb) refreshReadCachesLocked(accts []*accounts.Account) { for _, acct := range accts { if acct == nil { continue @@ -477,12 +593,26 @@ func (accountsDb *AccountsDb) refreshReadCaches(accts []*accounts.Account) { } else { accountsDb.VoteAcctCache.Delete(acct.Key) if accountsDb.CommonAcctsCache.Has(acct.Key) { - accountsDb.CommonAcctsCache.Set(acct.Key, acct) + if !accountsDb.CommonAcctsCache.Set(acct.Key, acct) { + // A weighted entry may outgrow Otter's per-item ceiling. + // Never retain the prior value when replacement is rejected. + accountsDb.CommonAcctsCache.Delete(acct.Key) + } } } } } +// resetReadCachesLocked requires readCacheEpochMu for writing. +func (accountsDb *AccountsDb) resetReadCachesLocked() { + accountsDb.CommonAcctsCache.Clear() + accountsDb.VoteAcctCache.Clear() + accountsDb.programCacheMu.Lock() + accountsDb.ProgramCache.Clear() + accountsDb.programCacheMu.Unlock() + accountsDb.commonAdmission = newCommonCacheAdmission() +} + func (accountsDb *AccountsDb) storeWorker() { defer close(accountsDb.storeWorkerDone) for elt := range accountsDb.storeRequestChan { diff --git a/pkg/accountsdb/batch.go b/pkg/accountsdb/batch.go index dbcf4449..30d576e2 100644 --- a/pkg/accountsdb/batch.go +++ b/pkg/accountsdb/batch.go @@ -1,6 +1,7 @@ package accountsdb import ( + "bytes" "context" "errors" "fmt" @@ -15,8 +16,10 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/cockroachdb/pebble" "github.com/gagliardetto/solana-go" + "golang.org/x/sync/errgroup" ) var systemProgramAddr [32]byte @@ -24,7 +27,21 @@ var systemProgramAddr [32]byte // appendVecReadChunkSize amortizes open/close calls without serializing every // read from a large fold segment behind one worker. Chunks are offset-sorted, // which also gives the kernel a chance to coalesce nearby appendvec reads. -const appendVecReadChunkSize = 64 +const ( + appendVecReadChunkSize = 64 + + // Point Gets remain useful for tiny requests and for recent keys high in a + // deep LSM. Above this conservative crossover, monotonic snapshot iterators + // amortize Pebble's per-key read-state and iterator setup. + batchIndexIteratorThreshold = 16 + batchIndexKeysPerIterator = 1024 + batchIndexMinIterators = 4 + batchIndexMaxIterators = 8 +) + +type batchReadTestHooks struct { + beforeCacheAdmission func(solana.PublicKey) +} type batchAccountLocation struct { outputIdx int @@ -64,6 +81,8 @@ type BatchReadStats struct { CommonCacheAdmissions uint64 CommonCacheAdmissionsSkipped uint64 VoteCacheAdmissions uint64 + VoteCacheAdmissionsSkipped uint64 + CachePublicationEpochRejects uint64 DecodedAccountObjects uint64 DecodedAccountBytes uint64 @@ -71,23 +90,22 @@ type BatchReadStats struct { WorkingSetLookupNanoseconds uint64 InProgressNanoseconds uint64 + AppendVecPinWaitNanoseconds uint64 CacheLookupNanoseconds uint64 + AdmissionFilterNanoseconds uint64 IndexLookupNanoseconds uint64 ReadPlanningNanoseconds uint64 AppendVecReadNanoseconds uint64 + CachePublicationNanoseconds uint64 } type batchChunkReadStats struct { - appendVecAccounts uint64 - openFailures uint64 - readFailures uint64 - retryAccounts uint64 - commonCacheAdmissions uint64 - commonCacheAdmissionsSkipped uint64 - voteCacheAdmissions uint64 - decodedAccountObjects uint64 - decodedAccountBytes uint64 - placeholderObjects uint64 + appendVecAccounts uint64 + openFailures uint64 + readFailures uint64 + decodedAccountObjects uint64 + decodedAccountBytes uint64 + placeholderObjects uint64 } func (db *AccountsDb) GetAccountsBatch(ctx context.Context, slot uint64, pks []solana.PublicKey) ([]*accounts.Account, error) { @@ -115,18 +133,37 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 if len(pks) == 0 { return nil, stats, nil } - + if err := ctx.Err(); err != nil { + return nil, stats, err + } phaseStart := time.Now() + db.appendVecReadMu.RLock() + stats.AppendVecPinWaitNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) + appendVecPinned := true + defer func() { + if appendVecPinned { + db.appendVecReadMu.RUnlock() + } + }() + + phaseStart = time.Now() out := db.getStoreInProgressAccounts(pks) stats.InProgressNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) phaseStart = time.Now() cold := make([]int, 0, len(pks)) + var indexSnapshot *pebble.Snapshot + var cacheEpoch uint64 + var admission *commonCacheAdmission + var snapshotSetupNanoseconds uint64 + db.readCacheEpochMu.RLock() + cacheEpoch = db.readCacheEpoch + admission = db.commonAdmission for i, pk := range pks { if out[i] != nil { stats.InProgressHits++ continue } - if acct, ok := db.getCachedAccount(pk); ok { + if acct, ok := db.getCachedAccountLocked(pk); ok { stats.CacheHits++ if acct == nil || acct.Lamports == 0 { stats.PlaceholderObjects++ @@ -137,6 +174,14 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 cold = append(cold, i) } stats.CacheLookupNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) + if len(cold) > 0 && db.Index != nil { + // The cache probes and snapshot share one publication epoch. A fold + // cannot flip the index and refresh caches between these two views. + snapshotStart := time.Now() + indexSnapshot = db.Index.NewSnapshot() + snapshotSetupNanoseconds = uint64(time.Since(snapshotStart).Nanoseconds()) + } + db.readCacheEpochMu.RUnlock() if len(cold) == 0 { return out, stats, nil } @@ -149,37 +194,19 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 return out, stats, nil } - // Resolve every cold key's index location with a fixed-size worker pool. - // The old implementation launched one goroutine per miss and merely gated - // them with a semaphore, creating tens of thousands of goroutine stacks and - // scheduler operations on a busy block. - locations := make([]batchAccountLocation, len(cold)) - found := make([]bool, len(cold)) phaseStart = time.Now() - err := runBatchWorkers(ctx, len(cold), func(job int) error { - idx := cold[job] - pk := pks[idx] - entryBytes, closer, err := db.Index.Get(pk[:]) - if err != nil { - if errors.Is(err, pebble.ErrNotFound) { - out[idx] = missingAccount(pk) - return nil - } - return fmt.Errorf("index get %s: %w", pk, err) - } - entry, decodeErr := UnmarshalAcctIdxEntry(entryBytes) - closer.Close() - if decodeErr != nil { - return fmt.Errorf("unmarshal index entry for %s: %w", pk, decodeErr) - } - locations[job] = batchAccountLocation{outputIdx: idx, pubkey: pk, entry: *entry} - found[job] = true - return nil - }) - stats.IndexLookupNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) + admitCommon := admission.classifyAndObserve(pks, cold) + stats.AdmissionFilterNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) + phaseStart = time.Now() + locations, found, err := resolveBatchAccountLocations(ctx, indexSnapshot, pks, cold, out) + closeErr := indexSnapshot.Close() + stats.IndexLookupNanoseconds = snapshotSetupNanoseconds + uint64(time.Since(phaseStart).Nanoseconds()) if err != nil { return nil, stats, err } + if closeErr != nil { + return nil, stats, fmt.Errorf("close account index snapshot: %w", closeErr) + } phaseStart = time.Now() groups := make(map[appendVecID][]batchAccountLocation) @@ -206,13 +233,13 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 } } stats.AppendVecChunks = uint64(len(chunks)) - stats.ReadPlanningNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) - admitCommon := len(cold) <= commonAccountCacheCapacity chunkStats := make([]batchChunkReadStats, len(chunks)) + decodedForCache := make([]*accounts.Account, len(pks)) + stats.ReadPlanningNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) // Each worker opens an appendvec once per small chunk, instead of once per - // account. A stale/unlinked compaction location falls back to the existing - // one-account read, which re-fetches the index and retries once. + // account. appendVecReadMu pins every resolved source path until all reads + // finish, so an error is real I/O/corruption rather than a compaction race. phaseStart = time.Now() err = runBatchWorkers(ctx, len(chunks), func(job int) error { chunk := chunks[job] @@ -221,8 +248,7 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 file, openErr := os.Open(path) if openErr != nil { jobStats.openFailures++ - jobStats.retryAccounts += uint64(len(chunk.locations)) - return db.retryBatchLocations(ctx, slot, out, chunk.locations, jobStats) + return fmt.Errorf("open account appendvec %s: %w", path, openErr) } defer file.Close() @@ -233,28 +259,12 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 acct, readErr := readBatchAccountAt(file, path, location) if readErr != nil { jobStats.readFailures++ - jobStats.retryAccounts++ - placeholder, err := db.retryBatchLocation(slot, out, location) - if err != nil { - return err - } - if placeholder { - jobStats.placeholderObjects++ - } - continue + return readErr } jobStats.appendVecAccounts++ jobStats.decodedAccountObjects++ jobStats.decodedAccountBytes += uint64(len(acct.Data)) - voteAdmitted, commonAdmitted := db.cacheBatchReadAccount(location.pubkey, acct, admitCommon) - switch { - case voteAdmitted: - jobStats.voteCacheAdmissions++ - case commonAdmitted: - jobStats.commonCacheAdmissions++ - default: - jobStats.commonCacheAdmissionsSkipped++ - } + decodedForCache[location.outputIdx] = acct if acct.Lamports == 0 { jobStats.placeholderObjects++ } @@ -262,25 +272,203 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 } return nil }) - stats.AppendVecReadNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) for _, chunkStat := range chunkStats { stats.AppendVecAccounts += chunkStat.appendVecAccounts stats.OpenFailures += chunkStat.openFailures stats.ReadFailures += chunkStat.readFailures - stats.RetryAccounts += chunkStat.retryAccounts - stats.CommonCacheAdmissions += chunkStat.commonCacheAdmissions - stats.CommonCacheAdmissionsSkipped += chunkStat.commonCacheAdmissionsSkipped - stats.VoteCacheAdmissions += chunkStat.voteCacheAdmissions stats.DecodedAccountObjects += chunkStat.decodedAccountObjects stats.DecodedAccountBytes += chunkStat.decodedAccountBytes stats.PlaceholderObjects += chunkStat.placeholderObjects } + stats.AppendVecReadNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) + if err != nil { + return nil, stats, err + } + + // File paths are no longer needed. Let compaction/rewind proceed while the + // decoded values pass through the epoch-checked selective cache policy. + db.appendVecReadMu.RUnlock() + appendVecPinned = false + phaseStart = time.Now() + publicationJobs := cold[:0] + for _, idx := range cold { + acct := decodedForCache[idx] + if acct == nil { + continue + } + isVote := solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr + if !isVote && !admitCommon[idx] { + stats.CommonCacheAdmissionsSkipped++ + continue + } + publicationJobs = append(publicationJobs, idx) + } + publicationResults := make([]batchCacheAdmission, len(publicationJobs)) + err = runBatchWorkers(ctx, len(publicationJobs), func(job int) error { + idx := publicationJobs[job] + acct := decodedForCache[idx] + publicationResults[job] = db.cacheBatchReadAccount( + pks[idx], acct, admitCommon[idx], cacheEpoch, + ) + return nil + }) + for _, result := range publicationResults { + switch result { + case batchCacheVote: + stats.VoteCacheAdmissions++ + case batchCacheCommon: + stats.CommonCacheAdmissions++ + case batchCacheCommonSkipped: + stats.CommonCacheAdmissionsSkipped++ + case batchCacheVoteSkipped: + stats.VoteCacheAdmissionsSkipped++ + case batchCacheEpochRejected: + stats.CachePublicationEpochRejects++ + } + } + stats.CachePublicationNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) if err != nil { return nil, stats, err } return out, stats, nil } +func resolveBatchAccountLocations( + ctx context.Context, + snapshot *pebble.Snapshot, + pks []solana.PublicKey, + cold []int, + out []*accounts.Account, +) ([]batchAccountLocation, []bool, error) { + if len(cold) < batchIndexIteratorThreshold { + return resolveBatchAccountLocationsPointGets(ctx, snapshot, pks, cold, out) + } + return resolveBatchAccountLocationsIterators( + ctx, + snapshot, + pks, + cold, + out, + batchIndexIteratorWorkers(len(cold)), + ) +} + +func batchIndexIteratorWorkers(keyCount int) int { + workers := (keyCount + batchIndexKeysPerIterator - 1) / batchIndexKeysPerIterator + workers = max(workers, batchIndexMinIterators) + workers = min(workers, batchIndexMaxIterators, runtime.GOMAXPROCS(0)) + return min(keyCount, max(1, workers)) +} + +// resolveBatchAccountLocationsPointGets retains Pebble's specialized point +// iterator for small batches, where sorting and a full merging iterator cost +// more than the per-key setup it would amortize. +func resolveBatchAccountLocationsPointGets( + ctx context.Context, + snapshot *pebble.Snapshot, + pks []solana.PublicKey, + cold []int, + out []*accounts.Account, +) ([]batchAccountLocation, []bool, error) { + locations := make([]batchAccountLocation, len(cold)) + found := make([]bool, len(cold)) + err := runBatchWorkers(ctx, len(cold), func(job int) error { + idx := cold[job] + entryBytes, closer, err := snapshot.Get(pks[idx][:]) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + out[idx] = missingAccount(pks[idx]) + return nil + } + return fmt.Errorf("index get %s: %w", pks[idx], err) + } + entry, decodeErr := UnmarshalAcctIdxEntryValue(entryBytes) + closeErr := closer.Close() + if decodeErr != nil { + return fmt.Errorf("unmarshal index entry for %s: %w", pks[idx], decodeErr) + } + if closeErr != nil { + return fmt.Errorf("close index value for %s: %w", pks[idx], closeErr) + } + locations[job] = batchAccountLocation{outputIdx: idx, pubkey: pks[idx], entry: entry} + found[job] = true + return nil + }) + return locations, found, err +} + +// resolveBatchAccountLocationsIterators sorts the local cold-index slice and +// assigns contiguous key ranges to worker-local iterators created from one +// snapshot. Repeated monotonic SeekGE calls let Pebble reuse forward seek state +// and amortize read-state/iterator setup without scanning sparse key gaps. +func resolveBatchAccountLocationsIterators( + ctx context.Context, + snapshot *pebble.Snapshot, + pks []solana.PublicKey, + cold []int, + out []*accounts.Account, + workerCount int, +) ([]batchAccountLocation, []bool, error) { + sort.Slice(cold, func(i, j int) bool { + return bytes.Compare(pks[cold[i]][:], pks[cold[j]][:]) < 0 + }) + locations := make([]batchAccountLocation, len(cold)) + found := make([]bool, len(cold)) + workerCount = min(len(cold), max(1, workerCount)) + + g, workerCtx := errgroup.WithContext(ctx) + for worker := range workerCount { + start := len(cold) * worker / workerCount + end := len(cold) * (worker + 1) / workerCount + g.Go(func() (retErr error) { + iter, err := snapshot.NewIterWithContext(workerCtx, &pebble.IterOptions{ + KeyTypes: pebble.IterKeyTypePointsOnly, + }) + if err != nil { + return fmt.Errorf("create account index iterator: %w", err) + } + defer func() { + if closeErr := iter.Close(); retErr == nil && closeErr != nil { + retErr = fmt.Errorf("close account index iterator: %w", closeErr) + } + }() + + for job := start; job < end; job++ { + if err := workerCtx.Err(); err != nil { + return err + } + idx := cold[job] + if !iter.SeekGE(pks[idx][:]) { + if err := iter.Error(); err != nil { + return fmt.Errorf("index seek %s: %w", pks[idx], err) + } + out[idx] = missingAccount(pks[idx]) + continue + } + if !bytes.Equal(iter.Key(), pks[idx][:]) { + out[idx] = missingAccount(pks[idx]) + continue + } + entryBytes, err := iter.ValueAndErr() + if err != nil { + return fmt.Errorf("read index value for %s: %w", pks[idx], err) + } + entry, err := UnmarshalAcctIdxEntryValue(entryBytes) + if err != nil { + return fmt.Errorf("unmarshal index entry for %s: %w", pks[idx], err) + } + locations[job] = batchAccountLocation{outputIdx: idx, pubkey: pks[idx], entry: entry} + found[job] = true + } + return iter.Error() + }) + } + if err := g.Wait(); err != nil { + return nil, nil, err + } + return locations, found, nil +} + func readBatchAccountAt(file *os.File, path string, location batchAccountLocation) (*accounts.Account, error) { if location.entry.Offset > math.MaxInt64 { return nil, fmt.Errorf("account offset %d overflows int64", location.entry.Offset) @@ -297,32 +485,6 @@ func readBatchAccountAt(file *os.File, path string, location batchAccountLocatio return acct, nil } -func (db *AccountsDb) retryBatchLocations(ctx context.Context, slot uint64, out []*accounts.Account, locations []batchAccountLocation, stats *batchChunkReadStats) error { - for _, location := range locations { - if err := ctx.Err(); err != nil { - return err - } - placeholder, err := db.retryBatchLocation(slot, out, location) - if err != nil { - return err - } - if placeholder { - stats.placeholderObjects++ - } - } - return nil -} - -func (db *AccountsDb) retryBatchLocation(slot uint64, out []*accounts.Account, location batchAccountLocation) (bool, error) { - acct, err := db.getStoredAccount(slot, location.pubkey) - if err != nil && err != ErrNoAccount { - return false, err - } - placeholder := acct == nil || acct.Lamports == 0 - out[location.outputIdx] = batchAccountOrPlaceholder(location.pubkey, acct) - return placeholder, nil -} - func batchAccountOrPlaceholder(pubkey solana.PublicKey, acct *accounts.Account) *accounts.Account { if acct == nil || acct.Lamports == 0 { return missingAccount(pubkey) diff --git a/pkg/accountsdb/batch_test.go b/pkg/accountsdb/batch_test.go index 6afc2c7a..709b5a5a 100644 --- a/pkg/accountsdb/batch_test.go +++ b/pkg/accountsdb/batch_test.go @@ -4,14 +4,17 @@ import ( "context" "encoding/binary" "errors" + "fmt" "math" "runtime" "sync" "sync/atomic" "testing" + "time" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -96,11 +99,11 @@ func TestGetAccountsBatchStatsAndSmallBatchAdmissions(t *testing.T) { assert.Equal(t, uint64(1), second.IndexMisses, "negative entries are deliberately not cached yet") } -func TestLargeBatchDoesNotFloodCommonCache(t *testing.T) { +func TestLargeBatchSelectivelyAdmitsReusableAccounts(t *testing.T) { db, _ := newFoldTestDb(t) defer db.CloseDb() - count := commonAccountCacheCapacity + 1 + count := commonAccountImmediateAdmissionLimit + 1 delta := make([]*accounts.Account, count) keys := make([]solana.PublicKey, count) for i := range keys { @@ -109,19 +112,142 @@ func TestLargeBatchDoesNotFloodCommonCache(t *testing.T) { } _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 100, Delta: delta}}, 100, nil, nil) require.NoError(t, err) + hotKey := solana.PublicKey{} + hotKey[len(hotKey)-1] = 0xfe + hot := &accounts.Account{Key: hotKey, Lamports: 99, Owner: [32]byte{7}} + require.True(t, db.CommonAcctsCache.Set(hot.Key, hot)) _, first, err := db.GetAccountsBatchSharedWithStats(context.Background(), 100, keys) require.NoError(t, err) assert.Equal(t, uint64(count), first.IndexHits) assert.Equal(t, uint64(count), first.CommonCacheAdmissionsSkipped) assert.Zero(t, first.CommonCacheAdmissions) - assert.False(t, db.CommonAcctsCache.Has(keys[0])) - assert.False(t, db.CommonAcctsCache.Has(keys[len(keys)-1])) + assert.True(t, db.CommonAcctsCache.Has(hot.Key), "one-shot scan must preserve established heat") _, second, err := db.GetAccountsBatchSharedWithStats(context.Background(), 100, keys) require.NoError(t, err) - assert.Zero(t, second.CacheHits, "one-shot scan must not churn cache or masquerade as reusable heat") + assert.Zero(t, second.CacheHits, "the second observation performs the selective admission") assert.Equal(t, uint64(count), second.IndexHits) + assert.Equal(t, uint64(count), second.CommonCacheAdmissions) + + _, third, err := db.GetAccountsBatchSharedWithStats(context.Background(), 100, keys) + require.NoError(t, err) + assert.Equal(t, uint64(count), third.CacheHits, "reusable large-batch accounts must become cache hits") + assert.Zero(t, third.IndexHits) + assert.True(t, db.CommonAcctsCache.Has(hot.Key)) +} + +func TestGetAccountsBatchIteratorExactMatchOrderAndInputImmutability(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + + const count = batchIndexIteratorThreshold*2 + 17 + delta := make([]*accounts.Account, count) + keys := make([]solana.PublicKey, count) + for i := range keys { + binary.BigEndian.PutUint64(keys[i][:8], uint64(2*i+2)) + delta[i] = &accounts.Account{Key: keys[i], Lamports: uint64(i + 1), Owner: [32]byte{7}} + } + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 100, Delta: delta}}, 100, nil, nil) + require.NoError(t, err) + for _, key := range keys { + db.CommonAcctsCache.Delete(key) + } + + request := make([]solana.PublicKey, 0, count+4) + for i := range keys { + requestIdx := (i * 257) % count + request = append(request, keys[requestIdx]) + } + between := solana.PublicKey{} + binary.BigEndian.PutUint64(between[:8], 3) // strictly between stored keys 2 and 4 + request = append(request, between, keys[0], keys[count/2], keys[count/2]) + original := append([]solana.PublicKey(nil), request...) + + out, err := db.GetAccountsBatch(context.Background(), 100, request) + require.NoError(t, err) + assert.Equal(t, original, request, "resolver must sort only its local index list") + require.Len(t, out, len(request)) + for i := 0; i < count; i++ { + requestIdx := (i * 257) % count + assert.Equal(t, uint64(requestIdx+1), out[i].Lamports) + } + assert.Equal(t, between, out[count].Key) + assert.Zero(t, out[count].Lamports, "SeekGE must not return the next stored key for an interior miss") + assert.Equal(t, keys[0], out[count+1].Key) + assert.Equal(t, out[count+2].Lamports, out[count+3].Lamports) +} + +func TestGetAccountsBatchRejectsCancelledContextAndMalformedIndex(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := db.GetAccountsBatch(ctx, 100, []solana.PublicKey{{1}}) + assert.ErrorIs(t, err, context.Canceled) + + key := solana.PublicKey{2} + require.NoError(t, db.Index.Set(key[:], []byte{1, 2, 3}, pebble.NoSync)) + _, err = db.GetAccountsBatch(context.Background(), 100, []solana.PublicKey{key}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unmarshal index entry") +} + +func TestBatchCacheAdmissionCannotOverwriteNewerFold(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + + old := foldAcct(1, 10, []byte{1}) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 100, Delta: []*accounts.Account{old}}}, 100, nil, nil) + require.NoError(t, err) + db.CommonAcctsCache.Delete(old.Key) + + readComplete := make(chan struct{}) + resumeAdmission := make(chan struct{}) + var once sync.Once + db.batchHooks.beforeCacheAdmission = func(pk solana.PublicKey) { + if pk != old.Key { + return + } + once.Do(func() { + close(readComplete) + <-resumeAdmission + }) + } + + type batchResult struct { + out []*accounts.Account + err error + } + done := make(chan batchResult, 1) + go func() { + out, err := db.GetAccountsBatch(context.Background(), 100, []solana.PublicKey{old.Key}) + done <- batchResult{out: out, err: err} + }() + + select { + case <-readComplete: + case <-time.After(5 * time.Second): + t.Fatal("old batch did not reach cache admission hook") + } + updated := foldAcct(1, 20, []byte{2}) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 101, Delta: []*accounts.Account{updated}}}, 101, nil, nil) + require.NoError(t, err) + close(resumeAdmission) + result := <-done + require.NoError(t, result.err) + require.Len(t, result.out, 1) + assert.Equal(t, uint64(10), result.out[0].Lamports, "in-flight snapshot remains internally old") + db.batchHooks.beforeCacheAdmission = nil + + got, err := db.GetAccount(101, old.Key) + require.NoError(t, err) + assert.Equal(t, uint64(20), got.Lamports) + assert.Equal(t, []byte{2}, got.Data) + if cached, ok := db.CommonAcctsCache.Get(old.Key); ok { + assert.Equal(t, uint64(20), cached.Lamports, "stale batch must never win cache publication") + } } func TestRefreshReadCachesIsScanResistantAndCoherent(t *testing.T) { @@ -234,6 +360,17 @@ func BenchmarkGetAccountsBatchColdFoldSegment(b *testing.B) { db.VoteAcctCache.Delete(key) } } + resetAdmission := func() { + db.readCacheEpochMu.Lock() + db.commonAdmission = newCommonCacheAdmission() + db.readCacheEpochMu.Unlock() + } + mustRead := func() { + out, err := db.GetAccountsBatch(context.Background(), 100, keys) + if err != nil || len(out) != len(keys) { + b.Fatalf("batch: len=%d err=%v", len(out), err) + } + } b.Run("former-goroutine-per-account", func(b *testing.B) { b.ReportAllocs() @@ -248,20 +385,144 @@ func BenchmarkGetAccountsBatchColdFoldSegment(b *testing.B) { } }) - b.Run("bounded-grouped-read", func(b *testing.B) { + b.Run("one-shot-cold", func(b *testing.B) { b.ReportAllocs() for range b.N { b.StopTimer() evict() + resetAdmission() b.StartTimer() - out, err := db.GetAccountsBatch(context.Background(), 100, keys) - if err != nil || len(out) != len(keys) { - b.Fatalf("grouped batch: len=%d err=%v", len(out), err) - } + mustRead() + } + }) + + b.Run("second-observation-admit", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + evict() + resetAdmission() + mustRead() + b.StartTimer() + mustRead() + } + }) + + b.Run("third-observation-cache-hit", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + evict() + resetAdmission() + mustRead() + mustRead() + b.StartTimer() + mustRead() + } + }) + + b.Run("three-block-amortized", func(b *testing.B) { + b.ReportAllocs() + b.ReportMetric(3, "blocks/op") + for range b.N { + b.StopTimer() + evict() + resetAdmission() + b.StartTimer() + mustRead() + mustRead() + mustRead() } }) } +func BenchmarkResolveBatchAccountLocations(b *testing.B) { + db, _ := newFoldTestDb(b) + defer db.CloseDb() + + const accountCount = 1 << 16 + storedKeys := make([]solana.PublicKey, accountCount) + indexBatch := db.Index.NewBatch() + var entryBuf [24]byte + for i := range storedKeys { + binary.BigEndian.PutUint64(storedKeys[i][:8], uint64(i+1)) + entry := AccountIndexEntry{Slot: 100, FileId: 1, Offset: uint64(i)} + entry.Marshal(&entryBuf) + require.NoError(b, indexBatch.Set(storedKeys[i][:], entryBuf[:], nil)) + } + require.NoError(b, indexBatch.Commit(pebble.NoSync)) + require.NoError(b, indexBatch.Close()) + require.NoError(b, db.Index.Flush()) + + run := func(b *testing.B, request []solana.PublicKey, pointGets, auto bool, workers int) { + b.Helper() + b.ReportAllocs() + b.ReportMetric(float64(len(request)), "keys/op") + validated := false + for range b.N { + cold := make([]int, len(request)) + for i := range cold { + cold[i] = i + } + out := make([]*accounts.Account, len(request)) + snapshot := db.Index.NewSnapshot() + var locations []batchAccountLocation + var found []bool + var err error + if pointGets { + locations, found, err = resolveBatchAccountLocationsPointGets( + context.Background(), snapshot, request, cold, out, + ) + } else if auto { + locations, found, err = resolveBatchAccountLocations( + context.Background(), snapshot, request, cold, out, + ) + } else { + locations, found, err = resolveBatchAccountLocationsIterators( + context.Background(), snapshot, request, cold, out, workers, + ) + } + closeErr := snapshot.Close() + if err != nil || closeErr != nil || len(locations) != len(request) || + len(found) != len(request) || !found[0] || !found[len(found)-1] { + b.Fatalf("resolve: locations=%d found=%d err=%v close=%v", len(locations), len(found), err, closeErr) + } + if !validated { + b.StopTimer() + for i, ok := range found { + if !ok { + b.Fatalf("resolver missed interior result %d", i) + } + } + validated = true + b.StartTimer() + } + } + } + + for _, size := range []int{1, 2, 4, 8, 16, 64, 256, 1024, 8192, 65536} { + request := make([]solana.PublicKey, size) + for i := range request { + // Spread every request across the fixture's full keyspace. Since 4051 + // is odd, the full-size case remains a complete permutation. + request[i] = storedKeys[(i*4051)&(accountCount-1)] + } + b.Run(fmt.Sprintf("%06d-keys", size), func(b *testing.B) { + b.Run("point", func(b *testing.B) { + run(b, request, true, false, 0) + }) + b.Run("auto", func(b *testing.B) { + run(b, request, false, true, 0) + }) + for _, workers := range []int{1, 2, 4, 8, 16, 32} { + b.Run(fmt.Sprintf("iterator-%02d", workers), func(b *testing.B) { + run(b, request, false, false, workers) + }) + } + }) + } +} + func BenchmarkCommonCacheBulkAdmission(b *testing.B) { const accountCount = 20_000 db := &AccountsDb{} @@ -273,11 +534,11 @@ func BenchmarkCommonCacheBulkAdmission(b *testing.B) { accts[i] = &accounts.Account{Key: key, Lamports: uint64(i + 1), Owner: [32]byte{7}} } - b.Run("flood-5k-cache", func(b *testing.B) { + b.Run("accepted-publication", func(b *testing.B) { b.ReportAllocs() for range b.N { for _, acct := range accts { - db.cacheBatchReadAccount(acct.Key, acct, true) + db.cacheBatchReadAccount(acct.Key, acct, true, 0) } } }) @@ -285,7 +546,7 @@ func BenchmarkCommonCacheBulkAdmission(b *testing.B) { b.ReportAllocs() for range b.N { for _, acct := range accts { - db.cacheBatchReadAccount(acct.Key, acct, false) + db.cacheBatchReadAccount(acct.Key, acct, false, 0) } } }) diff --git a/pkg/accountsdb/cache_admission.go b/pkg/accountsdb/cache_admission.go new file mode 100644 index 00000000..7b5b058e --- /dev/null +++ b/pkg/accountsdb/cache_admission.go @@ -0,0 +1,187 @@ +package accountsdb + +import ( + "math/bits" + "sync" + + "github.com/cespare/xxhash/v2" + "github.com/gagliardetto/solana-go" +) + +const ( + // Small batches are cheap enough to admit immediately. Large scans pass + // through the two-hit doorkeeper below so one-shot block accounts cannot + // churn the retained account cache. + commonAccountImmediateAdmissionLimit = 5000 + + // Bound cache writes from one large block. When more reusable candidates + // are present, successive batches rotate through deterministic hash + // partitions so a recurring working set is admitted progressively. + commonAccountMaxAdmissionsPerBatch = 16 * 1024 + + // Two 2 MiB Bloom generations retain roughly the previous 1-2 million + // observations. Four probes keep the worst-window false-admission rate near + // 2%; false positives affect cache performance only, never read correctness. + commonAdmissionBloomWords = (2 << 20) / 8 + commonAdmissionBloomProbes = 4 + commonAdmissionRotateAfterKeys = 1_000_000 + commonAdmissionSecondaryHashXor = uint64(0x9e3779b97f4a7c15) +) + +// commonCacheAdmission is a bounded two-hit doorkeeper in front of Otter's +// S3-FIFO value cache. Classification is deliberately batch-atomic: every key +// is queried before any key from the same request is inserted, so duplicate +// pubkeys within one request cannot manufacture reusable heat. +type commonCacheAdmission struct { + mu sync.Mutex + + current []uint64 + previous []uint64 + observed uint64 + + // partitionRound lets an oversized recurring working set fill the cache + // over several blocks instead of selecting the same subset forever. + partitionRound uint64 +} + +func newCommonCacheAdmission() *commonCacheAdmission { + return &commonCacheAdmission{ + current: make([]uint64, commonAdmissionBloomWords), + previous: make([]uint64, commonAdmissionBloomWords), + } +} + +// classifyAndObserve marks which requested cold keys may be admitted after a +// successful account read. Small batches admit immediately. Large batches +// require an observation from an earlier batch and cap admissions by rotating +// through deterministic hash partitions. +func (a *commonCacheAdmission) classifyAndObserve( + pks []solana.PublicKey, + cold []int, +) []bool { + admit := make([]bool, len(pks)) + if len(cold) == 0 { + return admit + } + if a == nil { + if len(cold) <= commonAccountImmediateAdmissionLimit { + for _, idx := range cold { + admit[idx] = true + } + } + return admit + } + + a.mu.Lock() + defer a.mu.Unlock() + + if len(cold) <= commonAccountImmediateAdmissionLimit { + for _, idx := range cold { + admit[idx] = true + } + a.rotateBeforeInsert(uint64(len(cold))) + for _, idx := range cold { + a.insert(pks[idx]) + } + a.observed += uint64(len(cold)) + return admit + } + + // Query the complete batch before inserting any current-batch key. + candidateCount := 0 + for _, idx := range cold { + if a.contains(pks[idx]) { + admit[idx] = true + candidateCount++ + } + } + + shift := admissionPartitionShift(candidateCount) + var partitionMask uint64 + var selectedPartition uint64 + if shift > 0 { + partitionMask = (uint64(1) << shift) - 1 + selectedPartition = a.partitionRound & partitionMask + a.partitionRound++ + } + admissionCount := 0 + for _, idx := range cold { + if !admit[idx] { + continue + } + if admissionCount < commonAccountMaxAdmissionsPerBatch && + (shift == 0 || admissionPartitionHash(pks[idx])&partitionMask == selectedPartition) { + admissionCount++ + } else { + admit[idx] = false + } + } + + a.rotateBeforeInsert(uint64(len(cold))) + for _, idx := range cold { + a.insert(pks[idx]) + } + a.observed += uint64(len(cold)) + return admit +} + +func admissionPartitionShift(candidateCount int) uint { + if candidateCount <= commonAccountMaxAdmissionsPerBatch { + return 0 + } + ratio := (candidateCount + commonAccountMaxAdmissionsPerBatch - 1) / + commonAccountMaxAdmissionsPerBatch + return uint(bits.Len(uint(ratio - 1))) +} + +func (a *commonCacheAdmission) rotateBeforeInsert(incoming uint64) { + if a.observed == 0 || a.observed+incoming <= commonAdmissionRotateAfterKeys { + return + } + a.previous, a.current = a.current, a.previous + clear(a.current) + a.observed = 0 +} + +func (a *commonCacheAdmission) contains(pk solana.PublicKey) bool { + h1, h2 := commonAdmissionHashes(pk) + mask := uint64(len(a.current)*64 - 1) + for i := uint64(0); i < commonAdmissionBloomProbes; i++ { + bit := (h1 + i*h2) & mask + word, bitInWord := bit>>6, bit&63 + flag := uint64(1) << bitInWord + if a.current[word]&flag == 0 && a.previous[word]&flag == 0 { + return false + } + } + return true +} + +func (a *commonCacheAdmission) insert(pk solana.PublicKey) { + h1, h2 := commonAdmissionHashes(pk) + mask := uint64(len(a.current)*64 - 1) + for i := uint64(0); i < commonAdmissionBloomProbes; i++ { + bit := (h1 + i*h2) & mask + a.current[bit>>6] |= uint64(1) << (bit & 63) + } +} + +func commonAdmissionHashes(pk solana.PublicKey) (uint64, uint64) { + h1 := xxhash.Sum64(pk[:]) + h2 := mixAdmissionHash(h1^commonAdmissionSecondaryHashXor) | 1 + return h1, h2 +} + +func admissionPartitionHash(pk solana.PublicKey) uint64 { + h1, h2 := commonAdmissionHashes(pk) + return mixAdmissionHash(h1 + h2) +} + +func mixAdmissionHash(x uint64) uint64 { + x ^= x >> 33 + x *= 0xff51afd7ed558ccd + x ^= x >> 33 + x *= 0xc4ceb9fe1a85ec53 + x ^= x >> 33 + return x +} diff --git a/pkg/accountsdb/cache_admission_test.go b/pkg/accountsdb/cache_admission_test.go new file mode 100644 index 00000000..b0449b66 --- /dev/null +++ b/pkg/accountsdb/cache_admission_test.go @@ -0,0 +1,89 @@ +package accountsdb + +import ( + "encoding/binary" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCommonCacheAdmissionDuplicatesDoNotFakeReuse(t *testing.T) { + admission := newCommonCacheAdmission() + count := commonAccountImmediateAdmissionLimit + 1 + pks := make([]solana.PublicKey, count) + cold := make([]int, count) + for i := range cold { + pks[i] = solana.PublicKey{1} + cold[i] = i + } + + first := admission.classifyAndObserve(pks, cold) + assert.Zero(t, countTrue(first), "duplicates in one large request must not manufacture a second observation") + second := admission.classifyAndObserve(pks, cold) + assert.Equal(t, count, countTrue(second), "a later request is a genuine second observation") +} + +func TestCommonCacheAdmissionIsBoundedAndRotates(t *testing.T) { + admission := newCommonCacheAdmission() + count := commonAccountMaxAdmissionsPerBatch * 2 + pks := make([]solana.PublicKey, count) + cold := make([]int, count) + for i := range pks { + binary.LittleEndian.PutUint64(pks[i][:8], uint64(i+1)) + cold[i] = i + } + + assert.Zero(t, countTrue(admission.classifyAndObserve(pks, cold))) + admitted := countTrue(admission.classifyAndObserve(pks, cold)) + assert.Positive(t, admitted) + assert.LessOrEqual(t, admitted, commonAccountMaxAdmissionsPerBatch) + assert.Len(t, admission.current, commonAdmissionBloomWords) + assert.Len(t, admission.previous, commonAdmissionBloomWords) + + old := solana.PublicKey{0xf0} + newer := solana.PublicKey{0xf1} + admission = newCommonCacheAdmission() + admission.insert(old) + admission.observed = commonAdmissionRotateAfterKeys + admission.rotateBeforeInsert(1) + assert.True(t, admission.contains(old), "one previous generation remains reusable") + admission.insert(newer) + admission.observed = commonAdmissionRotateAfterKeys + admission.rotateBeforeInsert(1) + assert.False(t, admission.contains(old), "keys expire after two rotations") + assert.True(t, admission.contains(newer)) +} + +func TestCommonAccountCacheUsesByteBudget(t *testing.T) { + previous := CommonAccountCacheMaxMB + CommonAccountCacheMaxMB = 1 + defer func() { CommonAccountCacheMaxMB = previous }() + + db := &AccountsDb{} + db.InitCaches() + small := &accounts.Account{Key: solana.PublicKey{1}, Data: make([]byte, 1024)} + require.True(t, db.CommonAcctsCache.Set(small.Key, small)) + assert.Equal(t, uint32(commonAccountCacheEntryOverheadBytes+1024), commonAccountCacheCost(small)) + + // Otter rejects a single value above its 10% small-queue budget instead + // of evicting the entire retained working set for one giant account. + oversized := &accounts.Account{Key: solana.PublicKey{2}, Data: make([]byte, 128<<10)} + assert.False(t, db.CommonAcctsCache.Set(oversized.Key, oversized)) + + grown := &accounts.Account{Key: small.Key, Lamports: 2, Data: make([]byte, 128<<10)} + db.refreshReadCaches([]*accounts.Account{grown}) + assert.False(t, db.CommonAcctsCache.Has(small.Key), "a rejected weighted refresh must evict the stale prior value") +} + +func countTrue(values []bool) int { + count := 0 + for _, value := range values { + if value { + count++ + } + } + return count +} diff --git a/pkg/accountsdb/compact.go b/pkg/accountsdb/compact.go index 1b351d68..6b167f7e 100644 --- a/pkg/accountsdb/compact.go +++ b/pkg/accountsdb/compact.go @@ -301,7 +301,10 @@ func (db *AccountsDb) compactFile(c compactCandidate, minDeadFraction float64) ( if len(data) == 0 || len(liveIdx) == 0 { // Fully dead (or empty bankhash-only segment past the horizon): no // index state references it; drop source + its manifest. - if err := removeSourceFiles(db.AcctsDir, srcPath, c.manifestPath); err != nil { + db.appendVecReadMu.Lock() + err := removeSourceFiles(db.AcctsDir, srcPath, c.manifestPath) + db.appendVecReadMu.Unlock() + if err != nil { return false, 0, err } return true, 0, nil @@ -376,7 +379,11 @@ func (db *AccountsDb) compactFile(c compactCandidate, minDeadFraction float64) ( // Move the index entries in one batch, then make the move durable BEFORE // unlinking the source — with the WAL off that means an explicit Flush, - // because compact manifests are not replayed at recovery. + // because compact manifests are not replayed at recovery. Exclude snapshot + // readers from this short move-to-unlink window, so each sees either the old + // path or the new path and never needs a cross-epoch per-key fallback. + db.appendVecReadMu.Lock() + defer db.appendVecReadMu.Unlock() batch := db.Index.NewBatch() defer batch.Close() var idxBuf [24]byte diff --git a/pkg/accountsdb/fold.go b/pkg/accountsdb/fold.go index ff09a894..07f75ed2 100644 --- a/pkg/accountsdb/fold.go +++ b/pkg/accountsdb/fold.go @@ -273,19 +273,27 @@ func (db *AccountsDb) CommitBatch( } } - // (7) The index epoch flip: entries + meta in one batch. + // Prepare the cache refresh before entering the short publication section. + live := make([]*accounts.Account, 0, len(union)) + for _, k := range keys { + live = append(live, union[k].acct) + } + + // (7) The index epoch flip: entries + meta in one batch. Hold the read-cache + // publication lock through the refresh so an older batch read can neither + // observe a mixed index/cache view nor admit stale bytes after this commit. fire(db.foldHooks.beforeIndexCommit) + db.readCacheEpochMu.Lock() if err := db.applyManifestToIndex(manifest); err != nil { + db.readCacheEpochMu.Unlock() return BatchCommitResult{}, err } + db.readCacheEpoch++ + db.refreshReadCachesLocked(live) + db.readCacheEpochMu.Unlock() fire(db.foldHooks.afterIndexCommit) // (8) Publish. - live := make([]*accounts.Account, 0, len(union)) - for _, k := range keys { - live = append(live, union[k].acct) - } - db.refreshReadCaches(live) db.lastBatchSeq = batchSeq db.durableThrough.Store(throughSlot) diff --git a/pkg/accountsdb/index.go b/pkg/accountsdb/index.go index 4cf25139..d3503265 100644 --- a/pkg/accountsdb/index.go +++ b/pkg/accountsdb/index.go @@ -30,10 +30,18 @@ func (entry *AccountIndexEntry) Unmarshal(in *[24]byte) { } func UnmarshalAcctIdxEntry(data []byte) (*AccountIndexEntry, error) { + out, err := UnmarshalAcctIdxEntryValue(data) + if err != nil { + return nil, err + } + return &out, nil +} + +func UnmarshalAcctIdxEntryValue(data []byte) (AccountIndexEntry, error) { if len(data) < 24 { - return nil, fmt.Errorf("UnmarshalAcctIdxEntry: input had %d < 24 minimum bytes", len(data)) + return AccountIndexEntry{}, fmt.Errorf("UnmarshalAcctIdxEntry: input had %d < 24 minimum bytes", len(data)) } - out := &AccountIndexEntry{} + out := AccountIndexEntry{} out.Unmarshal((*[24]byte)(data[:24])) return out, nil } diff --git a/pkg/accountsdb/rewind.go b/pkg/accountsdb/rewind.go index cb0273c7..7dd5cce1 100644 --- a/pkg/accountsdb/rewind.go +++ b/pkg/accountsdb/rewind.go @@ -141,6 +141,8 @@ func (db *AccountsDb) finalizeParkedRewindLeftoversLocked(throughSlot uint64) { func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, error) { db.foldMu.Lock() defer db.foldMu.Unlock() + db.appendVecReadMu.Lock() + defer db.appendVecReadMu.Unlock() res := RewindResult{} meta, haveMeta, err := db.readFoldMeta() @@ -151,6 +153,19 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e return res, fmt.Errorf("accountsdb: rewind: store has no fold meta (nothing folded)") } if meta.ThroughSlot == throughSlot { + // This also repairs the in-memory publication state after a prior + // WAL-less commit succeeded but its explicit Flush reported an error. + db.readCacheEpochMu.Lock() + db.readCacheEpoch++ + db.resetReadCachesLocked() + db.readCacheEpochMu.Unlock() + if db.IndexWALDisabled { + if err := db.Index.Flush(); err != nil { + return res, err + } + } + db.lastBatchSeq = meta.BatchSeq + db.durableThrough.Store(meta.ThroughSlot) res.NewThrough = throughSlot if path, ok := db.manifestPathEither(throughSlot, meta.FileId); ok { if m, rerr := ReadSegmentManifest(path); rerr == nil { @@ -262,8 +277,15 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e }), nil); err != nil { return res, err } - if err := batch.Commit(pebble.Sync); err != nil { - return res, err + db.readCacheEpochMu.Lock() + commitErr := batch.Commit(pebble.Sync) + if commitErr == nil { + db.readCacheEpoch++ + db.resetReadCachesLocked() + } + db.readCacheEpochMu.Unlock() + if commitErr != nil { + return res, commitErr } if db.IndexWALDisabled { if err := db.Index.Flush(); err != nil { @@ -292,8 +314,6 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e db.lastBatchSeq = seqT db.durableThrough.Store(target.manifest.ThroughSlot) - // Read caches may hold folded-then-rewound values; rebuild them. - db.InitCaches() res.NewThrough = target.manifest.ThroughSlot res.ResumeCtx = target.manifest.ResumeCtx diff --git a/pkg/bankhash/bankhash.go b/pkg/bankhash/bankhash.go index d09186ef..4d9e02ce 100644 --- a/pkg/bankhash/bankhash.go +++ b/pkg/bankhash/bankhash.go @@ -21,15 +21,24 @@ func CalculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Acco var acctDeltaHash []byte if adhEnabled { - start := time.Now() + var start time.Time + if slotCtx.Replay { + start = time.Now() + } acctDeltaHash = calculateAcctsDeltaHash(writableAccts) - metrics.GlobalBlockReplay.AccountsDeltaHash.AddTimingSince(start) + if slotCtx.Replay { + metrics.GlobalBlockReplay.AccountsDeltaHash.AddTimingSince(start) + } } if ltHashEnabled { updateAcctsLtHash(slotCtx, modifiedAccts) } + var finalizeStart time.Time + if slotCtx.Replay { + finalizeStart = time.Now() + } hasher := sha256.New() // lt accts hash enabled @@ -66,6 +75,9 @@ func CalculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Acco } else { bankHash = hasher.Sum(nil) } + if slotCtx.Replay { + metrics.GlobalBlockReplay.BankHashFinalize.AddTimingSince(finalizeStart) + } return bankHash } diff --git a/pkg/bankhash/lthash.go b/pkg/bankhash/lthash.go index 3f6b4737..4cbf4656 100644 --- a/pkg/bankhash/lthash.go +++ b/pkg/bankhash/lthash.go @@ -6,9 +6,11 @@ import ( "os" "strconv" "sync" + "time" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/lthash" + "github.com/Overclock-Validator/mithril/pkg/metrics" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/sealevel" ) @@ -47,16 +49,36 @@ func calculateDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.A // than once (e.g. both rent-collected and modified, or a sysvar collected via // multiple paths) would otherwise have its delta counted multiple times, // corrupting the cumulative LtHash. + recordMetrics := slotCtx.Replay + var dedupeStart time.Time + if recordMetrics { + metrics.GlobalBlockReplay.LtHashInputAccounts = uint64(len(modifiedAccts)) + metrics.GlobalBlockReplay.LtHashUniqueAccounts = 0 + metrics.GlobalBlockReplay.LtHashUnchangedAccounts = 0 + metrics.GlobalBlockReplay.LtHashCreatedAccounts = 0 + metrics.GlobalBlockReplay.LtHashDeletedAccounts = 0 + metrics.GlobalBlockReplay.LtHashOldDataBytes = 0 + metrics.GlobalBlockReplay.LtHashNewDataBytes = 0 + dedupeStart = time.Now() + } modifiedAccts = dedupeModifiedAccts(modifiedAccts) + if recordMetrics { + metrics.GlobalBlockReplay.LtHashDedupe.AddTimingSince(dedupeStart) + metrics.GlobalBlockReplay.LtHashUniqueAccounts = uint64(len(modifiedAccts)) + } if len(modifiedAccts) == 0 { return <hash.LtHash{} } numWorkers := min(32, len(modifiedAccts)) - - hashes := make([]*lthash.LtHash, len(modifiedAccts)) + partials := make([]lthash.LtHash, numWorkers) + workerStats := make([]ltHashWorkerStats, numWorkers) chunkSize := (len(modifiedAccts) + numWorkers - 1) / numWorkers + var workerStart time.Time + if recordMetrics { + workerStart = time.Now() + } var wg sync.WaitGroup for i := range numWorkers { wg.Add(1) @@ -64,29 +86,76 @@ func calculateDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.A defer wg.Done() start := workerID * chunkSize end := min(start+chunkSize, len(modifiedAccts)) + var partial lthash.LtHash + var stats ltHashWorkerStats + + // Reuse one 2 KiB hash and one BLAKE3 hasher for every old/new + // account value owned by this worker. + var scratch lthash.LtHash + var hasher lthash.AccountHasher for j := start; j < end; j++ { - acct := modifiedAccts[j] - hashes[j] = calculateSingleDeltaLtHash(slotCtx, acct) + accumulateSingleDeltaLtHash(slotCtx, modifiedAccts[j], &partial, &scratch, &hasher, &stats) } + partials[workerID] = partial + workerStats[workerID] = stats }(i) } wg.Wait() + if recordMetrics { + metrics.GlobalBlockReplay.LtHashWorkerCompute.AddTimingSince(workerStart) + } + var reduceStart time.Time + if recordMetrics { + reduceStart = time.Now() + } var deltaHash lthash.LtHash - for _, h := range hashes { - deltaHash.Add(h) + var totals ltHashWorkerStats + for i := range partials { + deltaHash.Add(&partials[i]) + totals.add(workerStats[i]) + } + if recordMetrics { + metrics.GlobalBlockReplay.LtHashPartialReduce.AddTimingSince(reduceStart) + metrics.GlobalBlockReplay.LtHashUnchangedAccounts = totals.unchangedAccounts + metrics.GlobalBlockReplay.LtHashCreatedAccounts = totals.createdAccounts + metrics.GlobalBlockReplay.LtHashDeletedAccounts = totals.deletedAccounts + metrics.GlobalBlockReplay.LtHashOldDataBytes = totals.oldDataBytes + metrics.GlobalBlockReplay.LtHashNewDataBytes = totals.newDataBytes } return &deltaHash } +type ltHashWorkerStats struct { + unchangedAccounts uint64 + createdAccounts uint64 + deletedAccounts uint64 + oldDataBytes uint64 + newDataBytes uint64 +} + +func (stats *ltHashWorkerStats) add(other ltHashWorkerStats) { + stats.unchangedAccounts += other.unchangedAccounts + stats.createdAccounts += other.createdAccounts + stats.deletedAccounts += other.deletedAccounts + stats.oldDataBytes += other.oldDataBytes + stats.newDataBytes += other.newDataBytes +} + // dedupeModifiedAccts collapses duplicate keys, keeping each key's last // occurrence (the newest value) and preserving first-seen order. nil entries are // dropped. Without this, a key appearing twice would have its LtHash delta // applied twice. func dedupeModifiedAccts(modifiedAccts []*accounts.Account) []*accounts.Account { - if len(modifiedAccts) < 2 { + if len(modifiedAccts) == 0 { + return modifiedAccts + } + if len(modifiedAccts) == 1 { + if modifiedAccts[0] == nil { + return nil + } return modifiedAccts } @@ -107,35 +176,60 @@ func dedupeModifiedAccts(modifiedAccts []*accounts.Account) []*accounts.Account return unique } -func calculateSingleDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAcct *accounts.Account) *lthash.LtHash { +func accumulateSingleDeltaLtHash( + slotCtx *sealevel.SlotCtx, + modifiedAcct *accounts.Account, + deltaLtHash *lthash.LtHash, + scratch *lthash.LtHash, + hasher *lthash.AccountHasher, + stats *ltHashWorkerStats, +) { previousAcct, err := slotCtx.GetParentAccount(modifiedAcct.Key) if err != nil { panic(fmt.Sprintf("couldn't find parent acct for %s for slot %d", modifiedAcct.Key, slotCtx.Slot)) } - var deltaLtHash lthash.LtHash - - if previousAcct.Lamports != 0 { - if acctsEqual(modifiedAcct, previousAcct) { - return &deltaLtHash + // Zero-lamport accounts do not contribute, regardless of their other + // fields. This also handles the zero-to-zero no-op without hashing. + if previousAcct.Lamports == 0 { + if modifiedAcct.Lamports == 0 { + stats.unchangedAccounts++ + return } - var oldLtHash lthash.LtHash - oldLtHash.InitWithAcct(previousAcct) - deltaLtHash.Sub(&oldLtHash) + stats.createdAccounts++ + stats.newDataBytes += uint64(len(modifiedAcct.Data)) + hasher.HashInto(scratch, modifiedAcct) + deltaLtHash.Add(scratch) + return } - var newLtHash lthash.LtHash - newLtHash.InitWithAcct(modifiedAcct) - deltaLtHash.Add(&newLtHash) + if modifiedAcct.Lamports == 0 { + stats.deletedAccounts++ + stats.oldDataBytes += uint64(len(previousAcct.Data)) + hasher.HashInto(scratch, previousAcct) + deltaLtHash.Sub(scratch) + return + } + + // Rent epoch is deliberately absent: it is not an input to the accounts + // LtHash. A rent-epoch-only write therefore has an exact zero delta. + if acctsLtHashEqual(modifiedAcct, previousAcct) { + stats.unchangedAccounts++ + return + } - return &deltaLtHash + stats.oldDataBytes += uint64(len(previousAcct.Data)) + stats.newDataBytes += uint64(len(modifiedAcct.Data)) + hasher.HashInto(scratch, previousAcct) + deltaLtHash.Sub(scratch) + hasher.HashInto(scratch, modifiedAcct) + deltaLtHash.Add(scratch) } -func acctsEqual(a *accounts.Account, b *accounts.Account) bool { +func acctsLtHashEqual(a *accounts.Account, b *accounts.Account) bool { return a.Lamports == b.Lamports && a.Executable == b.Executable && - a.RentEpoch == b.RentEpoch && a.Owner == b.Owner && bytes.Equal(a.Data, b.Data) } diff --git a/pkg/bankhash/lthash_optimized_test.go b/pkg/bankhash/lthash_optimized_test.go new file mode 100644 index 00000000..0394f8af --- /dev/null +++ b/pkg/bankhash/lthash_optimized_test.go @@ -0,0 +1,382 @@ +package bankhash + +import ( + "bytes" + "encoding/binary" + "fmt" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/lthash" + "github.com/Overclock-Validator/mithril/pkg/metrics" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/zeebo/blake3" +) + +const ltHashBytes = 2048 + +func ltHashFixtureKey(index int) solana.PublicKey { + var key solana.PublicKey + binary.LittleEndian.PutUint64(key[:8], uint64(index+1)) + binary.LittleEndian.PutUint64(key[8:16], uint64(index+1)*0x9e3779b97f4a7c15) + key[31] = byte(index*37 + 11) + return key +} + +func ltHashFixtureAccount(key solana.PublicKey, seed int, lamports uint64, dataLen int) *accounts.Account { + data := make([]byte, dataLen) + for i := range data { + data[i] = byte(seed*17 + i*29) + } + var owner [32]byte + for i := range owner { + owner[i] = byte(seed*13 + i*7) + } + return &accounts.Account{ + Key: key, + Lamports: lamports, + Data: data, + Owner: owner, + Executable: seed%3 == 0, + RentEpoch: uint64(seed * 5), + } +} + +func buildLtHashDifferentialFixture(t testing.TB, uniqueAccounts int) (*sealevel.SlotCtx, []*accounts.Account) { + t.Helper() + parent := accounts.NewMemAccounts() + modified := make([]*accounts.Account, 0, uniqueAccounts+uniqueAccounts/17+uniqueAccounts/29+2) + + for i := range uniqueAccounts { + key := ltHashFixtureKey(i) + oldAcct := ltHashFixtureAccount(key, i+1, uint64(i+101), i%257) + newAcct := oldAcct.Clone() + + switch i % 7 { + case 0: // unchanged + case 1: // RentEpoch is not part of the LtHash input. + newAcct.RentEpoch += 991 + case 2: // create + oldAcct.Lamports = 0 + newAcct.Lamports += 10_000 + case 3: // delete + newAcct.Lamports = 0 + newAcct.Data = []byte{0xde, 0xad} + case 4: // data and lamports update + newAcct.Lamports += 77 + newAcct.Data = append(newAcct.Data, byte(i), 0xa5) + case 5: // owner/executable update + newAcct.Owner[0] ^= 0xff + newAcct.Executable = !newAcct.Executable + case 6: // both zero; other fields cannot contribute + oldAcct.Lamports = 0 + newAcct.Lamports = 0 + newAcct.Data = append(newAcct.Data, 0x42) + } + + if err := parent.SetAccountWithoutLock(key, oldAcct); err != nil { + t.Fatalf("seed parent account %d: %v", i, err) + } + + if i%17 == 0 { + stale := newAcct.Clone() + stale.Lamports += 1234 + stale.Data = append(stale.Data, 0x17) + modified = append(modified, stale) + } + modified = append(modified, newAcct) + if i%29 == 0 { + modified = append(modified, nil) + } + } + + return &sealevel.SlotCtx{Slot: 4242, ParentAccts: parent, AcctsLtHash: <hash.LtHash{}}, modified +} + +func legacyDedupeModifiedAccts(modifiedAccts []*accounts.Account) []*accounts.Account { + unique := make([]*accounts.Account, 0, len(modifiedAccts)) + seen := make(map[[32]byte]int, len(modifiedAccts)) + for _, acct := range modifiedAccts { + if acct == nil { + continue + } + key := [32]byte(acct.Key) + if index, ok := seen[key]; ok { + unique[index] = acct + continue + } + seen[key] = len(unique) + unique = append(unique, acct) + } + return unique +} + +// legacyAccountHashBytes is an independent copy of the pre-optimization +// account hash: it materializes the 2 KiB BLAKE3 XOF before LtHash reduction. +func legacyAccountHashBytes(acct *accounts.Account) []byte { + if acct.Lamports == 0 { + return nil + } + hasher := blake3.New() + var lamportBytes [8]byte + binary.LittleEndian.PutUint64(lamportBytes[:], acct.Lamports) + _, _ = hasher.Write(lamportBytes[:]) + _, _ = hasher.Write(acct.Data) + if acct.Executable { + _, _ = hasher.Write([]byte{1}) + } else { + _, _ = hasher.Write([]byte{0}) + } + _, _ = hasher.Write(acct.Owner[:]) + _, _ = hasher.Write(acct.Key[:]) + output := make([]byte, ltHashBytes) + _, _ = hasher.Digest().Read(output) + return output +} + +func legacyAccountsEqual(a, b *accounts.Account) bool { + return a.Lamports == b.Lamports && + a.Executable == b.Executable && + a.RentEpoch == b.RentEpoch && + a.Owner == b.Owner && + bytes.Equal(a.Data, b.Data) +} + +func legacySingleDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAcct *accounts.Account) *lthash.LtHash { + previousAcct, err := slotCtx.GetParentAccount(modifiedAcct.Key) + if err != nil { + panic(fmt.Sprintf("couldn't find parent acct for %s for slot %d", modifiedAcct.Key, slotCtx.Slot)) + } + + var delta lthash.LtHash + if previousAcct.Lamports != 0 { + if legacyAccountsEqual(modifiedAcct, previousAcct) { + return &delta + } + var oldHash lthash.LtHash + oldHash.InitWithHash(legacyAccountHashBytes(previousAcct)) + delta.Sub(&oldHash) + } + + if modifiedAcct.Lamports != 0 { + var newHash lthash.LtHash + newHash.InitWithHash(legacyAccountHashBytes(modifiedAcct)) + delta.Add(&newHash) + } + return &delta +} + +func legacyCalculateDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account) *lthash.LtHash { + modifiedAccts = legacyDedupeModifiedAccts(modifiedAccts) + if len(modifiedAccts) == 0 { + return <hash.LtHash{} + } + + numWorkers := min(32, len(modifiedAccts)) + chunkSize := (len(modifiedAccts) + numWorkers - 1) / numWorkers + perAccount := make([]*lthash.LtHash, len(modifiedAccts)) + var wg sync.WaitGroup + for workerID := range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + start := workerID * chunkSize + end := min(start+chunkSize, len(modifiedAccts)) + for i := start; i < end; i++ { + perAccount[i] = legacySingleDeltaLtHash(slotCtx, modifiedAccts[i]) + } + }() + } + wg.Wait() + + var result lthash.LtHash + for _, accountDelta := range perAccount { + result.Add(accountDelta) + } + return &result +} + +func TestCalculateDeltaLtHashMatchesLegacyReference(t *testing.T) { + for _, size := range []int{0, 1, 2, 31, 32, 33, 257, 1025} { + t.Run(fmt.Sprintf("accounts_%d", size), func(t *testing.T) { + ctx, modified := buildLtHashDifferentialFixture(t, size) + want := legacyCalculateDeltaLtHash(ctx, modified) + got := calculateDeltaLtHash(ctx, modified) + if !bytes.Equal(got.Hash(), want.Hash()) { + t.Fatalf("optimized delta differs byte-for-byte from legacy reference for %d accounts", size) + } + }) + } +} + +func TestCalculateDeltaLtHashFastPathMetrics(t *testing.T) { + parent := accounts.NewMemAccounts() + modified := make([]*accounts.Account, 0, 8) + add := func(index, oldLen, newLen int, oldLamports, newLamports uint64) (*accounts.Account, *accounts.Account) { + key := ltHashFixtureKey(index) + oldAcct := ltHashFixtureAccount(key, index+31, oldLamports, oldLen) + newAcct := ltHashFixtureAccount(key, index+71, newLamports, newLen) + if err := parent.SetAccountWithoutLock(key, oldAcct); err != nil { + t.Fatalf("seed account %d: %v", index, err) + } + return oldAcct, newAcct + } + + oldUnchanged, _ := add(0, 2, 2, 10, 10) + modified = append(modified, oldUnchanged.Clone()) + oldRent, _ := add(1, 3, 3, 20, 20) + rentOnly := oldRent.Clone() + rentOnly.RentEpoch++ + modified = append(modified, rentOnly) + _, created := add(2, 4, 5, 0, 30) + modified = append(modified, created) + oldDeleted, _ := add(3, 7, 1, 40, 0) + deleted := oldDeleted.Clone() + deleted.Lamports = 0 + modified = append(modified, deleted) + oldUpdated, newest := add(4, 11, 13, 50, 60) + stale := newest.Clone() + stale.Lamports++ + modified = append(modified, stale, nil, newest) + oldZero, _ := add(5, 17, 19, 0, 0) + zeroToZero := oldZero.Clone() + zeroToZero.Data = append(zeroToZero.Data, 0xff) + modified = append(modified, zeroToZero) + + metrics.GlobalBlockReplay = metrics.BlockReplay{} + t.Cleanup(func() { metrics.GlobalBlockReplay = metrics.BlockReplay{} }) + ctx := &sealevel.SlotCtx{Slot: 55, ParentAccts: parent, AcctsLtHash: <hash.LtHash{}, Replay: true} + got := calculateDeltaLtHash(ctx, modified) + want := legacyCalculateDeltaLtHash(ctx, modified) + if !got.Equals(want) { + t.Fatal("fast-path fixture differs from the legacy delta") + } + + replayMetrics := metrics.GlobalBlockReplay + if replayMetrics.LtHashInputAccounts != 8 || replayMetrics.LtHashUniqueAccounts != 6 { + t.Fatalf("input/unique metrics = %d/%d, want 8/6", replayMetrics.LtHashInputAccounts, replayMetrics.LtHashUniqueAccounts) + } + if replayMetrics.LtHashUnchangedAccounts != 3 || replayMetrics.LtHashCreatedAccounts != 1 || replayMetrics.LtHashDeletedAccounts != 1 { + t.Fatalf("unchanged/created/deleted metrics = %d/%d/%d, want 3/1/1", + replayMetrics.LtHashUnchangedAccounts, replayMetrics.LtHashCreatedAccounts, replayMetrics.LtHashDeletedAccounts) + } + if replayMetrics.LtHashOldDataBytes != uint64(len(oldDeleted.Data)+len(oldUpdated.Data)) { + t.Fatalf("old hashed data bytes = %d, want %d", replayMetrics.LtHashOldDataBytes, len(oldDeleted.Data)+len(oldUpdated.Data)) + } + if replayMetrics.LtHashNewDataBytes != uint64(len(created.Data)+len(newest.Data)) { + t.Fatalf("new hashed data bytes = %d, want %d", replayMetrics.LtHashNewDataBytes, len(created.Data)+len(newest.Data)) + } + if replayMetrics.LtHashDedupe.Count != 1 || replayMetrics.LtHashWorkerCompute.Count != 1 || replayMetrics.LtHashPartialReduce.Count != 1 { + t.Fatalf("phase timing counts = %d/%d/%d, want 1/1/1", + replayMetrics.LtHashDedupe.Count, replayMetrics.LtHashWorkerCompute.Count, replayMetrics.LtHashPartialReduce.Count) + } +} + +func TestCalculateDeltaLtHashProducerDoesNotWriteReplayMetrics(t *testing.T) { + ctx, modified := buildLtHashDifferentialFixture(t, 64) + ctx.Replay = false + metrics.GlobalBlockReplay = metrics.BlockReplay{} + t.Cleanup(func() { metrics.GlobalBlockReplay = metrics.BlockReplay{} }) + + calculateDeltaLtHash(ctx, modified) + if metrics.GlobalBlockReplay.LtHashDedupe.Count != 0 || + metrics.GlobalBlockReplay.LtHashWorkerCompute.Count != 0 || + metrics.GlobalBlockReplay.LtHashPartialReduce.Count != 0 || + metrics.GlobalBlockReplay.LtHashInputAccounts != 0 { + t.Fatal("leader-production LtHash calculation wrote replay metrics") + } +} + +func TestBankHashPhaseMetricsAreReplayOnly(t *testing.T) { + featureSet := features.NewFeaturesDefault() + ctx := &sealevel.SlotCtx{Features: featureSet} + metrics.GlobalBlockReplay = metrics.BlockReplay{} + t.Cleanup(func() { metrics.GlobalBlockReplay = metrics.BlockReplay{} }) + + CalculateBankHash(ctx, nil, nil, [32]byte{}, 0, [32]byte{}) + if metrics.GlobalBlockReplay.BankHashFinalize.Count != 0 { + t.Fatal("leader-production finalization wrote replay metrics") + } + if metrics.GlobalBlockReplay.AccountsDeltaHash.Count != 0 { + t.Fatal("leader-production accounts delta hash wrote replay metrics") + } + + ctx.Replay = true + CalculateBankHash(ctx, nil, nil, [32]byte{}, 0, [32]byte{}) + if metrics.GlobalBlockReplay.BankHashFinalize.Count != 1 { + t.Fatalf("replay finalization timing count = %d, want 1", metrics.GlobalBlockReplay.BankHashFinalize.Count) + } + if metrics.GlobalBlockReplay.AccountsDeltaHash.Count != 1 { + t.Fatalf("replay accounts delta hash timing count = %d, want 1", metrics.GlobalBlockReplay.AccountsDeltaHash.Count) + } +} + +func TestAccumulateSingleDeltaLtHashMissingParentPanics(t *testing.T) { + ctx := &sealevel.SlotCtx{Slot: 77, ParentAccts: accounts.NewMemAccounts()} + modified := ltHashFixtureAccount(ltHashFixtureKey(9001), 1, 10, 4) + var delta, scratch lthash.LtHash + var hasher lthash.AccountHasher + var stats ltHashWorkerStats + + defer func() { + if recover() == nil { + t.Fatal("missing parent account did not retain the legacy panic behavior") + } + }() + accumulateSingleDeltaLtHash(ctx, modified, &delta, &scratch, &hasher, &stats) +} + +func TestDedupeModifiedAcctsDropsSingleNil(t *testing.T) { + if got := dedupeModifiedAccts([]*accounts.Account{nil}); got != nil { + t.Fatalf("single nil account was not dropped: %#v", got) + } +} + +func buildLtHashBenchmarkFixture(b *testing.B, accountCount, dataLen int) (*sealevel.SlotCtx, []*accounts.Account) { + b.Helper() + parent := accounts.NewMemAccounts() + modified := make([]*accounts.Account, accountCount) + for i := range accountCount { + key := ltHashFixtureKey(i) + oldAcct := ltHashFixtureAccount(key, i+101, uint64(i+1), dataLen) + newAcct := oldAcct.Clone() + newAcct.Lamports += 1_000_000 + newAcct.Data[len(newAcct.Data)-1] ^= 0xff + if err := parent.SetAccountWithoutLock(key, oldAcct); err != nil { + b.Fatalf("seed benchmark account %d: %v", i, err) + } + modified[i] = newAcct + } + return &sealevel.SlotCtx{Slot: 99, ParentAccts: parent, AcctsLtHash: <hash.LtHash{}}, modified +} + +var benchmarkDeltaLtHashByte byte + +func benchmarkCalculateDeltaLtHash(b *testing.B, accountCount int) { + ctx, modified := buildLtHashBenchmarkFixture(b, accountCount, 128) + b.Run("worker_partials", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + result := calculateDeltaLtHash(ctx, modified) + benchmarkDeltaLtHashByte = result.Hash()[0] + } + }) + b.Run("legacy_per_account", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + result := legacyCalculateDeltaLtHash(ctx, modified) + benchmarkDeltaLtHashByte = result.Hash()[0] + } + }) +} + +func BenchmarkCalculateDeltaLtHash8192(b *testing.B) { + benchmarkCalculateDeltaLtHash(b, 8192) +} + +func BenchmarkCalculateDeltaLtHash30000(b *testing.B) { + benchmarkCalculateDeltaLtHash(b, 30_000) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 446212bd..3cccd2af 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,7 +74,8 @@ type DevelopmentConfig struct { BorrowedAccountArenaSize uint64 `toml:"borrowed_account_arena_size" mapstructure:"borrowed_account_arena_size"` // was: borrowed-account-arena-size UsePool bool `toml:"use_pool" mapstructure:"use_pool"` // was: use-pool Pprof PprofConfig `toml:"pprof" mapstructure:"pprof"` - ProgramCacheMaxMB int `toml:"program_cache_max_mb" mapstructure:"program_cache_max_mb"` // Approximate SBPF program cache size in MiB + ProgramCacheMaxMB int `toml:"program_cache_max_mb" mapstructure:"program_cache_max_mb"` // Approximate SBPF program cache size in MiB + CommonAccountCacheMaxMB int `toml:"common_account_cache_max_mb" mapstructure:"common_account_cache_max_mb"` // Retained decoded account cache budget in MiB Debug DebugConfig `toml:"debug" mapstructure:"debug"` } diff --git a/pkg/epochstakes/epoch_stakes.go b/pkg/epochstakes/epoch_stakes.go index 81f51159..6a97728c 100644 --- a/pkg/epochstakes/epoch_stakes.go +++ b/pkg/epochstakes/epoch_stakes.go @@ -3,17 +3,22 @@ package epochstakes import ( "encoding/json" "fmt" + "sync" + "sync/atomic" "github.com/gagliardetto/solana-go" ) type EpochStakesCache struct { - pubkeys []solana.PublicKey + mu sync.RWMutex stakeCache map[uint64]map[solana.PublicKey]uint64 voteAcctCache map[uint64]map[solana.PublicKey]*VoteAccount totalStakeCache map[uint64]uint64 + generations map[uint64]uint64 } +var nextEpochStakesGeneration uint64 + type VoteAccount struct { Lamports uint64 NodePubkey solana.PublicKey @@ -25,49 +30,140 @@ type VoteAccount struct { RentEpoch uint64 } +// Snapshot is one immutable, atomically published view of an epoch's stake +// material. Its maps and vote-account records must be treated as read-only. +type Snapshot struct { + Epoch uint64 + Generation uint64 + Stakes map[solana.PublicKey]uint64 + VoteAccounts map[solana.PublicKey]*VoteAccount + TotalStake uint64 +} + func NewEpochStakesCache() *EpochStakesCache { - return &EpochStakesCache{stakeCache: make(map[uint64]map[solana.PublicKey]uint64), + return &EpochStakesCache{ + stakeCache: make(map[uint64]map[solana.PublicKey]uint64), voteAcctCache: make(map[uint64]map[solana.PublicKey]*VoteAccount), - totalStakeCache: make(map[uint64]uint64)} + totalStakeCache: make(map[uint64]uint64), + generations: make(map[uint64]uint64), + } } -func (cache *EpochStakesCache) PutEntry(epoch uint64, pubkey solana.PublicKey, stake uint64, voteAcct *VoteAccount) { - _, exists := cache.stakeCache[epoch] +// PutEpoch atomically replaces every piece of material for epoch. The inputs +// are cloned before publication so callers cannot mutate the installed view. +func (cache *EpochStakesCache) PutEpoch( + epoch uint64, + stakes map[solana.PublicKey]uint64, + voteAccounts map[solana.PublicKey]*VoteAccount, + totalStake uint64, +) uint64 { + return cache.putEpochOwned(epoch, cloneStakes(stakes), cloneVoteAccounts(voteAccounts), totalStake) +} + +// Snapshot returns an internally coherent, allocation-free view of epoch. Its +// fields are cache-owned and must be treated as immutable. PutEpoch clones its +// inputs and compatibility mutators use copy-on-write, so a published Snapshot +// remains unchanged across later cache updates. +func (cache *EpochStakesCache) Snapshot(epoch uint64) (Snapshot, bool) { + cache.mu.RLock() + defer cache.mu.RUnlock() + + stakes, exists := cache.stakeCache[epoch] if !exists { - cache.stakeCache[epoch] = make(map[solana.PublicKey]uint64) - cache.voteAcctCache[epoch] = make(map[solana.PublicKey]*VoteAccount) + return Snapshot{Epoch: epoch, Generation: cache.generations[epoch]}, false + } + return Snapshot{ + Epoch: epoch, + Generation: cache.generations[epoch], + Stakes: stakes, + VoteAccounts: cache.voteAcctCache[epoch], + TotalStake: cache.totalStakeCache[epoch], + }, true +} + +// PutEntry is retained for compatibility. It uses copy-on-write so readers can +// never observe a map while it is being modified. +func (cache *EpochStakesCache) PutEntry(epoch uint64, pubkey solana.PublicKey, stake uint64, voteAcct *VoteAccount) { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.ensureMapsLocked() + + stakes := cloneStakes(cache.stakeCache[epoch]) + if stakes == nil { + stakes = make(map[solana.PublicKey]uint64) } - cache.stakeCache[epoch][pubkey] = stake - cache.voteAcctCache[epoch][pubkey] = voteAcct + voteAccounts := cloneVoteAccounts(cache.voteAcctCache[epoch]) + if voteAccounts == nil { + voteAccounts = make(map[solana.PublicKey]*VoteAccount) + } + stakes[pubkey] = stake + voteAccounts[pubkey] = cloneVoteAccount(voteAcct) + cache.stakeCache[epoch] = stakes + cache.voteAcctCache[epoch] = voteAccounts + cache.bumpGenerationLocked(epoch) } func (cache *EpochStakesCache) PutTotalEpochStake(epoch uint64, totalStake uint64) { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.ensureMapsLocked() cache.totalStakeCache[epoch] = totalStake + cache.bumpGenerationLocked(epoch) } func (cache *EpochStakesCache) EpochStakes(epoch uint64) map[solana.PublicKey]uint64 { - return cache.stakeCache[epoch] + cache.mu.RLock() + defer cache.mu.RUnlock() + return cloneStakes(cache.stakeCache[epoch]) } func (cache *EpochStakesCache) HasEpochStakes(epoch uint64) bool { + cache.mu.RLock() + defer cache.mu.RUnlock() _, exists := cache.stakeCache[epoch] return exists } func (cache *EpochStakesCache) EpochStakesAccts(epoch uint64) map[solana.PublicKey]*VoteAccount { - return cache.voteAcctCache[epoch] + cache.mu.RLock() + defer cache.mu.RUnlock() + return cloneVoteAccounts(cache.voteAcctCache[epoch]) } func (cache *EpochStakesCache) TotalStake(epoch uint64) uint64 { + cache.mu.RLock() + defer cache.mu.RUnlock() return cache.totalStakeCache[epoch] } +// Generation identifies the exact epoch-stakes material currently installed. +// It changes on every mutation so derived verifier caches cannot survive a +// same-epoch reload with stale validator keys or stakes. +func (cache *EpochStakesCache) Generation(epoch uint64) uint64 { + cache.mu.RLock() + defer cache.mu.RUnlock() + return cache.generations[epoch] +} + +func (cache *EpochStakesCache) bumpGenerationLocked(epoch uint64) uint64 { + generation := atomic.AddUint64(&nextEpochStakesGeneration, 1) + if generation == 0 { + generation = atomic.AddUint64(&nextEpochStakesGeneration, 1) + } + cache.generations[epoch] = generation + return generation +} + // ClearEpochStakes removes all stakes for a specific epoch. // Used on resume to force rebuild from AccountsDB. func (cache *EpochStakesCache) ClearEpochStakes(epoch uint64) { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.ensureMapsLocked() delete(cache.stakeCache, epoch) delete(cache.voteAcctCache, epoch) delete(cache.totalStakeCache, epoch) + cache.bumpGenerationLocked(epoch) } // PersistedEpochStakes is the JSON-serializable format for epoch stakes. @@ -92,26 +188,23 @@ type VoteAccountJSON struct { // SerializeEpoch serializes the stakes for a single epoch to JSON. func (cache *EpochStakesCache) SerializeEpoch(epoch uint64) ([]byte, error) { - stakes := cache.stakeCache[epoch] - voteAccts := cache.voteAcctCache[epoch] - totalStake := cache.totalStakeCache[epoch] - - if stakes == nil { + snapshot, exists := cache.Snapshot(epoch) + if !exists { return nil, fmt.Errorf("no stakes for epoch %d", epoch) } persisted := PersistedEpochStakes{ Epoch: epoch, - TotalStake: totalStake, - Stakes: make(map[string]uint64, len(stakes)), - VoteAccts: make(map[string]*VoteAccountJSON, len(voteAccts)), + TotalStake: snapshot.TotalStake, + Stakes: make(map[string]uint64, len(snapshot.Stakes)), + VoteAccts: make(map[string]*VoteAccountJSON, len(snapshot.VoteAccounts)), } - for pk, stake := range stakes { + for pk, stake := range snapshot.Stakes { persisted.Stakes[pk.String()] = stake } - for pk, va := range voteAccts { + for pk, va := range snapshot.VoteAccounts { if va != nil { var bls []byte if va.BlsPubkeyCompressed != nil { @@ -143,20 +236,23 @@ func (cache *EpochStakesCache) DeserializeAndLoadEpoch(data []byte) (uint64, err epoch := persisted.Epoch - // Initialize maps for this epoch - cache.stakeCache[epoch] = make(map[solana.PublicKey]uint64, len(persisted.Stakes)) - cache.voteAcctCache[epoch] = make(map[solana.PublicKey]*VoteAccount, len(persisted.VoteAccts)) - cache.totalStakeCache[epoch] = persisted.TotalStake + // Decode privately and publish only after every key and account validates. A + // failed reload must not mutate material behind an unchanged generation. + stakes := make(map[solana.PublicKey]uint64, len(persisted.Stakes)) + voteAccounts := make(map[solana.PublicKey]*VoteAccount, len(persisted.VoteAccts)) for pkStr, stake := range persisted.Stakes { pk, err := solana.PublicKeyFromBase58(pkStr) if err != nil { return 0, fmt.Errorf("invalid stake pubkey %q for epoch %d: %w", pkStr, epoch, err) } - cache.stakeCache[epoch][pk] = stake + stakes[pk] = stake } for pkStr, vaJSON := range persisted.VoteAccts { + if vaJSON == nil { + return 0, fmt.Errorf("nil vote account metadata for vote acct %s epoch %d", pkStr, epoch) + } pk, err := solana.PublicKeyFromBase58(pkStr) if err != nil { return 0, fmt.Errorf("invalid vote acct pubkey %q for epoch %d: %w", pkStr, epoch, err) @@ -178,7 +274,7 @@ func (cache *EpochStakesCache) DeserializeAndLoadEpoch(data []byte) (uint64, err copy(b[:], vaJSON.BlsPubkeyCompressed) bls = &b } - cache.voteAcctCache[epoch][pk] = &VoteAccount{ + voteAccounts[pk] = &VoteAccount{ Lamports: vaJSON.Lamports, NodePubkey: nodePubkey, BlsPubkeyCompressed: bls, @@ -190,14 +286,83 @@ func (cache *EpochStakesCache) DeserializeAndLoadEpoch(data []byte) (uint64, err } } + cache.putEpochOwned(epoch, stakes, voteAccounts, persisted.TotalStake) return epoch, nil } // GetAllEpochs returns a list of all epochs in the cache. func (cache *EpochStakesCache) GetAllEpochs() []uint64 { + cache.mu.RLock() + defer cache.mu.RUnlock() epochs := make([]uint64, 0, len(cache.stakeCache)) for epoch := range cache.stakeCache { epochs = append(epochs, epoch) } return epochs } + +// putEpochOwned publishes maps already owned by the cache in one critical +// section. Callers must not retain or mutate the maps after this call. +func (cache *EpochStakesCache) putEpochOwned( + epoch uint64, + stakes map[solana.PublicKey]uint64, + voteAccounts map[solana.PublicKey]*VoteAccount, + totalStake uint64, +) uint64 { + cache.mu.Lock() + defer cache.mu.Unlock() + cache.ensureMapsLocked() + cache.stakeCache[epoch] = stakes + cache.voteAcctCache[epoch] = voteAccounts + cache.totalStakeCache[epoch] = totalStake + return cache.bumpGenerationLocked(epoch) +} + +func (cache *EpochStakesCache) ensureMapsLocked() { + if cache.stakeCache == nil { + cache.stakeCache = make(map[uint64]map[solana.PublicKey]uint64) + } + if cache.voteAcctCache == nil { + cache.voteAcctCache = make(map[uint64]map[solana.PublicKey]*VoteAccount) + } + if cache.totalStakeCache == nil { + cache.totalStakeCache = make(map[uint64]uint64) + } + if cache.generations == nil { + cache.generations = make(map[uint64]uint64) + } +} + +func cloneStakes(stakes map[solana.PublicKey]uint64) map[solana.PublicKey]uint64 { + if stakes == nil { + return nil + } + cloned := make(map[solana.PublicKey]uint64, len(stakes)) + for pubkey, stake := range stakes { + cloned[pubkey] = stake + } + return cloned +} + +func cloneVoteAccounts(voteAccounts map[solana.PublicKey]*VoteAccount) map[solana.PublicKey]*VoteAccount { + if voteAccounts == nil { + return nil + } + cloned := make(map[solana.PublicKey]*VoteAccount, len(voteAccounts)) + for pubkey, voteAccount := range voteAccounts { + cloned[pubkey] = cloneVoteAccount(voteAccount) + } + return cloned +} + +func cloneVoteAccount(voteAccount *VoteAccount) *VoteAccount { + if voteAccount == nil { + return nil + } + cloned := *voteAccount + if voteAccount.BlsPubkeyCompressed != nil { + bls := *voteAccount.BlsPubkeyCompressed + cloned.BlsPubkeyCompressed = &bls + } + return &cloned +} diff --git a/pkg/epochstakes/epoch_stakes_generation_test.go b/pkg/epochstakes/epoch_stakes_generation_test.go new file mode 100644 index 00000000..92a08d33 --- /dev/null +++ b/pkg/epochstakes/epoch_stakes_generation_test.go @@ -0,0 +1,256 @@ +package epochstakes + +import ( + "encoding/json" + "fmt" + "sync" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEpochStakesGenerationTracksMaterialChanges(t *testing.T) { + const epoch = uint64(17) + cache := NewEpochStakesCache() + assert.Zero(t, cache.Generation(epoch)) + + var vote solana.PublicKey + vote[0] = 1 + cache.PutEntry(epoch, vote, 100, nil) + afterEntry := cache.Generation(epoch) + require.NotZero(t, afterEntry) + + cache.PutTotalEpochStake(epoch, 100) + afterTotal := cache.Generation(epoch) + assert.NotEqual(t, afterEntry, afterTotal) + + cache.ClearEpochStakes(epoch) + afterClear := cache.Generation(epoch) + assert.NotEqual(t, afterTotal, afterClear) + + data, err := json.Marshal(PersistedEpochStakes{ + Epoch: epoch, + TotalStake: 0, + Stakes: map[string]uint64{}, + VoteAccts: map[string]*VoteAccountJSON{}, + }) + require.NoError(t, err) + loadedEpoch, err := cache.DeserializeAndLoadEpoch(data) + require.NoError(t, err) + assert.Equal(t, epoch, loadedEpoch) + assert.NotEqual(t, afterClear, cache.Generation(epoch)) + + beforeInvalidReload := cache.Generation(epoch) + invalidData, err := json.Marshal(PersistedEpochStakes{ + Epoch: epoch, + TotalStake: 1, + Stakes: map[string]uint64{"not-a-pubkey": 1}, + VoteAccts: map[string]*VoteAccountJSON{}, + }) + require.NoError(t, err) + _, err = cache.DeserializeAndLoadEpoch(invalidData) + require.Error(t, err) + assert.Equal(t, beforeInvalidReload, cache.Generation(epoch)) + assert.Empty(t, cache.EpochStakes(epoch)) +} + +func TestEpochStakesGenerationIsUniqueAcrossCacheReplacement(t *testing.T) { + const epoch = uint64(29) + first := NewEpochStakesCache() + first.PutTotalEpochStake(epoch, 1) + + second := NewEpochStakesCache() + second.PutTotalEpochStake(epoch, 1) + + assert.NotEqual(t, first.Generation(epoch), second.Generation(epoch)) +} + +func TestEpochStakesPutEpochAndCompatibilityAccessorsAreDeeplyImmutable(t *testing.T) { + const epoch = uint64(31) + cache := NewEpochStakesCache() + + var votePubkey solana.PublicKey + votePubkey[0] = 1 + var nodePubkey solana.PublicKey + nodePubkey[0] = 2 + var owner solana.PublicKey + owner[0] = 3 + var bls [48]byte + bls[0] = 4 + voteAccount := &VoteAccount{ + Lamports: 100, + NodePubkey: nodePubkey, + BlsPubkeyCompressed: &bls, + LastTimestampTs: 5, + LastTimestampSlot: 6, + Owner: owner, + Executable: 1, + RentEpoch: 7, + } + stakes := map[solana.PublicKey]uint64{votePubkey: 100} + voteAccounts := map[solana.PublicKey]*VoteAccount{votePubkey: voteAccount} + + generation := cache.PutEpoch(epoch, stakes, voteAccounts, 100) + require.NotZero(t, generation) + + // Mutating every caller-owned layer after PutEpoch must not affect the cache. + stakes[votePubkey] = 999 + voteAccounts[votePubkey] = nil + voteAccount.Lamports = 999 + bls[0] = 99 + + snapshot, ok := cache.Snapshot(epoch) + require.True(t, ok) + assert.Equal(t, generation, snapshot.Generation) + assert.Equal(t, uint64(100), snapshot.TotalStake) + assert.Equal(t, uint64(100), snapshot.Stakes[votePubkey]) + require.NotNil(t, snapshot.VoteAccounts[votePubkey]) + assert.Equal(t, uint64(100), snapshot.VoteAccounts[votePubkey].Lamports) + require.NotNil(t, snapshot.VoteAccounts[votePubkey].BlsPubkeyCompressed) + assert.Equal(t, byte(4), snapshot.VoteAccounts[votePubkey].BlsPubkeyCompressed[0]) + + // Compatibility getters are detached deep copies. + stakesCopy := cache.EpochStakes(epoch) + voteAccountsCopy := cache.EpochStakesAccts(epoch) + stakesCopy[votePubkey] = 777 + voteAccountsCopy[votePubkey].Lamports = 777 + voteAccountsCopy[votePubkey].BlsPubkeyCompressed[0] = 77 + + unchanged, ok := cache.Snapshot(epoch) + require.True(t, ok) + assert.Equal(t, uint64(100), unchanged.Stakes[votePubkey]) + assert.Equal(t, uint64(100), unchanged.VoteAccounts[votePubkey].Lamports) + assert.Equal(t, byte(4), unchanged.VoteAccounts[votePubkey].BlsPubkeyCompressed[0]) + + // Compatibility mutation is copy-on-write, so the previously published + // snapshot remains immutable while a new generation receives the update. + var replacementBLS [48]byte + replacementBLS[0] = 5 + cache.PutEntry(epoch, votePubkey, 200, &VoteAccount{Lamports: 200, BlsPubkeyCompressed: &replacementBLS}) + assert.Equal(t, uint64(100), snapshot.Stakes[votePubkey]) + assert.Equal(t, uint64(100), snapshot.VoteAccounts[votePubkey].Lamports) + assert.Equal(t, byte(4), snapshot.VoteAccounts[votePubkey].BlsPubkeyCompressed[0]) + replaced, ok := cache.Snapshot(epoch) + require.True(t, ok) + assert.NotEqual(t, snapshot.Generation, replaced.Generation) + assert.Equal(t, uint64(200), replaced.Stakes[votePubkey]) + assert.Equal(t, uint64(200), replaced.VoteAccounts[votePubkey].Lamports) + assert.Equal(t, byte(5), replaced.VoteAccounts[votePubkey].BlsPubkeyCompressed[0]) +} + +func TestEpochStakesCompatibilityPutEntryClonesVoteAccount(t *testing.T) { + const epoch = uint64(32) + cache := NewEpochStakesCache() + var votePubkey solana.PublicKey + votePubkey[0] = 1 + var bls [48]byte + bls[0] = 2 + voteAccount := &VoteAccount{Lamports: 3, BlsPubkeyCompressed: &bls} + + cache.PutEntry(epoch, votePubkey, 4, voteAccount) + voteAccount.Lamports = 30 + bls[0] = 20 + + snapshot, ok := cache.Snapshot(epoch) + require.True(t, ok) + assert.Equal(t, uint64(4), snapshot.Stakes[votePubkey]) + assert.Equal(t, uint64(3), snapshot.VoteAccounts[votePubkey].Lamports) + assert.Equal(t, byte(2), snapshot.VoteAccounts[votePubkey].BlsPubkeyCompressed[0]) +} + +func TestEpochStakesSnapshotPublicationIsAtomic(t *testing.T) { + const ( + epoch = uint64(33) + iterations = 2_000 + readers = 4 + ) + cache := NewEpochStakesCache() + var votePubkey solana.PublicKey + votePubkey[0] = 1 + + makeMaterial := func(value uint64) (map[solana.PublicKey]uint64, map[solana.PublicKey]*VoteAccount) { + var bls [48]byte + bls[0] = byte(value) + return map[solana.PublicKey]uint64{votePubkey: value}, map[solana.PublicKey]*VoteAccount{ + votePubkey: {Lamports: value, BlsPubkeyCompressed: &bls}, + } + } + stakesA, voteAccountsA := makeMaterial(11) + stakesB, voteAccountsB := makeMaterial(22) + cache.PutEpoch(epoch, stakesA, voteAccountsA, 11) + + start := make(chan struct{}) + errs := make(chan error, readers) + var wg sync.WaitGroup + wg.Add(readers + 1) + go func() { + defer wg.Done() + <-start + for i := 0; i < iterations; i++ { + if i%2 == 0 { + cache.PutEpoch(epoch, stakesB, voteAccountsB, 22) + } else { + cache.PutEpoch(epoch, stakesA, voteAccountsA, 11) + } + } + }() + for reader := 0; reader < readers; reader++ { + go func() { + defer wg.Done() + <-start + for i := 0; i < iterations; i++ { + snapshot, ok := cache.Snapshot(epoch) + if !ok { + errs <- fmt.Errorf("epoch disappeared") + return + } + stake := snapshot.Stakes[votePubkey] + voteAccount := snapshot.VoteAccounts[votePubkey] + if voteAccount == nil || voteAccount.BlsPubkeyCompressed == nil { + errs <- fmt.Errorf("missing vote account material") + return + } + if snapshot.Generation == 0 || snapshot.TotalStake != stake || voteAccount.Lamports != stake || uint64(voteAccount.BlsPubkeyCompressed[0]) != stake { + errs <- fmt.Errorf("mixed snapshot: generation=%d stake=%d total=%d lamports=%d bls=%d", snapshot.Generation, stake, snapshot.TotalStake, voteAccount.Lamports, voteAccount.BlsPubkeyCompressed[0]) + return + } + if stake != 11 && stake != 22 { + errs <- fmt.Errorf("unexpected stake %d", stake) + return + } + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } +} + +func TestEpochStakesDeserializeNilVoteAccountIsFailureAtomic(t *testing.T) { + const epoch = uint64(34) + cache := NewEpochStakesCache() + var votePubkey solana.PublicKey + votePubkey[0] = 1 + cache.PutEpoch(epoch, map[solana.PublicKey]uint64{votePubkey: 10}, map[solana.PublicKey]*VoteAccount{votePubkey: {Lamports: 10}}, 10) + before, ok := cache.Snapshot(epoch) + require.True(t, ok) + + data, err := json.Marshal(PersistedEpochStakes{ + Epoch: epoch, + TotalStake: 20, + Stakes: map[string]uint64{votePubkey.String(): 20}, + VoteAccts: map[string]*VoteAccountJSON{votePubkey.String(): nil}, + }) + require.NoError(t, err) + _, err = cache.DeserializeAndLoadEpoch(data) + require.Error(t, err) + + after, ok := cache.Snapshot(epoch) + require.True(t, ok) + assert.Equal(t, before, after) +} diff --git a/pkg/global/global_ctx.go b/pkg/global/global_ctx.go index d51de244..a3d7cdf8 100644 --- a/pkg/global/global_ctx.go +++ b/pkg/global/global_ctx.go @@ -49,7 +49,7 @@ type GlobalCtx struct { mu sync.Mutex } -var instance GlobalCtx +var instance = GlobalCtx{epochStakes: epochstakes.NewEpochStakesCache()} func SetLatestBlockHash(blockHash [32]byte) { instance.SetLatestBlockhash(blockHash) @@ -221,14 +221,29 @@ func ClearEpochVoteStateSnapshots() { } func PutEpochStakesEntry(epoch uint64, pubkey solana.PublicKey, stake uint64, voteAcct *epochstakes.VoteAccount) { - if instance.epochStakes == nil { - instance.epochStakes = epochstakes.NewEpochStakesCache() - } instance.epochStakes.PutEntry(epoch, pubkey, stake, voteAcct) } +// PutEpochStakes atomically replaces all stake material for epoch. +func PutEpochStakes(epoch uint64, stakes map[solana.PublicKey]uint64, voteAccts map[solana.PublicKey]*epochstakes.VoteAccount, totalStake uint64) uint64 { + return instance.epochStakes.PutEpoch(epoch, stakes, voteAccts, totalStake) +} + +// EpochStakesSnapshot returns one coherent, immutable view of epoch. +func EpochStakesSnapshot(epoch uint64) (epochstakes.Snapshot, bool) { + return instance.epochStakes.Snapshot(epoch) +} + +// EpochStakes returns the immutable, cache-owned epoch view. Callers must not +// mutate it. Epoch publication is copy-on-write, so the view remains valid +// across later updates without copying hundreds of thousands of entries on +// per-slot read paths. func EpochStakes(epoch uint64) map[solana.PublicKey]uint64 { - return instance.epochStakes.EpochStakes(epoch) + snapshot, ok := instance.epochStakes.Snapshot(epoch) + if !ok { + return nil + } + return snapshot.Stakes } func HasEpochStakes(epoch uint64) bool { @@ -236,9 +251,6 @@ func HasEpochStakes(epoch uint64) bool { } func PutEpochTotalStake(epoch uint64, totalStake uint64) { - if instance.epochStakes == nil { - instance.epochStakes = epochstakes.NewEpochStakesCache() - } instance.epochStakes.PutTotalEpochStake(epoch, totalStake) } @@ -246,45 +258,47 @@ func EpochTotalStake(epoch uint64) uint64 { return instance.epochStakes.TotalStake(epoch) } +func EpochStakesGeneration(epoch uint64) uint64 { + return instance.epochStakes.Generation(epoch) +} + func StakeForVoteAcct(epoch uint64, voteAcct solana.PublicKey) uint64 { - epochStakes := instance.epochStakes.EpochStakes(epoch) - return epochStakes[voteAcct] + snapshot, ok := instance.epochStakes.Snapshot(epoch) + if !ok { + return 0 + } + return snapshot.Stakes[voteAcct] } +// EpochStakesVoteAccts follows the same immutable-view contract as +// EpochStakes. func EpochStakesVoteAccts(epoch uint64) map[solana.PublicKey]*epochstakes.VoteAccount { - return instance.epochStakes.EpochStakesAccts(epoch) + snapshot, ok := instance.epochStakes.Snapshot(epoch) + if !ok { + return nil + } + return snapshot.VoteAccounts } // ClearEpochStakes removes all stakes for a specific epoch. // Used on resume to force rebuild from AccountsDB. func ClearEpochStakes(epoch uint64) { - if instance.epochStakes != nil { - instance.epochStakes.ClearEpochStakes(epoch) - } + instance.epochStakes.ClearEpochStakes(epoch) } // SerializeEpochStakes serializes the stakes for a single epoch to JSON. func SerializeEpochStakes(epoch uint64) ([]byte, error) { - if instance.epochStakes == nil { - return nil, nil - } return instance.epochStakes.SerializeEpoch(epoch) } // DeserializeAndLoadEpochStakes deserializes and loads epoch stakes from JSON. // Returns the epoch number that was loaded. func DeserializeAndLoadEpochStakes(data []byte) (uint64, error) { - if instance.epochStakes == nil { - instance.epochStakes = epochstakes.NewEpochStakesCache() - } return instance.epochStakes.DeserializeAndLoadEpoch(data) } // GetAllCachedEpochs returns all epochs currently in the epoch stakes cache. func GetAllCachedEpochs() []uint64 { - if instance.epochStakes == nil { - return nil - } return instance.epochStakes.GetAllEpochs() } @@ -367,6 +381,25 @@ func LeaderForSlot(slot uint64) (solana.PublicKey, bool) { return solana.PublicKey{}, false } +// LeaderForSlotWithVoteAccount returns the coherent node and vote-account pair +// sampled for slot by a locally built vote-keyed schedule. Compatibility +// schedules built from node-keyed RPC data return false. +func LeaderForSlotWithVoteAccount(slot uint64) (solana.PublicKey, solana.PublicKey, bool) { + instance.leaderScheduleMutex.RLock() + defer instance.leaderScheduleMutex.RUnlock() + + if instance.leaderSchedule != nil { + return instance.leaderSchedule.LeaderForSlotWithVoteAccount(slot) + } + for _, schedule := range instance.leaderSchedules { + if _, ok := schedule.LeaderForSlot(slot); !ok { + continue + } + return schedule.LeaderForSlotWithVoteAccount(slot) + } + return solana.PublicKey{}, solana.PublicKey{}, false +} + func (globctx *GlobalCtx) SetLatestBlockhash(blockhash [32]byte) { globctx.mu.Lock() defer globctx.mu.Unlock() diff --git a/pkg/global/leader_vote_account_test.go b/pkg/global/leader_vote_account_test.go new file mode 100644 index 00000000..daba48cc --- /dev/null +++ b/pkg/global/leader_vote_account_test.go @@ -0,0 +1,89 @@ +package global + +import ( + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/leaderschedule" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func testVoteKeyedSchedule( + node solana.PublicKey, + vote solana.PublicKey, + epoch uint64, + epochSchedule *sealevel.SysvarEpochSchedule, +) *leaderschedule.LeaderSchedule { + return leaderschedule.New( + map[solana.PublicKey]*epochstakes.VoteAccount{ + vote: {NodePubkey: node}, + }, + map[solana.PublicKey]uint64{vote: 1}, + epochSchedule, + epoch, + epochSchedule.SlotsInEpoch(epoch), + 1, + ) +} + +func TestEpochLeaderVoteAccountsCoexist(t *testing.T) { + SetLeaderSchedule(nil) + t.Cleanup(func() { SetLeaderSchedule(nil) }) + + node := solana.PublicKey{9} + oldVote := solana.PublicKey{1} + newVote := solana.PublicKey{2} + epochSchedule := &sealevel.SysvarEpochSchedule{ + SlotsPerEpoch: 32, + LeaderScheduleSlotOffset: 32, + } + SetLeaderScheduleForEpoch(0, testVoteKeyedSchedule(node, oldVote, 0, epochSchedule)) + SetLeaderScheduleForEpoch(1, testVoteKeyedSchedule(node, newVote, 1, epochSchedule)) + + gotNode, gotVote, ok := LeaderForSlotWithVoteAccount(31) + require.True(t, ok) + require.Equal(t, node, gotNode) + require.Equal(t, oldVote, gotVote) + + gotNode, gotVote, ok = LeaderForSlotWithVoteAccount(32) + require.True(t, ok) + require.Equal(t, node, gotNode) + require.Equal(t, newVote, gotVote) +} + +func TestLeaderVoteAccountReplacementReturnsCoherentPair(t *testing.T) { + SetLeaderSchedule(nil) + t.Cleanup(func() { SetLeaderSchedule(nil) }) + + nodeA, voteA := solana.PublicKey{1}, solana.PublicKey{11} + nodeB, voteB := solana.PublicKey{2}, solana.PublicKey{12} + epochSchedule := &sealevel.SysvarEpochSchedule{ + SlotsPerEpoch: 32, + LeaderScheduleSlotOffset: 32, + } + scheduleA := testVoteKeyedSchedule(nodeA, voteA, 0, epochSchedule) + scheduleB := testVoteKeyedSchedule(nodeB, voteB, 0, epochSchedule) + SetLeaderScheduleForEpoch(0, scheduleA) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for range 1_000 { + SetLeaderScheduleForEpoch(0, scheduleA) + SetLeaderScheduleForEpoch(0, scheduleB) + } + }() + for range 2_000 { + node, vote, ok := LeaderForSlotWithVoteAccount(0) + require.True(t, ok) + require.True(t, + (node == nodeA && vote == voteA) || (node == nodeB && vote == voteB), + "incoherent leader pair node=%s vote=%s", node, vote, + ) + } + wg.Wait() +} diff --git a/pkg/leaderschedule/leader_schedule.go b/pkg/leaderschedule/leader_schedule.go index 07e3f187..9688d5a5 100644 --- a/pkg/leaderschedule/leader_schedule.go +++ b/pkg/leaderschedule/leader_schedule.go @@ -15,12 +15,18 @@ import ( "github.com/nixberg/chacha-rng-go" ) +type slotLeader struct { + nodePubkey solana.PublicKey + voteAccount solana.PublicKey + hasVoteAccount bool +} + type LeaderSchedule struct { - lsMap map[uint64]solana.PublicKey + lsMap map[uint64]slotLeader } func NewLeaderScheduleFromKeyedSlots(ls map[solana.PublicKey][]uint64, epochStartSlot uint64) *LeaderSchedule { - lsMap := make(map[uint64]solana.PublicKey) + lsMap := make(map[uint64]slotLeader) for pubkey, epochIndices := range ls { for _, idx := range epochIndices { @@ -28,11 +34,11 @@ func NewLeaderScheduleFromKeyedSlots(ls map[solana.PublicKey][]uint64, epochStar if err != nil { panic(fmt.Sprintf("overflow for %s, idx %d, epochStartSlot = %d", pubkey, idx, epochStartSlot)) } - existingPubkey, exists := lsMap[slot] + existingLeader, exists := lsMap[slot] if exists { - panic(fmt.Sprintf("error adding %s as leader for slot %d - there's already an entry for %s", pubkey, slot, existingPubkey)) + panic(fmt.Sprintf("error adding %s as leader for slot %d - there's already an entry for %s", pubkey, slot, existingLeader.nodePubkey)) } - lsMap[slot] = pubkey + lsMap[slot] = slotLeader{nodePubkey: pubkey} } } @@ -104,16 +110,24 @@ func New( voteToNode[va.voteAcct] = va.nodePubkey } - // Convert vote account leaders → node identity leaders - nodeLeaders := make([]solana.PublicKey, len(voteAccountLeaders)) + // Preserve both sides of the vote-keyed selection. The node identity remains + // the public leader used for shred verification, while the selected vote + // account is needed by Alpenglow to credit the leader reward deterministically + // when multiple vote accounts share one node identity. + leaderScheduleMap := make(map[uint64]slotLeader, len(voteAccountLeaders)) + firstSlotInEpoch := epochSchedule.FirstSlotInEpoch(epoch) for i, voteAcctPk := range voteAccountLeaders { - nodeLeaders[i] = voteToNode[voteAcctPk] + slotNum := uint64(i) + firstSlotInEpoch + leaderScheduleMap[slotNum] = slotLeader{ + nodePubkey: voteToNode[voteAcctPk], + voteAccount: voteAcctPk, + hasVoteAccount: true, + } } - firstSlotInEpoch := epochSchedule.FirstSlotInEpoch(epoch) - leaderSchedule := newFromLeadersDirect(nodeLeaders, firstSlotInEpoch) - - return leaderSchedule + return &LeaderSchedule{ + lsMap: leaderScheduleMap, + } } func stakeWeightedSlotLeaders(keyedStakes []pubkeyAndStakePair, @@ -209,17 +223,6 @@ func uint64n(rng *chacha.ChaCha, n uint64) uint64 { } } -// newFromLeadersDirect creates a LeaderSchedule from node identity pubkeys directly. -// Used when leaders are already node identities (after aggregating by node). -func newFromLeadersDirect(nodeLeaders []solana.PublicKey, firstSlotInEpoch uint64) *LeaderSchedule { - leaderScheduleMap := make(map[uint64]solana.PublicKey, len(nodeLeaders)) - for i, leader := range nodeLeaders { - slotNum := uint64(i) + firstSlotInEpoch - leaderScheduleMap[slotNum] = leader - } - return &LeaderSchedule{lsMap: leaderScheduleMap} -} - func sortStakes(stakes []pubkeyAndStakePair) []pubkeyAndStakePair { slices.SortFunc(stakes, func(l, r pubkeyAndStakePair) int { if r.stake != l.stake { @@ -328,5 +331,19 @@ func GetSortedStakesDebug( func (ls *LeaderSchedule) LeaderForSlot(slot uint64) (solana.PublicKey, bool) { leader, exists := ls.lsMap[slot] - return leader, exists + if !exists { + return solana.PublicKey{}, false + } + return leader.nodePubkey, true +} + +// LeaderForSlotWithVoteAccount returns the coherent node and vote-account pair +// selected by the vote-keyed schedule. Schedules constructed from node-keyed +// RPC data do not have the vote account and return false. +func (ls *LeaderSchedule) LeaderForSlotWithVoteAccount(slot uint64) (solana.PublicKey, solana.PublicKey, bool) { + leader, exists := ls.lsMap[slot] + if !exists || !leader.hasVoteAccount { + return solana.PublicKey{}, solana.PublicKey{}, false + } + return leader.nodePubkey, leader.voteAccount, true } diff --git a/pkg/leaderschedule/leader_vote_account_test.go b/pkg/leaderschedule/leader_vote_account_test.go new file mode 100644 index 00000000..334e1dce --- /dev/null +++ b/pkg/leaderschedule/leader_vote_account_test.go @@ -0,0 +1,73 @@ +package leaderschedule + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestVoteKeyedSchedulePreservesSelectedVoteAccount(t *testing.T) { + const ( + epoch = uint64(0) + length = uint64(16) + repeat = uint64(4) + ) + node := solana.PublicKey{9} + voteA := solana.PublicKey{1} + voteB := solana.PublicKey{2} + voteAccounts := map[solana.PublicKey]*epochstakes.VoteAccount{ + voteA: {NodePubkey: node}, + voteB: {NodePubkey: node}, + } + stakes := map[solana.PublicKey]uint64{ + voteA: 10, + voteB: 20, + } + epochSchedule := &sealevel.SysvarEpochSchedule{ + SlotsPerEpoch: 32, + LeaderScheduleSlotOffset: 32, + } + + schedule := New(voteAccounts, stakes, epochSchedule, epoch, length, repeat) + expectedVotes := stakeWeightedSlotLeaders( + []pubkeyAndStakePair{ + {pubkey: voteA, stake: stakes[voteA]}, + {pubkey: voteB, stake: stakes[voteB]}, + }, + epoch, + length, + repeat, + ) + + firstSlot := epochSchedule.FirstSlotInEpoch(epoch) + for i, expectedVote := range expectedVotes { + slot := firstSlot + uint64(i) + gotNode, gotVote, ok := schedule.LeaderForSlotWithVoteAccount(slot) + require.True(t, ok, "slot %d missing vote-keyed leader metadata", slot) + require.Equal(t, node, gotNode) + require.Equal(t, expectedVote, gotVote) + if i%int(repeat) != 0 { + require.Equal(t, expectedVotes[i-1], gotVote, "repeat group changed within slot %d", slot) + } + } +} + +func TestNodeKeyedScheduleHasNoVoteAccountProvenance(t *testing.T) { + node := solana.PublicKey{9} + schedule := NewLeaderScheduleFromKeyedSlots( + map[solana.PublicKey][]uint64{node: {0}}, + 100, + ) + + gotNode, ok := schedule.LeaderForSlot(100) + require.True(t, ok) + require.Equal(t, node, gotNode) + + _, _, ok = schedule.LeaderForSlotWithVoteAccount(100) + require.False(t, ok) + _, _, ok = schedule.LeaderForSlotWithVoteAccount(101) + require.False(t, ok) +} diff --git a/pkg/lthash/account_hasher_test.go b/pkg/lthash/account_hasher_test.go new file mode 100644 index 00000000..8338f754 --- /dev/null +++ b/pkg/lthash/account_hasher_test.go @@ -0,0 +1,141 @@ +package lthash + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/gagliardetto/solana-go" + "github.com/zeebo/blake3" +) + +func referenceAccountHash(acct *accounts.Account) []byte { + output := make([]byte, numElements*2) + if acct.Lamports == 0 { + return output + } + + hasher := blake3.New() + var lamportBytes [8]byte + binary.LittleEndian.PutUint64(lamportBytes[:], acct.Lamports) + _, _ = hasher.Write(lamportBytes[:]) + _, _ = hasher.Write(acct.Data) + if acct.Executable { + _, _ = hasher.Write([]byte{1}) + } else { + _, _ = hasher.Write([]byte{0}) + } + _, _ = hasher.Write(acct.Owner[:]) + _, _ = hasher.Write(acct.Key[:]) + _, _ = hasher.Digest().Read(output) + return output +} + +func accountHashTestAccount(seed byte, dataLen int) *accounts.Account { + var key solana.PublicKey + var owner [32]byte + for i := range key { + key[i] = seed + byte(i*3) + owner[i] = seed ^ byte(i*7) + } + data := make([]byte, dataLen) + for i := range data { + data[i] = seed + byte(i*11) + } + return &accounts.Account{ + Key: key, + Lamports: uint64(seed)*1_000_003 + 17, + Data: data, + Owner: owner, + Executable: seed%2 != 0, + RentEpoch: uint64(seed) * 19, + } +} + +func TestAccountHasherMatchesReferenceAcrossReuse(t *testing.T) { + accountsToHash := []*accounts.Account{ + accountHashTestAccount(1, 0), + accountHashTestAccount(2, 1), + accountHashTestAccount(3, 63), + accountHashTestAccount(4, 1024), + accountHashTestAccount(5, 4097), + } + + var hasher AccountHasher + var got LtHash + for i, acct := range accountsToHash { + hasher.HashInto(&got, acct) + if want := referenceAccountHash(acct); !bytes.Equal(got.Hash(), want) { + t.Fatalf("account %d differs from independent BLAKE3-XOF reference", i) + } + } + + // Reusing both objects must reset all hasher and destination state. + hasher.HashInto(&got, accountsToHash[0]) + if want := referenceAccountHash(accountsToHash[0]); !bytes.Equal(got.Hash(), want) { + t.Fatal("reused account hasher retained state from a previous account") + } +} + +func TestAccountHasherZeroLamportsClearsReusedDestination(t *testing.T) { + var hasher AccountHasher + var got LtHash + hasher.HashInto(&got, accountHashTestAccount(9, 128)) + + zero := accountHashTestAccount(10, 128) + zero.Lamports = 0 + hasher.HashInto(&got, zero) + if !bytes.Equal(got.Hash(), make([]byte, numElements*2)) { + t.Fatal("zero-lamport account did not clear the reused destination") + } + + // InitWithAcct has the same in-place contract and must return the receiver. + got.InitWithHash(bytes.Repeat([]byte{0xff}, numElements*2)) + if returned := got.InitWithAcct(zero); returned != &got { + t.Fatal("InitWithAcct returned a different LtHash for a zero-lamport account") + } + if !bytes.Equal(got.Hash(), make([]byte, numElements*2)) { + t.Fatal("InitWithAcct did not clear its receiver for a zero-lamport account") + } +} + +func TestAccountHasherRentEpochIsNotHashed(t *testing.T) { + first := accountHashTestAccount(13, 96) + second := first.Clone() + second.RentEpoch++ + + var hasher AccountHasher + var firstHash, secondHash LtHash + hasher.HashInto(&firstHash, first) + hasher.HashInto(&secondHash, second) + if !firstHash.Equals(&secondHash) { + t.Fatal("rent epoch unexpectedly changed the accounts LtHash contribution") + } +} + +var benchmarkAccountHashByte byte +var benchmarkReferenceAccountHash []byte + +func BenchmarkAccountHasherHashInto(b *testing.B) { + acct := accountHashTestAccount(23, 256) + var hasher AccountHasher + var output LtHash + b.ReportAllocs() + b.SetBytes(int64(8 + len(acct.Data) + 1 + len(acct.Owner) + len(acct.Key))) + b.ResetTimer() + for range b.N { + hasher.HashInto(&output, acct) + } + benchmarkAccountHashByte = output.Hash()[0] +} + +func BenchmarkAccountHasherLegacyReference(b *testing.B) { + acct := accountHashTestAccount(23, 256) + b.ReportAllocs() + b.SetBytes(int64(8 + len(acct.Data) + 1 + len(acct.Owner) + len(acct.Key))) + b.ResetTimer() + for range b.N { + benchmarkReferenceAccountHash = referenceAccountHash(acct) + } +} diff --git a/pkg/lthash/lthash.go b/pkg/lthash/lthash.go index 7f2cb8c7..9c4aaee1 100644 --- a/pkg/lthash/lthash.go +++ b/pkg/lthash/lthash.go @@ -16,40 +16,59 @@ type LtHash struct { value [numElements]uint16 } -func (ltHash *LtHash) calculateAcctHash(acct *accounts.Account) []byte { - hasher := blake3.New() +// AccountHasher hashes accounts into reusable LtHash storage. It is intentionally +// stateful and must not be shared between goroutines. +// +// Its zero value is ready to use. Keeping one AccountHasher per worker avoids +// rebuilding the BLAKE3 state for every old and new account value. +type AccountHasher struct { + hasher blake3.Hasher + initialized bool +} + +func (hasher *AccountHasher) reset() { + if hasher.initialized { + hasher.hasher.Reset() + return + } + + hasher.hasher = *blake3.New() + hasher.initialized = true +} + +// HashInto replaces dst with the LtHash contribution for acct. Accounts with +// zero lamports do not contribute to the accounts LtHash. +func (hasher *AccountHasher) HashInto(dst *LtHash, acct *accounts.Account) { + if acct.Lamports == 0 { + dst.Reset() + return + } + + hasher.reset() var lamportBytes [8]byte binary.LittleEndian.PutUint64(lamportBytes[:], acct.Lamports) - _, _ = hasher.Write(lamportBytes[:]) + _, _ = hasher.hasher.Write(lamportBytes[:]) - _, _ = hasher.Write(acct.Data) + _, _ = hasher.hasher.Write(acct.Data) + var executableByte [1]byte if acct.Executable { - _, _ = hasher.Write([]byte{1}) - } else { - _, _ = hasher.Write([]byte{0}) + executableByte[0] = 1 } + _, _ = hasher.hasher.Write(executableByte[:]) - _, _ = hasher.Write(acct.Owner[:]) - _, _ = hasher.Write(acct.Key[:]) - - var data [2048]byte - digest := hasher.Digest() - digest.Read(data[:]) + _, _ = hasher.hasher.Write(acct.Owner[:]) + _, _ = hasher.hasher.Write(acct.Key[:]) - return data[:] + // BLAKE3's XOF writes directly into the reusable LtHash. The previous + // implementation first allocated a separate 2 KiB output and copied it. + _, _ = hasher.hasher.Digest().Read(dst.bytes()) } func (ltHash *LtHash) InitWithAcct(acct *accounts.Account) *LtHash { - if acct.Lamports == 0 { - return &LtHash{} - } - - hashData := ltHash.calculateAcctHash(acct) - - bytes := unsafe.Slice((*uint8)(unsafe.Pointer(<Hash.value[0])), numElements*2) - copy(bytes, hashData) + var hasher AccountHasher + hasher.HashInto(ltHash, acct) return ltHash } @@ -75,18 +94,26 @@ func (ltHash *LtHash) InitWithHash(data []byte) *LtHash { panic(fmt.Sprintf("wrong len of input data (%d)", len(data))) } - bytes := unsafe.Slice((*uint8)(unsafe.Pointer(<Hash.value[0])), numElements*2) - copy(bytes, data) + for i := range numElements { + ltHash.value[i] = binary.LittleEndian.Uint16(data[i*2 : (i*2)+2]) + } return ltHash } func (ltHash *LtHash) initRandom() *LtHash { - randBytes := unsafe.Slice((*uint8)(unsafe.Pointer(<Hash.value[0])), numElements*2) - rand.Read(randBytes) + rand.Read(ltHash.bytes()) return ltHash } +func (ltHash *LtHash) bytes() []byte { + return unsafe.Slice((*uint8)(unsafe.Pointer(<Hash.value[0])), numElements*2) +} + +func (ltHash *LtHash) Reset() { + clear(ltHash.value[:]) +} + func (ltHash *LtHash) Clone() *LtHash { new := &LtHash{} copy(new.value[:], ltHash.value[:]) @@ -125,13 +152,11 @@ func (ltHash *LtHash) Equals(other *LtHash) bool { } func (ltHash *LtHash) Checksum() []byte { - data := unsafe.Slice((*uint8)(unsafe.Pointer(<Hash.value[0])), numElements*2) hasher := blake3.New() - hasher.Write(data) + hasher.Write(ltHash.bytes()) return hasher.Sum(nil) } func (ltHash *LtHash) Hash() []byte { - data := unsafe.Slice((*uint8)(unsafe.Pointer(<Hash.value[0])), numElements*2) - return data + return ltHash.bytes() } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 30ac702b..1b6296fd 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -31,10 +31,13 @@ type AccountLoader struct { WorkingSetLookup Timing InProgressLookup Timing + AppendVecPinWait Timing CacheLookup Timing + AdmissionFilter Timing IndexLookup Timing ReadPlanning Timing AppendVecRead Timing + CachePublication Timing RequestedKeys uint64 DurableKeys uint64 @@ -55,6 +58,8 @@ type AccountLoader struct { CommonCacheAdmissions uint64 CommonCacheAdmissionsSkipped uint64 VoteCacheAdmissions uint64 + VoteCacheAdmissionsSkipped uint64 + CachePublicationEpochRejects uint64 DecodedAccountObjects uint64 DecodedAccountBytes uint64 @@ -72,22 +77,86 @@ type TurbineIngress struct { ReplayAdmission Timing } +// VoteRewardDetails decomposes AlpenglowVoteRewards. Certificate timers retain +// exact BLS verification; validator preparation measures only immutable +// epoch-material lookup/build work. +type VoteRewardDetails struct { + ValidatorPreparation Timing + SkipCertificateValidation Timing + NotarCertificateValidation Timing + FinalCertificateDecode Timing + FinalCertificateValidation Timing + StatePreparation Timing + AccountMutation Timing + + ValidatorCacheHits uint64 + ValidatorCacheMisses uint64 + RewardValidators uint64 + FinalSigners uint64 + VoteAccountsUpdated uint64 +} + // Metrics for replaying a single block type BlockReplay struct { Slot uint64 AccountLoader AccountLoader TurbineIngress TurbineIngress - // Block-level latencies. - PreprocessBlock Timing - LoadBlockAccounts Timing - TxLoop Timing - Reward Timing - Rent Timing - RunIncinerator Timing - BlockUpdateAccounts Timing - AccountsDeltaHash Timing - BankHash Timing + // Exact slot wall-clock closure: SlotReplay equals the sum of the disjoint + // PreprocessBlock, ProcessBlock, and PostProcessBlock intervals. The more + // detailed timers below are nested diagnostics and must not be added to that + // top-level sum. + SlotReplay Timing + PreprocessBlock Timing + ProcessBlock Timing + TransactionExecutionPlan Timing + TransactionStatusValidation Timing + DependencyPlannerPreparation Timing + LoadBlockAccounts Timing + SlotCtxSetup Timing + // DependencyPlannerBuild is the graph/batch construction time nested + // within TxLoop. DependencyPlannerDispatch is the scheduler's wall-clock + // lifetime, including dependency waits, and is also nested within TxLoop. + DependencyPlannerBuild Timing + DependencyPlannerDispatch Timing + TxLoop Timing + Reward Timing + Rent Timing + RunIncinerator Timing + AlpenglowFooterClock Timing + AlpenglowVoteRewards Timing + VoteRewardDetails VoteRewardDetails + CompileWritableAndModifiedAccts Timing + EnsureParentAccountsForModified Timing + // BlockUpdateAccounts is synchronous critical-path work: rooted-tail + // buffering (including its callback) or legacy store enqueue. It excludes + // legacy asynchronous disk completion. + BlockUpdateAccounts Timing + TransactionStatusCommit Timing + SignatureVerificationJoin Timing + AccountsDeltaHash Timing + LtHashDedupe Timing + LtHashWorkerCompute Timing + LtHashPartialReduce Timing + BankHashFinalize Timing + BankHash Timing + AlpenglowFooterVerification Timing + // PostProcessBlock is caller-side state publication and replay + // bookkeeping after ProcessBlock returns. TransactionStatusView, + // ChainTipUpdate, and ResumeContext are nested sub-phases; logging, summary + // generation, and metric I/O are deliberately excluded. + PostProcessBlock Timing + TransactionStatusView Timing + ChainTipUpdate Timing + ResumeContext Timing + + LtHashInputAccounts uint64 + LtHashUniqueAccounts uint64 + LtHashUnchangedAccounts uint64 + LtHashCreatedAccounts uint64 + LtHashDeletedAccounts uint64 + LtHashOldDataBytes uint64 + LtHashNewDataBytes uint64 // Tx-level latencies summed for all the txs in a block. InstructionsAndAccountMetasFromTx Timing @@ -100,9 +169,28 @@ type BlockReplay struct { IxLoop Timing PostTxRentStates Timing PostBalanceDivergenceCheck Timing - TxUpdateAccounts Timing + // TxUpdateAccounts is the inclusive successful-transaction publication + // total. The TxPublish* fields below are nested children and must not be + // added to it. TouchedAccountState intentionally covers the complete scan, + // zero-lamport cleanup, MemAccounts.SetAccount, and RecordModifiedAcct loop: + // separating those calls requires observer-costly per-account clocks. + TxUpdateAccounts Timing + TxPublishRecordWritableAcct Timing + TxPublishTouchedAccountState Timing + TxPublishStakeVoteBookkeeping Timing + TxPublicationTouchedAccounts uint64 + TxPublicationTouchedAccountBytes uint64 + + // TxFailedUpdateAccounts is the inclusive publication total for failed + // transactions that still charge the payer and may advance a durable nonce. + // Preparation, payer, and nonce timers are nested children. + TxFailedUpdateAccounts Timing + TxFailedPublicationPreparation Timing + TxFailedPayerPublication Timing + TxFailedNoncePublication Timing - // Async part of tx latency + // Sigverify is summed asynchronous worker time. It overlaps other wall-clock + // phases; only SignatureVerificationJoin above is a disjoint blocking phase. Sigverify Timing // Ix-level latencies summed across all the instructions in a block. diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 1a14673d..ab2c6a96 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -778,15 +778,20 @@ func recordAccountLoaderBatchStats(dst *metrics.AccountLoader, src accountsdb.Ba dst.CommonCacheAdmissions = src.CommonCacheAdmissions dst.CommonCacheAdmissionsSkipped = src.CommonCacheAdmissionsSkipped dst.VoteCacheAdmissions = src.VoteCacheAdmissions + dst.VoteCacheAdmissionsSkipped = src.VoteCacheAdmissionsSkipped + dst.CachePublicationEpochRejects = src.CachePublicationEpochRejects dst.DecodedAccountObjects = src.DecodedAccountObjects dst.DecodedAccountBytes = src.DecodedAccountBytes dst.PlaceholderObjects = src.PlaceholderObjects dst.WorkingSetLookup.AddTiming(time.Duration(src.WorkingSetLookupNanoseconds)) dst.InProgressLookup.AddTiming(time.Duration(src.InProgressNanoseconds)) + dst.AppendVecPinWait.AddTiming(time.Duration(src.AppendVecPinWaitNanoseconds)) dst.CacheLookup.AddTiming(time.Duration(src.CacheLookupNanoseconds)) + dst.AdmissionFilter.AddTiming(time.Duration(src.AdmissionFilterNanoseconds)) dst.IndexLookup.AddTiming(time.Duration(src.IndexLookupNanoseconds)) dst.ReadPlanning.AddTiming(time.Duration(src.ReadPlanningNanoseconds)) dst.AppendVecRead.AddTiming(time.Duration(src.AppendVecReadNanoseconds)) + dst.CachePublication.AddTiming(time.Duration(src.CachePublicationNanoseconds)) } // setupInitialVoteAcctsAndStakeAccts populates the vote and stake caches at startup. @@ -2611,10 +2616,12 @@ func ReplayBlocks( block.EpochUpdatedAccts, block.ParentEpochUpdatedAccts, ) - metrics.GlobalBlockReplay.PreprocessBlock.AddTimingSince(start) - + processBlockStart := time.Now() + metrics.GlobalBlockReplay.PreprocessBlock.AddTiming(processBlockStart.Sub(start)) alpenglowClock := alpenglowMode lastSlotCtx, err = ProcessBlock(acctsDb, block, epochSchedule, txParallelism, dbgOpts, persistedHashes, unrootedTailState, transactionStatuses, alpenglowClock) + processBlockEnd := time.Now() + metrics.GlobalBlockReplay.ProcessBlock.AddTiming(processBlockEnd.Sub(processBlockStart)) if err != nil { mlog.Log.Errorf("error encountered during block replay: %s\n", err) result.Error = err @@ -2622,7 +2629,11 @@ func ReplayBlocks( global.ClearPendingStakePubkeys() break } + postProcessBlockStart := processBlockEnd + statusViewStart := time.Now() statuses := transactionStatuses.View() + metrics.GlobalBlockReplay.TransactionStatusView.AddTimingSince(statusViewStart) + chainTipStart := time.Now() identity := ChainTipIdentity{} if alpenglowMode { identity = ChainTipIdentity{ @@ -2643,6 +2654,7 @@ func ReplayBlocks( if alpenglowMode && block.HasAlpenglowLastChainedRoot { global.SetAlpenglowChainedMerkleRoot(block.Slot, solana.Hash(block.AlpenglowLastChainedRoot)) } + metrics.GlobalBlockReplay.ChainTipUpdate.AddTimingSince(chainTipStart) // Rooted-durable backpressure: if the unrooted tail grew past its cap (rooting // stalled), halt rather than grow RAM unbounded; resume re-replays from the last rooted slot. @@ -2704,6 +2716,7 @@ func ReplayBlocks( // no pointers into the global SysvarCache) and retain it in the tail until // promotion, so resume restarts from the last rooted slot not the lost in-RAM replayed tip. if unrootedTailState != nil && lastSlotCtx != nil { + resumeContextStart := time.Now() txCountAtSlot := global.TransactionCount() // ProcessBlock already added this block's txs resumeCtx := &state.ResumeContext{ Slot: block.Slot, @@ -2743,6 +2756,7 @@ func ReplayBlocks( resumeCtx.PrevLamportsPerSig = lastSlotCtx.FeeRateGovernor.PrevLamportsPerSignature } unrootedTailState.SetContext(block.Slot, resumeCtx) + metrics.GlobalBlockReplay.ResumeContext.AddTimingSince(resumeContextStart) } // Clear ManifestEpochStakes after first replayed slot past snapshot @@ -2799,7 +2813,12 @@ func ReplayBlocks( break } - slotReplayDuration := time.Since(start) + // Stop before per-slot logging, summary generation, and metric export so + // PostProcessBlock accounts only for execution-critical state publication. + slotReplayEnd := time.Now() + metrics.GlobalBlockReplay.PostProcessBlock.AddTiming(slotReplayEnd.Sub(postProcessBlockStart)) + slotReplayDuration := slotReplayEnd.Sub(start) + metrics.GlobalBlockReplay.SlotReplay.AddTiming(slotReplayDuration) txnCount := len(block.Transactions) totalCU := lastSlotCtx.TotalComputeUnitsConsumed @@ -3474,7 +3493,15 @@ func parallelTxLoop(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, bloc if canUseDependencyPlanner(plannerBlock) { do := make(chan int, len(block.Transactions)) done := make(chan int, len(block.Transactions)) - go TopsortPlannerStream(plannerBlock, do, done) + plannerDone := make(chan struct{}) + go func() { + defer close(plannerDone) + plannerStart := time.Now() + topsortPlannerStream(plannerBlock, do, done, func() { + metrics.GlobalBlockReplay.DependencyPlannerBuild.AddTimingSince(plannerStart) + }) + metrics.GlobalBlockReplay.DependencyPlannerDispatch.AddTimingSince(plannerStart) + }() wg := &sync.WaitGroup{} wg.Add(txParallelism) @@ -3511,7 +3538,10 @@ func parallelTxLoop(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, bloc wg.Wait() close(done) + <-plannerDone } else if rblock.FromLiveStream { + plannerDispatchStart := time.Now() + var plannerBuildDuration time.Duration batchWg := &sync.WaitGroup{} workersWg := &sync.WaitGroup{} do := make(chan uint64, txParallelism) @@ -3544,7 +3574,9 @@ func parallelTxLoop(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, bloc relaxIntraBatchAccountLocks := rblock.Features != nil && rblock.Features.IsActive(features.RelaxIntraBatchAccountLocks) for _, entry := range rblock.Entries { + plannerBuildStart := time.Now() batches := lightbringerEntryExecutionBatches(rblock.Transactions, entry, relaxIntraBatchAccountLocks) + plannerBuildDuration += time.Since(plannerBuildStart) for _, batch := range batches { executable := 0 for _, txIdx := range batch { @@ -3563,6 +3595,8 @@ func parallelTxLoop(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, bloc } close(do) workersWg.Wait() + metrics.GlobalBlockReplay.DependencyPlannerBuild.AddTiming(plannerBuildDuration) + metrics.GlobalBlockReplay.DependencyPlannerDispatch.AddTimingSince(plannerDispatchStart) } else { panic("dependency planner unavailable for non-Lightbringer block") } @@ -3646,12 +3680,17 @@ func ProcessBlock( if block == nil { return nil, errors.New("validate transaction messages: nil block") } + executionPlanStart := time.Now() executionPlan, err := planBlockTransactionExecution(block) + metrics.GlobalBlockReplay.TransactionExecutionPlan.AddTimingSince(executionPlanStart) if err != nil { return nil, fmt.Errorf("validate transaction messages for slot %d: %w", block.Slot, err) } - if err := transactionStatuses.validateBlockWithPlan(block, executionPlan); err != nil { - return nil, fmt.Errorf("validate transaction statuses for slot %d: %w", block.Slot, err) + statusValidationStart := time.Now() + statusValidationErr := transactionStatuses.validateBlockWithPlan(block, executionPlan) + metrics.GlobalBlockReplay.TransactionStatusValidation.AddTimingSince(statusValidationStart) + if statusValidationErr != nil { + return nil, fmt.Errorf("validate transaction statuses for slot %d: %w", block.Slot, statusValidationErr) } ctx, task := trace.NewTask(context.Background(), "ProcessBlock") defer task.End() @@ -3708,8 +3747,14 @@ func ProcessBlock( } var sigverifyWg sync.WaitGroup - defer sigverifyWg.Wait() + defer func() { + sigverifyJoinStart := time.Now() + sigverifyWg.Wait() + metrics.GlobalBlockReplay.SignatureVerificationJoin.AddTimingSince(sigverifyJoinStart) + }() + plannerPreparationStart := time.Now() plannerBlock, err := prepareDependencyPlannerBlock(block, txParallelism) + metrics.GlobalBlockReplay.DependencyPlannerPreparation.AddTimingSince(plannerPreparationStart) if err != nil { panic(fmt.Sprintf("unable to prepare dependency planner block for slot %d: %v", block.Slot, err)) } @@ -3730,9 +3775,11 @@ func ProcessBlock( } metrics.GlobalBlockReplay.LoadBlockAccounts.AddTimingSince(start) + slotCtxSetupStart := time.Now() slotCtx := newSlotCtx(block, accts, parentAccts, acctsDb, tail) slotCtx.TraceCtx = ctx slotCtx.NumSignatures = executionPlan.processedSignatures + metrics.GlobalBlockReplay.SlotCtxSetup.AddTimingSince(slotCtxSetupStart) var txFeeAccumulator fees.TxFeeInfoAccumulator var totalComputeUnitsConsumed uint64 start = time.Now() @@ -3776,22 +3823,33 @@ func ProcessBlock( // Alpenglow banks set the Clock timestamp from the block footer after execution. if alpenglowClock { + footerClockStart := time.Now() if err := applyAlpenglowFooterClock(slotCtx, block, epochSchedule); err != nil { + metrics.GlobalBlockReplay.AlpenglowFooterClock.AddTimingSince(footerClockStart) return nil, fmt.Errorf("apply alpenglow footer clock at slot %d: %w", block.Slot, err) } if err := updateAlpenglowNanosecondClockAccount(slotCtx, block); err != nil { + metrics.GlobalBlockReplay.AlpenglowFooterClock.AddTimingSince(footerClockStart) return nil, err } - if err := ApplyAlpenglowVoteRewards(slotCtx, block, epochSchedule, block.SkipRewardCert, block.NotarRewardCert, block.BlockFinalCert, block.AlpenglowShredVersion); err != nil { - return nil, err + metrics.GlobalBlockReplay.AlpenglowFooterClock.AddTimingSince(footerClockStart) + voteRewardsStart := time.Now() + voteRewardsErr := ApplyAlpenglowVoteRewards(slotCtx, block, epochSchedule, block.SkipRewardCert, block.NotarRewardCert, block.BlockFinalCert, block.AlpenglowShredVersion) + metrics.GlobalBlockReplay.AlpenglowVoteRewards.AddTimingSince(voteRewardsStart) + if voteRewardsErr != nil { + return nil, voteRewardsErr } } - start = time.Now() setReplayStage("compile_accounts") + start = time.Now() writableAccts, modifiedAccts := compileWritableAndModifiedAccts(slotCtx, block, rentAccts) - if err := ensureParentAccountsForModified(slotCtx); err != nil { - return nil, err + metrics.GlobalBlockReplay.CompileWritableAndModifiedAccts.AddTimingSince(start) + start = time.Now() + ensureParentsErr := ensureParentAccountsForModified(slotCtx) + metrics.GlobalBlockReplay.EnsureParentAccountsForModified.AddTimingSince(start) + if ensureParentsErr != nil { + return nil, ensureParentsErr } start = time.Now() @@ -3799,9 +3857,12 @@ func ProcessBlock( slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) metrics.GlobalBlockReplay.BankHash.AddTimingSince(start) if alpenglowClock { - if err := verifyAlpenglowBlockFooter(slotCtx, block, alpenglowClock); err != nil { - writeFooterBankhashMismatchArtifact(err, block, slotCtx, writableAccts, modifiedAccts) - return nil, err + footerVerificationStart := time.Now() + footerVerificationErr := verifyAlpenglowBlockFooter(slotCtx, block, alpenglowClock) + metrics.GlobalBlockReplay.AlpenglowFooterVerification.AddTimingSince(footerVerificationStart) + if footerVerificationErr != nil { + writeFooterBankhashMismatchArtifact(footerVerificationErr, block, slotCtx, writableAccts, modifiedAccts) + return nil, footerVerificationErr } } @@ -3812,14 +3873,13 @@ func ProcessBlock( // Enter critical commit window - panics here may leave AccountsDB inconsistent commitSlot.Store(slotCtx.Slot) commitInProgress.Store(true) - start = time.Now() + blockUpdateStart := time.Now() setReplayStage("store_accounts") persistedSlot := slotCtx.Slot persistedBankhash := append([]byte(nil), slotCtx.FinalBankhash...) persistedBlockSlot := block.Slot stakeIndexDir := filepath.Join(acctsDb.AcctsDir, "..") afterStoreAccounts := func() { - metrics.GlobalBlockReplay.BlockUpdateAccounts.AddTimingSince(start) if tail != nil { // Rooted-durable: accounts + bankhash are buffered in the overlay and // become durable only on promotion; nothing written here (rooted-only). @@ -3856,10 +3916,18 @@ func ProcessBlock( } else if len(modifiedAccts) > 0 { err = acctsDb.StoreAccounts(modifiedAccts, slotCtx.Slot, afterStoreAccounts) } + // In rooted-durable mode the callback above is synchronous, so this includes + // the complete critical-path overlay publication. Legacy StoreAccounts only + // enqueues here; its asynchronous disk work deliberately belongs to no slot's + // replay wall time and must never update a later slot's metrics record. + metrics.GlobalBlockReplay.BlockUpdateAccounts.AddTimingSince(blockUpdateStart) if err != nil { return slotCtx, err } - if statusErr := transactionStatuses.commitBlockWithPlan(block, executionPlan); statusErr != nil { + statusCommitStart := time.Now() + statusErr := transactionStatuses.commitBlockWithPlan(block, executionPlan) + metrics.GlobalBlockReplay.TransactionStatusCommit.AddTimingSince(statusCommitStart) + if statusErr != nil { return nil, fmt.Errorf("commit transaction statuses for slot %d after bank state commit: %w", block.Slot, statusErr) } diff --git a/pkg/replay/commit.go b/pkg/replay/commit.go index cf8fd061..81ee0ea1 100644 --- a/pkg/replay/commit.go +++ b/pkg/replay/commit.go @@ -2,7 +2,10 @@ package replay import ( "fmt" + "sync/atomic" + "time" + "github.com/Overclock-Validator/mithril/pkg/metrics" "github.com/Overclock-Validator/mithril/pkg/sealevel" ) @@ -10,21 +13,47 @@ func applySuccessfulTransactionState(slotCtx *sealevel.SlotCtx, execCtx *sealeve if execCtx == nil { return fmt.Errorf("missing execution context") } + if executionResult == nil && !accountsDeltaHashRemoved(slotCtx) { + return fmt.Errorf("missing execution result while accounts delta hash is enabled") + } - if executionResult == nil { - if !accountsDeltaHashRemoved(slotCtx) { - return fmt.Errorf("missing execution result while accounts delta hash is enabled") + recordMetrics := slotCtx != nil && slotCtx.Replay + if executionResult != nil { + var writableStart time.Time + if recordMetrics { + writableStart = time.Now() } - handleModifiedAccounts(slotCtx, execCtx) - recordStakeAndVoteAccountsFromMetas(slotCtx, execCtx) - return nil + for _, pk := range executionResult.WritableAccounts { + slotCtx.RecordWritableAcct(pk) + } + if recordMetrics { + metrics.GlobalBlockReplay.TxPublishRecordWritableAcct.AddTimingSince(writableStart) + } + } + + var touchedStart time.Time + if recordMetrics { + touchedStart = time.Now() + } + stats := handleModifiedAccounts(slotCtx, execCtx) + if recordMetrics { + metrics.GlobalBlockReplay.TxPublishTouchedAccountState.AddTimingSince(touchedStart) + atomic.AddUint64(&metrics.GlobalBlockReplay.TxPublicationTouchedAccounts, stats.touchedAccounts) + atomic.AddUint64(&metrics.GlobalBlockReplay.TxPublicationTouchedAccountBytes, stats.touchedAccountBytes) } - for _, pk := range executionResult.WritableAccounts { - slotCtx.RecordWritableAcct(pk) + var stakeVoteStart time.Time + if recordMetrics { + stakeVoteStart = time.Now() + } + if executionResult == nil { + recordStakeAndVoteAccountsFromMetas(slotCtx, execCtx) + } else { + recordStakeAndVoteAccounts(slotCtx, execCtx, executionResult.WritableAccountSet) + } + if recordMetrics { + metrics.GlobalBlockReplay.TxPublishStakeVoteBookkeeping.AddTimingSince(stakeVoteStart) } - handleModifiedAccounts(slotCtx, execCtx) - recordStakeAndVoteAccounts(slotCtx, execCtx, executionResult.WritableAccountSet) return nil } diff --git a/pkg/replay/commit_test.go b/pkg/replay/commit_test.go index c9ba470d..69162d51 100644 --- a/pkg/replay/commit_test.go +++ b/pkg/replay/commit_test.go @@ -1,6 +1,7 @@ package replay import ( + "encoding/binary" "math" "sync" "testing" @@ -8,6 +9,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/metrics" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" "github.com/gagliardetto/solana-go" @@ -105,6 +107,183 @@ func TestLeanResultCanCaptureDiagnostics(t *testing.T) { assert.Nil(t, output.ProcessingResult.ProcessedTransaction) } +func TestProcessTransactionPublicationMetrics(t *testing.T) { + for _, test := range []struct { + name string + replay bool + removeADH bool + }{ + {name: "replay-rich", replay: true}, + {name: "replay-lean-without-adh", replay: true, removeADH: true}, + {name: "block-production-is-not-recorded"}, + } { + t.Run(test.name, func(t *testing.T) { + previousMetrics := metrics.GlobalBlockReplay + defer func() { metrics.GlobalBlockReplay = previousMetrics }() + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Replay = test.replay + if test.removeADH { + slotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) + } + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(0)) + require.NoError(t, err) + metrics.GlobalBlockReplay = metrics.BlockReplay{} + var sigverify sync.WaitGroup + feeInfo, computeUnits, err := ProcessTransaction(slotCtx, &sigverify, tx, nil, nil, nil, false) + sigverify.Wait() + require.NoError(t, err) + require.NotNil(t, feeInfo) + assert.NotZero(t, computeUnits) + + got := metrics.GlobalBlockReplay + if !test.replay { + assert.Zero(t, got.TxUpdateAccounts.Count) + assert.Zero(t, got.TxPublishRecordWritableAcct.Count) + assert.Zero(t, got.TxPublishTouchedAccountState.Count) + assert.Zero(t, got.TxPublishStakeVoteBookkeeping.Count) + assert.Zero(t, got.TxPublicationTouchedAccounts) + assert.Zero(t, got.TxPublicationTouchedAccountBytes) + return + } + assert.Equal(t, uint64(1), got.TxUpdateAccounts.Count) + if test.removeADH { + assert.Zero(t, got.TxPublishRecordWritableAcct.Count) + } else { + assert.Equal(t, uint64(1), got.TxPublishRecordWritableAcct.Count) + } + assert.Equal(t, uint64(1), got.TxPublishTouchedAccountState.Count) + assert.Equal(t, uint64(1), got.TxPublishStakeVoteBookkeeping.Count) + assert.Equal(t, uint64(2), got.TxPublicationTouchedAccounts) + assert.Zero(t, got.TxPublicationTouchedAccountBytes) + children := got.TxPublishRecordWritableAcct.SumNanoseconds + + got.TxPublishTouchedAccountState.SumNanoseconds + + got.TxPublishStakeVoteBookkeeping.SumNanoseconds + assert.LessOrEqual(t, children, got.TxUpdateAccounts.SumNanoseconds) + }) + } +} + +func TestProcessTransactionFailedPublicationMetrics(t *testing.T) { + for _, replay := range []bool{true, false} { + name := "block-production-is-not-recorded" + if replay { + name = "replay" + } + t.Run(name, func(t *testing.T) { + previousMetrics := metrics.GlobalBlockReplay + defer func() { metrics.GlobalBlockReplay = previousMetrics }() + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Replay = replay + + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(0)) + require.NoError(t, err) + require.GreaterOrEqual(t, len(tx.Message.Instructions[0].Data), 12) + binary.LittleEndian.PutUint64(tx.Message.Instructions[0].Data[4:], math.MaxUint64) + payerBefore, err := slotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + + metrics.GlobalBlockReplay = metrics.BlockReplay{} + var sigverify sync.WaitGroup + feeInfo, _, processErr := ProcessTransaction(slotCtx, &sigverify, tx, nil, nil, nil, false) + sigverify.Wait() + require.Error(t, processErr) + require.NotNil(t, feeInfo) + payerAfter, err := slotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + assert.Less(t, payerAfter.Lamports, payerBefore.Lamports) + + got := metrics.GlobalBlockReplay + if !replay { + assert.Zero(t, got.TxFailedUpdateAccounts.Count) + assert.Zero(t, got.TxFailedPublicationPreparation.Count) + assert.Zero(t, got.TxFailedPayerPublication.Count) + assert.Zero(t, got.TxFailedNoncePublication.Count) + return + } + assert.Equal(t, uint64(1), got.TxFailedUpdateAccounts.Count) + assert.Equal(t, uint64(1), got.TxFailedPublicationPreparation.Count) + assert.Equal(t, uint64(1), got.TxFailedPayerPublication.Count) + assert.Zero(t, got.TxFailedNoncePublication.Count) + children := got.TxFailedPublicationPreparation.SumNanoseconds + + got.TxFailedPayerPublication.SumNanoseconds + + got.TxFailedNoncePublication.SumNanoseconds + assert.LessOrEqual(t, children, got.TxFailedUpdateAccounts.SumNanoseconds) + assert.Zero(t, got.TxUpdateAccounts.Count) + }) + } +} + +func TestHandleFailedTxNoncePublicationMetrics(t *testing.T) { + previousMetrics := metrics.GlobalBlockReplay + defer func() { metrics.GlobalBlockReplay = previousMetrics }() + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Replay = true + + previousRecent := sealevel.SysvarCache.RecentBlockHashes.Sysvar + emptyRecent := sealevel.SysvarRecentBlockhashes{} + sealevel.SysvarCache.RecentBlockHashes.Sysvar = &emptyRecent + defer func() { sealevel.SysvarCache.RecentBlockHashes.Sysvar = previousRecent }() + + authority := txfixture.PayerPubkey() + nonceKey := solana.PublicKey{0xD5} + durableNonce := [32]byte{0xAA} + nonceState := sealevel.NonceStateVersions{ + Type: sealevel.NonceVersionCurrent, + Current: sealevel.NonceData{ + IsInitialized: true, + Authority: authority, + DurableNonce: durableNonce, + FeeCalculator: sealevel.FeeCalculator{LamportsPerSignature: 5000}, + }, + } + nonceData, err := nonceState.Marshal() + require.NoError(t, err) + require.NoError(t, slotCtx.SetAccount(nonceKey, &accounts.Account{ + Key: nonceKey, Lamports: 1, Owner: addresses.SystemProgramAddr, + Data: nonceData, RentEpoch: math.MaxUint64, + })) + slotCtx.LastBlockhash = [32]byte{0x77} + + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(0)) + require.NoError(t, err) + tx.Message.RecentBlockhash = durableNonce + instructionData := make([]byte, 4) + binary.LittleEndian.PutUint32(instructionData, sealevel.SystemProgramInstrTypeAdvanceNonceAccount) + instruction := sealevel.Instruction{ + ProgramId: addresses.SystemProgramAddr, + Accounts: []sealevel.AccountMeta{ + {Pubkey: nonceKey, IsWritable: true}, + {Pubkey: authority, IsSigner: true}, + }, + Data: instructionData, + } + + metrics.GlobalBlockReplay = metrics.BlockReplay{} + feeInfo, handleErr := handleFailedTx( + slotCtx, + tx, + []sealevel.Instruction{instruction}, + &sealevel.ComputeBudgetLimits{}, + sealevel.InstrErrInvalidArgument, + nil, + ) + require.ErrorIs(t, handleErr, sealevel.InstrErrInvalidArgument) + require.NotNil(t, feeInfo) + assert.Contains(t, slotCtx.ModifiedAccts, txfixture.PayerPubkey()) + assert.Contains(t, slotCtx.ModifiedAccts, nonceKey) + got := metrics.GlobalBlockReplay + assert.Equal(t, uint64(1), got.TxFailedUpdateAccounts.Count) + assert.Equal(t, uint64(1), got.TxFailedPublicationPreparation.Count) + assert.Equal(t, uint64(1), got.TxFailedPayerPublication.Count) + assert.Equal(t, uint64(1), got.TxFailedNoncePublication.Count) + children := got.TxFailedPublicationPreparation.SumNanoseconds + + got.TxFailedPayerPublication.SumNanoseconds + got.TxFailedNoncePublication.SumNanoseconds + assert.LessOrEqual(t, children, got.TxFailedUpdateAccounts.SumNanoseconds) +} + func BenchmarkLoadAndExecuteTransferResultMode(b *testing.B) { slotCtx, cleanup := newCommitTestSlotCtx() defer cleanup() @@ -138,6 +317,44 @@ func BenchmarkLoadAndExecuteTransferResultMode(b *testing.B) { } } +func BenchmarkApplySuccessfulTransactionPublicationMetrics(b *testing.B) { + for _, enabled := range []bool{false, true} { + name := "disabled" + if enabled { + name = "enabled" + } + b.Run(name, func(b *testing.B) { + previousMetrics := metrics.GlobalBlockReplay + defer func() { metrics.GlobalBlockReplay = previousMetrics }() + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Replay = enabled + + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(0)) + require.NoError(b, err) + output := LoadAndExecuteTransaction(LoadAndExecuteTransactionInput{ + SlotCtx: slotCtx, Transaction: tx, + }) + require.Nil(b, output.ProcessingResult.TransactionError) + require.NotNil(b, output.ExecCtx) + require.NotNil(b, output.ExecutionResult) + + metrics.GlobalBlockReplay = metrics.BlockReplay{} + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := applySuccessfulTransactionState(slotCtx, output.ExecCtx, output.ExecutionResult); err != nil { + panic(err) + } + } + }) + b.StopTimer() + b.ReportMetric(2, "touched/op") + }) + } +} + func TestApplySuccessfulTransactionRejectsFailedOutput(t *testing.T) { slotCtx, cleanup := newCommitTestSlotCtx() defer cleanup() diff --git a/pkg/replay/epoch.go b/pkg/replay/epoch.go index 79129103..a35ab25b 100644 --- a/pkg/replay/epoch.go +++ b/pkg/replay/epoch.go @@ -399,6 +399,8 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B effectiveStakes, totalEffectiveStake = filterEpochStakesForVAT(effectiveStakes, voteCache, voteMetadata, minimumBalance) mlog.Log.FileOnlyf("VAT epoch stakes: admitted=%d/%d minimum_vote_balance=%d", len(effectiveStakes), len(scanResult.EffectiveStakes), minimumBalance) } + epochStakes := make(map[solana.PublicKey]uint64, len(effectiveStakes)) + epochVoteAccounts := make(map[solana.PublicKey]*epochstakes.VoteAccount, len(effectiveStakes)) for votePk, stake := range effectiveStakes { voteAcct, exists := voteCache[votePk] meta, hasMeta := voteMetadata[votePk] @@ -414,7 +416,8 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B if meta.Executable { executable = 1 } - global.PutEpochStakesEntry(leaderScheduleEpoch, votePk, stake, &epochstakes.VoteAccount{ + epochStakes[votePk] = stake + epochVoteAccounts[votePk] = &epochstakes.VoteAccount{ Lamports: meta.Lamports, NodePubkey: voteAcct.NodePubkey(), BlsPubkeyCompressed: voteAcct.BlsPubkeyCompressed(), @@ -423,11 +426,11 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B Owner: meta.Owner, Executable: executable, RentEpoch: meta.RentEpoch, - }) + } } } - global.PutEpochTotalStake(leaderScheduleEpoch, totalEffectiveStake) + global.PutEpochStakes(leaderScheduleEpoch, epochStakes, epochVoteAccounts, totalEffectiveStake) - maps.Copy(b.EpochStakesPerVoteAcct, global.EpochStakes(leaderScheduleEpoch)) + maps.Copy(b.EpochStakesPerVoteAcct, epochStakes) b.TotalEpochStake = totalEffectiveStake } diff --git a/pkg/replay/failed_publication_integration_test.go b/pkg/replay/failed_publication_integration_test.go new file mode 100644 index 00000000..ea087a9f --- /dev/null +++ b/pkg/replay/failed_publication_integration_test.go @@ -0,0 +1,135 @@ +package replay + +import ( + "crypto/sha256" + "math" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/metrics" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProcessTransactionFailedDurableNoncePublicationMetrics(t *testing.T) { + previousMetrics := metrics.GlobalBlockReplay + defer func() { metrics.GlobalBlockReplay = previousMetrics }() + + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Replay = true + + previousRecent := sealevel.SysvarCache.RecentBlockHashes.Sysvar + // Keep the queue non-empty so AdvanceNonceAccount itself is valid, while + // excluding the transaction's durable nonce so age validation takes the + // nonce-account path. + recent := sealevel.SysvarRecentBlockhashes{{ + Blockhash: [32]byte{0xBB}, + FeeCalculator: sealevel.FeeCalculator{LamportsPerSignature: 5000}, + }} + sealevel.SysvarCache.RecentBlockHashes.Sysvar = &recent + defer func() { sealevel.SysvarCache.RecentBlockHashes.Sysvar = previousRecent }() + + payer := txfixture.PayerPubkey() + nonceKey := solana.PublicKey{0xD5} + initialNonce := [32]byte{0xAA} + nonceState := sealevel.NonceStateVersions{ + Type: sealevel.NonceVersionCurrent, + Current: sealevel.NonceData{ + IsInitialized: true, + Authority: payer, + DurableNonce: initialNonce, + FeeCalculator: sealevel.FeeCalculator{LamportsPerSignature: 5000}, + }, + } + nonceData, err := nonceState.Marshal() + require.NoError(t, err) + require.NoError(t, slotCtx.SetAccount(nonceKey, &accounts.Account{ + Key: nonceKey, + Lamports: 10_000_000, + Owner: addresses.SystemProgramAddr, + Data: nonceData, + RentEpoch: math.MaxUint64, + })) + slotCtx.LastBlockhash = [32]byte{0x77} + + advanceNonce := system.NewAdvanceNonceAccountInstruction( + nonceKey, + solana.SysVarRecentBlockHashesPubkey, + payer, + ).Build() + failAfterAdvance := solana.NewInstruction( + addresses.SystemProgramAddr, + nil, + []byte{0xff, 0xff, 0xff, 0xff}, + ) + tx, err := solana.NewTransaction( + []solana.Instruction{advanceNonce, failAfterAdvance}, + solana.Hash(initialNonce), + solana.TransactionPayer(payer), + ) + require.NoError(t, err) + payerPrivateKey := txfixture.PayerPrivateKey() + _, err = tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key == payer { + return &payerPrivateKey + } + return nil + }) + require.NoError(t, err) + + payerBefore, err := slotCtx.GetAccount(payer) + require.NoError(t, err) + metrics.GlobalBlockReplay = metrics.BlockReplay{} + var sigverify sync.WaitGroup + feeInfo, _, processErr := ProcessTransaction( + slotCtx, + &sigverify, + tx, + nil, + nil, + nil, + false, + ) + sigverify.Wait() + require.ErrorIs(t, processErr, sealevel.InstrErrInvalidInstructionData) + require.NotNil(t, feeInfo) + + payerAfter, err := slotCtx.GetAccount(payer) + require.NoError(t, err) + require.Positive(t, feeInfo.TotalFee) + assert.Less(t, payerAfter.Lamports, payerBefore.Lamports) + assert.Equal(t, payerBefore.Lamports-feeInfo.TotalFee, payerAfter.Lamports) + + nonceAfter, err := slotCtx.GetAccount(nonceKey) + require.NoError(t, err) + decodedNonce, err := sealevel.UnmarshalNonceStateVersions(nonceAfter.Data) + require.NoError(t, err) + require.Equal(t, uint32(sealevel.NonceVersionCurrent), decodedNonce.Type) + state := decodedNonce.State() + require.True(t, state.IsInitialized) + assert.Equal(t, payer, state.Authority) + expectedNonce := sha256.Sum256(append([]byte("DURABLE_NONCE"), slotCtx.LastBlockhash[:]...)) + assert.Equal(t, expectedNonce, state.DurableNonce) + assert.NotEqual(t, initialNonce, state.DurableNonce) + assert.Equal(t, slotCtx.FeeRateGovernor.PrevLamportsPerSignature, state.FeeCalculator.LamportsPerSignature) + + assert.Contains(t, slotCtx.ModifiedAccts, payer) + assert.Contains(t, slotCtx.ModifiedAccts, nonceKey) + got := metrics.GlobalBlockReplay + assert.Equal(t, uint64(1), got.TxFailedUpdateAccounts.Count) + assert.Equal(t, uint64(1), got.TxFailedPublicationPreparation.Count) + assert.Equal(t, uint64(1), got.TxFailedPayerPublication.Count) + assert.Equal(t, uint64(1), got.TxFailedNoncePublication.Count) + assert.Zero(t, got.TxUpdateAccounts.Count) + children := got.TxFailedPublicationPreparation.SumNanoseconds + + got.TxFailedPayerPublication.SumNanoseconds + + got.TxFailedNoncePublication.SumNanoseconds + assert.LessOrEqual(t, children, got.TxFailedUpdateAccounts.SumNanoseconds) +} diff --git a/pkg/replay/leader_vote_account_test.go b/pkg/replay/leader_vote_account_test.go new file mode 100644 index 00000000..b3c4893c --- /dev/null +++ b/pkg/replay/leader_vote_account_test.go @@ -0,0 +1,98 @@ +package replay + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/leaderschedule" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func testExactLeaderVoteSchedule( + node solana.PublicKey, + vote solana.PublicKey, +) *leaderschedule.LeaderSchedule { + epochSchedule := &sealevel.SysvarEpochSchedule{ + SlotsPerEpoch: 32, + LeaderScheduleSlotOffset: 32, + } + return leaderschedule.New( + map[solana.PublicKey]*epochstakes.VoteAccount{ + vote: {NodePubkey: node}, + }, + map[solana.PublicKey]uint64{vote: 1}, + epochSchedule, + 0, + 1, + 1, + ) +} + +func TestLeaderVotePubkeyUsesExactScheduleMetadata(t *testing.T) { + global.SetLeaderSchedule(nil) + t.Cleanup(func() { global.SetLeaderSchedule(nil) }) + + node := solana.PublicKey{9} + selectedVote := solana.PublicKey{1} + otherVote := solana.PublicKey{2} + global.SetLeaderSchedule(testExactLeaderVoteSchedule(node, selectedVote)) + + got, err := leaderVotePubkey( + 0, + map[solana.PublicKey]*epochstakes.VoteAccount{ + selectedVote: {NodePubkey: node}, + otherVote: {NodePubkey: node}, + }, + node, + ) + require.NoError(t, err) + require.Equal(t, selectedVote, got) +} + +func TestLeaderVotePubkeyRejectsScheduledNodeMismatch(t *testing.T) { + global.SetLeaderSchedule(nil) + t.Cleanup(func() { global.SetLeaderSchedule(nil) }) + + scheduledNode := solana.PublicKey{9} + blockLeader := solana.PublicKey{10} + selectedVote := solana.PublicKey{1} + global.SetLeaderSchedule(testExactLeaderVoteSchedule(scheduledNode, selectedVote)) + + _, err := leaderVotePubkey(0, nil, blockLeader) + require.ErrorContains(t, err, "does not match block leader") +} + +func TestLeaderVotePubkeyNodeFallbackRequiresUniqueMatch(t *testing.T) { + global.SetLeaderSchedule(nil) + t.Cleanup(func() { global.SetLeaderSchedule(nil) }) + + node := solana.PublicKey{9} + voteA := solana.PublicKey{1} + voteB := solana.PublicKey{2} + + got, err := leaderVotePubkey( + 0, + map[solana.PublicKey]*epochstakes.VoteAccount{ + voteA: {NodePubkey: node}, + }, + node, + ) + require.NoError(t, err) + require.Equal(t, voteA, got) + + _, err = leaderVotePubkey( + 0, + map[solana.PublicKey]*epochstakes.VoteAccount{ + voteA: {NodePubkey: node}, + voteB: {NodePubkey: node}, + }, + node, + ) + require.ErrorContains(t, err, "ambiguous") + + _, err = leaderVotePubkey(0, nil, node) + require.ErrorContains(t, err, "not found") +} diff --git a/pkg/replay/topsort_planner.go b/pkg/replay/topsort_planner.go index a36a257d..0caa3c88 100644 --- a/pkg/replay/topsort_planner.go +++ b/pkg/replay/topsort_planner.go @@ -283,7 +283,16 @@ func TopsortPlanner(b *block.Block) [][]int { // TopsortPlanner outputs ints on out channel which have had their dependencies satisfied and can be run. On completion, return the int to the done channel. func TopsortPlannerStream(b *block.Block, out chan int, done chan int) { + topsortPlannerStream(b, out, done, nil) +} + +// topsortPlannerStream exposes the graph-build boundary to replay metrics +// without coupling the generally useful planner to the global collector. +func topsortPlannerStream(b *block.Block, out chan int, done chan int, onGraphBuilt func()) { adjList, inDegree := blockToDependencyGraph(b) + if onGraphBuilt != nil { + onGraphBuilt() + } sent := 0 // Output a topological sorting of the transactions diff --git a/pkg/replay/transaction.go b/pkg/replay/transaction.go index 21987c48..e3ba94a0 100644 --- a/pkg/replay/transaction.go +++ b/pkg/replay/transaction.go @@ -204,14 +204,19 @@ func isWritableForInstr(am *solana.AccountMeta, isProgramID bool, demoteProgramI return true } -func handleModifiedAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.ExecutionCtx) { +type transactionPublicationStats struct { + touchedAccounts uint64 + touchedAccountBytes uint64 +} + +func handleModifiedAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.ExecutionCtx) transactionPublicationStats { // update account states in slotCtx for all accounts 'touched' during the tx's execution - var touchedCount, touchedBytes uint64 + var stats transactionPublicationStats for idx, newAcctState := range execCtx.TransactionContext.Accounts.Accounts { if execCtx.TransactionContext.Accounts.Touched[idx] { // Track touched account stats for profiling - touchedCount++ - touchedBytes += uint64(len(newAcctState.Data)) + stats.touchedAccounts++ + stats.touchedAccountBytes += uint64(len(newAcctState.Data)) // clean up accounts closed during the tx (garbage collection) if newAcctState.Lamports == 0 { @@ -228,8 +233,9 @@ func handleModifiedAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.Executi } // Record touched stats for clone optimization profiling - TxAcctsTouched.Add(touchedCount) - TxAcctsTouchedBytes.Add(touchedBytes) + TxAcctsTouched.Add(stats.touchedAccounts) + TxAcctsTouchedBytes.Add(stats.touchedAccountBytes) + return stats } func recordStakeDelegation(slot uint64, acct *accounts.Account) { @@ -347,6 +353,17 @@ func recordStakeAndVoteAccountsFromMetas(slotCtx *sealevel.SlotCtx, execCtx *sea } func handleFailedTx(slotCtx *sealevel.SlotCtx, tx *solana.Transaction, instrs []sealevel.Instruction, computeBudgetLimits *sealevel.ComputeBudgetLimits, instrErr error, rentStateErr error) (*fees.TxFeeInfo, error) { + recordMetrics := slotCtx != nil && slotCtx.Replay + var totalStart time.Time + var preparationStart time.Time + if recordMetrics { + totalStart = time.Now() + preparationStart = totalStart + defer func() { + metrics.GlobalBlockReplay.TxFailedUpdateAccounts.AddTimingSince(totalStart) + }() + } + txFeeInfo := fees.CalculateTxFees(tx, instrs, computeBudgetLimits, slotCtx.Features) payerAcctKey := tx.Message.AccountKeys[0] @@ -356,21 +373,42 @@ func handleFailedTx(slotCtx *sealevel.SlotCtx, tx *solana.Transaction, instrs [] } if txFeeInfo.TotalFee > p.Lamports { + if recordMetrics { + metrics.GlobalBlockReplay.TxFailedPublicationPreparation.AddTimingSince(preparationStart) + } return nil, sealevel.InstrErrInsufficientFunds } + if recordMetrics { + metrics.GlobalBlockReplay.TxFailedPublicationPreparation.AddTimingSince(preparationStart) + } + var payerStart time.Time + if recordMetrics { + payerStart = time.Now() + } p.Lamports -= txFeeInfo.TotalFee err = slotCtx.SetAccount(payerAcctKey, p) if err != nil { panic(fmt.Sprintf("unable to set slot account to update state of payer acct after failed t: %s", err)) } slotCtx.RecordModifiedAcct(payerAcctKey) + if recordMetrics { + metrics.GlobalBlockReplay.TxFailedPayerPublication.AddTimingSince(payerStart) + } if len(instrs) >= 1 { instr := instrs[0] + recordNonce := recordMetrics && sealevel.IsNonceInstr(instr) + var nonceStart time.Time + if recordNonce { + nonceStart = time.Now() + } noncePubkey, didAdvanceNonceAcct := sealevel.MaybeAdvanceNonceAccountForFailedTx(slotCtx, tx, instr) if didAdvanceNonceAcct { slotCtx.RecordModifiedAcct(noncePubkey) + if recordNonce { + metrics.GlobalBlockReplay.TxFailedNoncePublication.AddTimingSince(nonceStart) + } } } @@ -736,11 +774,15 @@ func ProcessTransaction(slotCtx *sealevel.SlotCtx, sigverifyWg *sync.WaitGroup, } // Apply state changes to slotCtx - start = time.Now() + if slotCtx.Replay { + start = time.Now() + } if err := applySuccessfulTransactionState(slotCtx, execCtx, output.ExecutionResult); err != nil { panic(fmt.Sprintf("unable to apply successful transaction %s in slot %d: %v", tx.Signatures[0], slotCtx.Slot, err)) } - metrics.GlobalBlockReplay.TxUpdateAccounts.AddTimingSince(start) + if slotCtx.Replay { + metrics.GlobalBlockReplay.TxUpdateAccounts.AddTimingSince(start) + } return txFeeInfo, processTransactionComputeUnits(execCtx), nil } diff --git a/pkg/replay/vote_reward.go b/pkg/replay/vote_reward.go index a03d0c4a..57f7df6e 100644 --- a/pkg/replay/vote_reward.go +++ b/pkg/replay/vote_reward.go @@ -4,12 +4,15 @@ import ( "fmt" "math" "math/big" + "sync/atomic" + "time" "github.com/Overclock-Validator/mithril/pkg/accounts" - "github.com/Overclock-Validator/mithril/pkg/alpenglow" b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/metrics" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/rewardcerts" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -41,6 +44,12 @@ func ApplyAlpenglowVoteRewards( return nil } + var rewardDetails *metrics.VoteRewardDetails + if slotCtx.Replay { + rewardDetails = &metrics.GlobalBlockReplay.VoteRewardDetails + } + var statePreparationDuration time.Duration + var rewardValidators map[solana.PublicKey]struct{} var rewardSlot uint64 var rewardEpoch uint64 @@ -52,27 +61,55 @@ func ApplyAlpenglowVoteRewards( var leaderVoteOK bool if len(skipRaw) > 0 || len(notarRaw) > 0 { + var stateStart time.Time + if rewardDetails != nil { + stateStart = time.Now() + } var ok bool rewardSlot, ok = rewardcerts.RewardSlotForLeader(block.Slot) if !ok { return fmt.Errorf("slot %d vote rewards: invalid reward slot offset", block.Slot) } rewardEpoch = epochSchedule.GetEpoch(rewardSlot) + if rewardDetails != nil { + statePreparationDuration += time.Since(stateStart) + } - validatorSet, err := buildValidatorSetForEpoch(rewardEpoch) + verifierMaterial, err := loadVoteRewardVerifierMaterial(rewardEpoch, shredVersion, rewardDetails) if err != nil { return fmt.Errorf("slot %d vote rewards: %w", block.Slot, err) } - validated, err := rewardcerts.ValidateRewardCertificates(block.Slot, skipRaw, notarRaw, validatorSet, shredVersion) + validated, validationTimings, err := rewardcerts.ValidateRewardCertificatesWithVerifier( + block.Slot, + skipRaw, + notarRaw, + rewardEpoch, + verifierMaterial.verifier, + rewardDetails != nil, + ) + if rewardDetails != nil { + if len(skipRaw) > 0 { + rewardDetails.SkipCertificateValidation.AddTiming(validationTimings.Skip) + } + if len(notarRaw) > 0 { + rewardDetails.NotarCertificateValidation.AddTiming(validationTimings.Notar) + } + } if err != nil { return fmt.Errorf("slot %d validate reward certs: %w", block.Slot, err) } if validated != nil { rewardValidators = validated.Validators rewardSlot = validated.RewardSlot + if rewardDetails != nil { + atomic.AddUint64(&rewardDetails.RewardValidators, uint64(len(rewardValidators))) + } } + if rewardDetails != nil { + stateStart = time.Now() + } inflationAcct, err := loadEpochInflationAccountStateForReplay(slotCtx) if err != nil { return fmt.Errorf("slot %d vote rewards: %w", block.Slot, err) @@ -88,39 +125,76 @@ func ApplyAlpenglowVoteRewards( return fmt.Errorf("slot %d vote rewards: %w", block.Slot, err) } - leaderVote, leaderVoteOK = leaderVotePubkey(rewardEpoch, block.Leader) - if !leaderVoteOK { - return fmt.Errorf("slot %d vote rewards: leader vote account not found for %s", block.Slot, block.Leader) + rewardEpochStakes = verifierMaterial.snapshot.Stakes + totalStake = verifierMaterial.snapshot.TotalStake + // The leader reward belongs to the current block's selected vote account, + // while validator rewards use rewardSlot (eight slots earlier). At an epoch + // boundary, a compatibility fallback must therefore use the leader epoch. + leaderVoteAccounts := verifierMaterial.snapshot.VoteAccounts + leaderEpoch := epochSchedule.GetEpoch(block.Slot) + if leaderEpoch != rewardEpoch { + if leaderStakes, ok := global.EpochStakesSnapshot(leaderEpoch); ok { + leaderVoteAccounts = leaderStakes.VoteAccounts + } else { + leaderVoteAccounts = nil + } } + leaderVote, err = leaderVotePubkey(block.Slot, leaderVoteAccounts, block.Leader) + if err != nil { + return fmt.Errorf("slot %d vote rewards: %w", block.Slot, err) + } + leaderVoteOK = true - rewardEpochStakes = global.EpochStakes(rewardEpoch) - totalStake = global.EpochTotalStake(rewardEpoch) if totalStake == 0 { for _, stake := range rewardEpochStakes { totalStake += stake } } + if rewardDetails != nil { + statePreparationDuration += time.Since(stateStart) + } } var finalSigners map[solana.PublicKey]struct{} var finalSlot uint64 if len(finalCertRaw) > 0 { + var decodeStart time.Time + if rewardDetails != nil { + decodeStart = time.Now() + } fc, err := rewardcerts.DecodeFinalCertificate(finalCertRaw) + if rewardDetails != nil { + rewardDetails.FinalCertificateDecode.AddTimingSince(decodeStart) + } if err != nil { return fmt.Errorf("slot %d decode final cert: %w", block.Slot, err) } finalEpoch := epochSchedule.GetEpoch(fc.Slot) - finalValidatorSet, err := buildValidatorSetForEpoch(finalEpoch) + verifierMaterial, err := loadVoteRewardVerifierMaterial(finalEpoch, shredVersion, rewardDetails) if err != nil { return fmt.Errorf("slot %d final cert: %w", block.Slot, err) } - validatedFinal, err := rewardcerts.ValidateBlockFinalCertificate(finalCertRaw, finalValidatorSet, shredVersion) + var validationStart time.Time + if rewardDetails != nil { + validationStart = time.Now() + } + validatedFinal, err := rewardcerts.ValidateDecodedBlockFinalCertificateWithVerifier( + fc, + finalEpoch, + verifierMaterial.verifier, + ) + if rewardDetails != nil { + rewardDetails.FinalCertificateValidation.AddTimingSince(validationStart) + } if err != nil { return fmt.Errorf("slot %d validate final cert: %w", block.Slot, err) } if validatedFinal != nil { finalSigners = validatedFinal.Signers finalSlot = validatedFinal.FinalSlot + if rewardDetails != nil { + atomic.AddUint64(&rewardDetails.FinalSigners, uint64(len(finalSigners))) + } } } @@ -128,6 +202,10 @@ func ApplyAlpenglowVoteRewards( return nil } + var stateStart time.Time + if rewardDetails != nil { + stateStart = time.Now() + } producerTimeNanos, ok, err := alpenglowFooterProducerTimeNanos(block) if err != nil { return fmt.Errorf("slot %d vote rewards: footer producer time: %w", block.Slot, err) @@ -143,10 +221,17 @@ func ApplyAlpenglowVoteRewards( if len(finalSigners) > 0 { finalSlotTimestampNs = calcSlotTimestampNanos(finalSlot, block.Slot, producerTimeNanos) } + var accountMutationStart time.Time + if rewardDetails != nil { + statePreparationDuration += time.Since(stateStart) + rewardDetails.StatePreparation.AddTiming(statePreparationDuration) + accountMutationStart = time.Now() + } currentEpoch := block.Epoch var leaderRewardAccum uint64 var voteAccountsUpdated int + var leaderUpdatedInUnion bool for votePubkey := range unionVotePubkeys(rewardValidators, finalSigners) { // Read the live in-slot account (reflecting same-slot transaction writes such as a @@ -194,6 +279,9 @@ func ApplyAlpenglowVoteRewards( } slotCtx.RecordModifiedAcct(votePubkey) voteAccountsUpdated++ + if votePubkey == leaderVote { + leaderUpdatedInUnion = true + } } if leaderRewardAccum > 0 { @@ -203,7 +291,6 @@ func ApplyAlpenglowVoteRewards( // Read the live account so the leader reward stacks on top of any update the union loop // (or a same-slot transaction) already applied to the leader's vote account, mirroring // single-map Occupied/Vacant handling. - _, leaderAlreadyUpdated := slotCtx.ModifiedAccts[leaderVote] acct, err := loadAccountLiveOrParentForReplay(slotCtx, leaderVote) if err != nil { return fmt.Errorf("slot %d vote rewards: load leader vote %s: %w", block.Slot, leaderVote, err) @@ -219,11 +306,15 @@ func ApplyAlpenglowVoteRewards( return fmt.Errorf("slot %d vote rewards: set leader vote %s: %w", block.Slot, leaderVote, err) } slotCtx.RecordModifiedAcct(leaderVote) - if !leaderAlreadyUpdated { + if !leaderUpdatedInUnion { voteAccountsUpdated++ } } + if rewardDetails != nil { + rewardDetails.AccountMutation.AddTimingSince(accountMutationStart) + atomic.AddUint64(&rewardDetails.VoteAccountsUpdated, uint64(voteAccountsUpdated)) + } return nil } @@ -258,14 +349,6 @@ func unionVotePubkeys(rewardValidators, finalSigners map[solana.PublicKey]struct return out } -func buildValidatorSetForEpoch(epoch uint64) (alpenglow.ValidatorSet, error) { - stakes := global.EpochStakes(epoch) - if len(stakes) == 0 { - return alpenglow.ValidatorSet{}, fmt.Errorf("missing epoch stakes for epoch %d", epoch) - } - return alpenglow.BuildValidatorSet(epoch, stakes, global.EpochStakesVoteAccts(epoch), global.EpochTotalStake(epoch)) -} - func alpenglowMigrationEpoch(block *b.Block, epochSchedule *sealevel.SysvarEpochSchedule) (uint64, error) { if block.Features == nil { return 0, fmt.Errorf("missing feature set") @@ -277,13 +360,39 @@ func alpenglowMigrationEpoch(block *b.Block, epochSchedule *sealevel.SysvarEpoch return epochSchedule.GetEpoch(slot), nil } -func leaderVotePubkey(epoch uint64, leaderNode solana.PublicKey) (solana.PublicKey, bool) { - for pk, va := range global.EpochStakesVoteAccts(epoch) { - if va.NodePubkey == leaderNode { - return pk, true +func leaderVotePubkey( + leaderSlot uint64, + voteAccounts map[solana.PublicKey]*epochstakes.VoteAccount, + leaderNode solana.PublicKey, +) (solana.PublicKey, error) { + if scheduledNode, scheduledVote, ok := global.LeaderForSlotWithVoteAccount(leaderSlot); ok { + if scheduledNode != leaderNode { + return solana.PublicKey{}, fmt.Errorf( + "scheduled leader %s does not match block leader %s for slot %d", + scheduledNode, leaderNode, leaderSlot, + ) + } + return scheduledVote, nil + } + + var leaderVote solana.PublicKey + found := false + for pk, va := range voteAccounts { + if va != nil && va.NodePubkey == leaderNode { + if found { + return solana.PublicKey{}, fmt.Errorf( + "leader vote account is ambiguous for node %s without vote-keyed schedule metadata", + leaderNode, + ) + } + leaderVote = pk + found = true } } - return solana.PublicKey{}, false + if !found { + return solana.PublicKey{}, fmt.Errorf("leader vote account not found for %s", leaderNode) + } + return leaderVote, nil } func applyVoteRewardToAccount( diff --git a/pkg/replay/vote_reward_verifier_cache.go b/pkg/replay/vote_reward_verifier_cache.go new file mode 100644 index 00000000..0e8ca49e --- /dev/null +++ b/pkg/replay/vote_reward_verifier_cache.go @@ -0,0 +1,253 @@ +package replay + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/metrics" + "github.com/gagliardetto/solana-go" +) + +const voteRewardVerifierCacheCapacity = 4 + +type voteRewardVerifierCacheKey struct { + epoch uint64 + generation uint64 + shredVersion uint16 +} + +type voteRewardVerifierMaterial struct { + verifier *alpenglow.CertificateVerifier + snapshot epochstakes.Snapshot +} + +type voteRewardVerifierCacheEntry struct { + material *voteRewardVerifierMaterial + lastUsed uint64 +} + +type voteRewardVerifierMaterialCache struct { + mu sync.Mutex + entries map[voteRewardVerifierCacheKey]voteRewardVerifierCacheEntry + epochIdentity map[uint64][sha256.Size]byte + useSeq uint64 +} + +var cachedVoteRewardVerifiers voteRewardVerifierMaterialCache + +func (cache *voteRewardVerifierMaterialCache) get( + epoch uint64, + shredVersion uint16, +) (*voteRewardVerifierMaterial, bool, error) { + snapshot, ok := global.EpochStakesSnapshot(epoch) + if !ok || len(snapshot.Stakes) == 0 { + return nil, false, fmt.Errorf("missing epoch stakes for epoch %d", epoch) + } + key := voteRewardVerifierCacheKey{ + epoch: epoch, + generation: snapshot.Generation, + shredVersion: shredVersion, + } + + cache.mu.Lock() + defer cache.mu.Unlock() + + if entry, ok := cache.entries[key]; ok { + cache.useSeq++ + entry.lastUsed = cache.useSeq + cache.entries[key] = entry + return entry.material, true, nil + } + + identity := voteRewardSnapshotIdentity(snapshot) + if installed, ok := cache.epochIdentity[epoch]; ok && installed != identity { + return nil, false, fmt.Errorf( + "vote-reward material for epoch %d changed after installation; epoch material is immutable", + epoch, + ) + } + + // An identical same-epoch reload receives a new generation. Re-key the + // existing verifier and bind it to the newly published immutable snapshot + // rather than decompressing every validator BLS key again. + for oldKey, entry := range cache.entries { + if oldKey.epoch == epoch && oldKey.shredVersion == shredVersion { + material := &voteRewardVerifierMaterial{ + verifier: entry.material.verifier, + snapshot: snapshot, + } + cache.removeStaleEpochEntries(epoch, snapshot.Generation) + cache.insert(key, material) + return material, true, nil + } + } + + validatorSet, err := buildValidatorSetForSnapshot(snapshot) + if err != nil { + return nil, false, err + } + verifier := alpenglow.NewCertificateVerifier() + if err := verifier.SetValidatorSet(validatorSet); err != nil { + return nil, false, fmt.Errorf("configure validator set for epoch %d: %w", epoch, err) + } + verifier.SetShredVersion(shredVersion) + + if cache.epochIdentity == nil { + cache.epochIdentity = make(map[uint64][sha256.Size]byte) + } + cache.epochIdentity[epoch] = identity + cache.removeStaleEpochEntries(epoch, snapshot.Generation) + material := &voteRewardVerifierMaterial{ + verifier: verifier, + snapshot: snapshot, + } + cache.insert(key, material) + return material, false, nil +} + +func (cache *voteRewardVerifierMaterialCache) removeStaleEpochEntries(epoch, generation uint64) { + for staleKey := range cache.entries { + if staleKey.epoch == epoch && staleKey.generation != generation { + delete(cache.entries, staleKey) + } + } +} + +func (cache *voteRewardVerifierMaterialCache) insert( + key voteRewardVerifierCacheKey, + material *voteRewardVerifierMaterial, +) { + if cache.entries == nil { + cache.entries = make(map[voteRewardVerifierCacheKey]voteRewardVerifierCacheEntry) + } + if len(cache.entries) >= voteRewardVerifierCacheCapacity { + var oldestKey voteRewardVerifierCacheKey + var oldestUse uint64 + first := true + for candidateKey, candidate := range cache.entries { + if first || candidate.lastUsed < oldestUse { + oldestKey = candidateKey + oldestUse = candidate.lastUsed + first = false + } + } + delete(cache.entries, oldestKey) + } + cache.useSeq++ + cache.entries[key] = voteRewardVerifierCacheEntry{ + material: material, + lastUsed: cache.useSeq, + } +} + +func loadVoteRewardVerifierMaterial( + epoch uint64, + shredVersion uint16, + details *metrics.VoteRewardDetails, +) (*voteRewardVerifierMaterial, error) { + var start time.Time + if details != nil { + start = time.Now() + } + material, hit, err := cachedVoteRewardVerifiers.get(epoch, shredVersion) + if details != nil { + details.ValidatorPreparation.AddTimingSince(start) + if hit { + atomic.AddUint64(&details.ValidatorCacheHits, 1) + } else { + atomic.AddUint64(&details.ValidatorCacheMisses, 1) + } + } + return material, err +} + +func clearVoteRewardVerifierCacheForTest() { + cachedVoteRewardVerifiers.mu.Lock() + cachedVoteRewardVerifiers.entries = nil + cachedVoteRewardVerifiers.epochIdentity = nil + cachedVoteRewardVerifiers.useSeq = 0 + cachedVoteRewardVerifiers.mu.Unlock() +} + +func buildValidatorSetForSnapshot(snapshot epochstakes.Snapshot) (alpenglow.ValidatorSet, error) { + return alpenglow.BuildValidatorSet( + snapshot.Epoch, + snapshot.Stakes, + snapshot.VoteAccounts, + snapshot.TotalStake, + ) +} + +// voteRewardSnapshotIdentity covers both signature-verification material and +// reward/leader metadata. This makes a same-epoch reload either an exact, +// reusable copy or a fail-closed consensus error. +func voteRewardSnapshotIdentity(snapshot epochstakes.Snapshot) [sha256.Size]byte { + hasher := sha256.New() + var scratch [8]byte + writeUint64 := func(value uint64) { + binary.LittleEndian.PutUint64(scratch[:], value) + _, _ = hasher.Write(scratch[:]) + } + writePubkey := func(pubkey solana.PublicKey) { + _, _ = hasher.Write(pubkey[:]) + } + + writeUint64(snapshot.Epoch) + writeUint64(snapshot.TotalStake) + + stakeKeys := sortedEpochPublicKeys(snapshot.Stakes) + writeUint64(uint64(len(stakeKeys))) + for _, pubkey := range stakeKeys { + writePubkey(pubkey) + writeUint64(snapshot.Stakes[pubkey]) + } + + voteKeys := sortedEpochPublicKeys(snapshot.VoteAccounts) + writeUint64(uint64(len(voteKeys))) + for _, pubkey := range voteKeys { + writePubkey(pubkey) + voteAccount := snapshot.VoteAccounts[pubkey] + if voteAccount == nil { + _, _ = hasher.Write([]byte{0}) + continue + } + _, _ = hasher.Write([]byte{1}) + writeUint64(voteAccount.Lamports) + writePubkey(voteAccount.NodePubkey) + if voteAccount.BlsPubkeyCompressed == nil { + _, _ = hasher.Write([]byte{0}) + } else { + _, _ = hasher.Write([]byte{1}) + _, _ = hasher.Write(voteAccount.BlsPubkeyCompressed[:]) + } + writeUint64(uint64(voteAccount.LastTimestampTs)) + writeUint64(voteAccount.LastTimestampSlot) + writePubkey(voteAccount.Owner) + _, _ = hasher.Write([]byte{voteAccount.Executable}) + writeUint64(voteAccount.RentEpoch) + } + + var identity [sha256.Size]byte + copy(identity[:], hasher.Sum(nil)) + return identity +} + +func sortedEpochPublicKeys[V any](values map[solana.PublicKey]V) []solana.PublicKey { + keys := make([]solana.PublicKey, 0, len(values)) + for pubkey := range values { + keys = append(keys, pubkey) + } + sort.Slice(keys, func(i, j int) bool { + return bytes.Compare(keys[i][:], keys[j][:]) < 0 + }) + return keys +} diff --git a/pkg/replay/vote_reward_verifier_cache_bench_test.go b/pkg/replay/vote_reward_verifier_cache_bench_test.go new file mode 100644 index 00000000..640a9c93 --- /dev/null +++ b/pkg/replay/vote_reward_verifier_cache_bench_test.go @@ -0,0 +1,120 @@ +package replay + +import ( + "encoding/binary" + "math/big" + "testing" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/gagliardetto/solana-go" +) + +const benchmarkVoteRewardValidatorCount = 1_000 + +var ( + benchmarkVoteRewardMaterial *voteRewardVerifierMaterial + benchmarkVoteRewardVerifier *alpenglow.CertificateVerifier +) + +func BenchmarkVoteRewardVerifierCache(b *testing.B) { + const ( + epoch = uint64(9_100_000_001) + shredVersion = uint16(88) + ) + + stakes, voteAccounts, totalStake := benchmarkVoteRewardEpochMaterial(benchmarkVoteRewardValidatorCount) + global.ClearEpochStakes(epoch) + clearVoteRewardVerifierCacheForTest() + global.PutEpochStakes(epoch, stakes, voteAccounts, totalStake) + b.Cleanup(func() { + global.ClearEpochStakes(epoch) + clearVoteRewardVerifierCacheForTest() + }) + + primed, hit, err := cachedVoteRewardVerifiers.get(epoch, shredVersion) + if err != nil { + b.Fatalf("prime verifier cache: %v", err) + } + if hit { + b.Fatal("first verifier-cache lookup unexpectedly hit") + } + benchmarkVoteRewardMaterial = primed + + b.Run("HotHit_1000Validators", func(b *testing.B) { + b.ReportAllocs() + var ( + material *voteRewardVerifierMaterial + hit bool + err error + ) + b.ResetTimer() + for i := 0; i < b.N; i++ { + material, hit, err = cachedVoteRewardVerifiers.get(epoch, shredVersion) + } + b.StopTimer() + if err != nil { + b.Fatalf("cached lookup: %v", err) + } + if !hit || material != primed { + b.Fatal("cached lookup did not return the primed material") + } + benchmarkVoteRewardMaterial = material + }) + + b.Run("Rebuild_1000Validators", func(b *testing.B) { + b.ReportAllocs() + var verifier *alpenglow.CertificateVerifier + b.ResetTimer() + for i := 0; i < b.N; i++ { + snapshot, ok := global.EpochStakesSnapshot(epoch) + if !ok { + b.Fatal("benchmark epoch disappeared") + } + validatorSet, err := buildValidatorSetForSnapshot(snapshot) + if err != nil { + b.Fatalf("build validator set: %v", err) + } + verifier = alpenglow.NewCertificateVerifier() + if err := verifier.SetValidatorSet(validatorSet); err != nil { + b.Fatalf("configure verifier: %v", err) + } + verifier.SetShredVersion(shredVersion) + } + b.StopTimer() + benchmarkVoteRewardVerifier = verifier + }) +} + +func benchmarkVoteRewardEpochMaterial( + count int, +) ( + map[solana.PublicKey]uint64, + map[solana.PublicKey]*epochstakes.VoteAccount, + uint64, +) { + stakes := make(map[solana.PublicKey]uint64, count) + voteAccounts := make(map[solana.PublicKey]*epochstakes.VoteAccount, count) + var totalStake uint64 + for i := 0; i < count; i++ { + var voteAccount solana.PublicKey + binary.LittleEndian.PutUint64(voteAccount[:8], uint64(i+1)) + var nodePubkey solana.PublicKey + nodePubkey[0] = 1 + binary.LittleEndian.PutUint64(nodePubkey[1:9], uint64(i+1)) + + var blsPubkey bls12381.G1Affine + blsPubkey.ScalarMultiplicationBase(big.NewInt(int64(i + 1))) + compressed := blsPubkey.Bytes() + stake := uint64(count - i) + stakes[voteAccount] = stake + voteAccounts[voteAccount] = &epochstakes.VoteAccount{ + NodePubkey: nodePubkey, + BlsPubkeyCompressed: &compressed, + } + totalStake += stake + } + return stakes, voteAccounts, totalStake +} diff --git a/pkg/replay/vote_reward_verifier_cache_test.go b/pkg/replay/vote_reward_verifier_cache_test.go new file mode 100644 index 00000000..f3ed009f --- /dev/null +++ b/pkg/replay/vote_reward_verifier_cache_test.go @@ -0,0 +1,178 @@ +package replay + +import ( + "math/big" + "sync" + "testing" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/metrics" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVoteRewardVerifierCacheReusesIdenticalAndRejectsChangedEpochMaterial(t *testing.T) { + const epoch = uint64(9_000_000_017) + clearVoteRewardVerifierCacheForTest() + global.ClearEpochStakes(epoch) + t.Cleanup(func() { + global.ClearEpochStakes(epoch) + clearVoteRewardVerifierCacheForTest() + }) + + installTestVoteRewardValidator(epoch, 3) + var details metrics.VoteRewardDetails + + first, err := loadVoteRewardVerifierMaterial(epoch, 42, &details) + require.NoError(t, err) + require.NotNil(t, first) + assert.Equal(t, uint64(1), details.ValidatorCacheMisses) + assert.Zero(t, details.ValidatorCacheHits) + + second, err := loadVoteRewardVerifierMaterial(epoch, 42, &details) + require.NoError(t, err) + assert.Same(t, first, second) + assert.Equal(t, uint64(1), details.ValidatorCacheMisses) + assert.Equal(t, uint64(1), details.ValidatorCacheHits) + assert.Equal(t, uint64(2), details.ValidatorPreparation.Count) + + otherVersion, err := loadVoteRewardVerifierMaterial(epoch, 43, &details) + require.NoError(t, err) + assert.NotSame(t, first.verifier, otherVersion.verifier) + assert.Equal(t, uint16(43), otherVersion.verifier.ShredVersion()) + + oldGeneration := first.snapshot.Generation + installTestVoteRewardValidator(epoch, 3) + reloaded, err := loadVoteRewardVerifierMaterial(epoch, 42, &details) + require.NoError(t, err) + assert.NotSame(t, first, reloaded) + assert.Same(t, first.verifier, reloaded.verifier) + assert.NotEqual(t, oldGeneration, reloaded.snapshot.Generation) + assert.Equal(t, first.snapshot.Stakes, reloaded.snapshot.Stakes) + + installTestVoteRewardValidator(epoch, 7) + changed, err := loadVoteRewardVerifierMaterial(epoch, 42, &details) + require.Error(t, err) + assert.Nil(t, changed) + assert.Contains(t, err.Error(), "epoch material is immutable") + + firstSet, firstSetOK := first.verifier.ValidatorSetForEpoch(epoch) + require.True(t, firstSetOK) + require.Len(t, firstSet.Validators, 1) + assert.Equal(t, byte(3), firstSet.Validators[0].NodePubkey[1]) +} + +func TestVoteRewardVerifierCacheConcurrentIdenticalReloads(t *testing.T) { + const ( + epoch = uint64(9_000_000_018) + readers = 4 + iterations = 300 + ) + clearVoteRewardVerifierCacheForTest() + global.ClearEpochStakes(epoch) + t.Cleanup(func() { + global.ClearEpochStakes(epoch) + clearVoteRewardVerifierCacheForTest() + }) + + installTestVoteRewardValidator(epoch, 5) + first, _, err := cachedVoteRewardVerifiers.get(epoch, 77) + require.NoError(t, err) + + start := make(chan struct{}) + errs := make(chan error, readers) + var wg sync.WaitGroup + wg.Add(readers + 1) + go func() { + defer wg.Done() + <-start + for i := 0; i < iterations; i++ { + installTestVoteRewardValidator(epoch, 5) + } + }() + for i := 0; i < readers; i++ { + go func() { + defer wg.Done() + <-start + for j := 0; j < iterations; j++ { + material, _, getErr := cachedVoteRewardVerifiers.get(epoch, 77) + if getErr != nil { + errs <- getErr + return + } + if material.verifier != first.verifier { + errs <- assert.AnError + return + } + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } +} + +func TestVoteRewardVerifierCacheLRUEvictionIsBounded(t *testing.T) { + const baseEpoch = uint64(9_000_000_100) + clearVoteRewardVerifierCacheForTest() + t.Cleanup(func() { + for i := 0; i <= voteRewardVerifierCacheCapacity; i++ { + global.ClearEpochStakes(baseEpoch + uint64(i)) + } + clearVoteRewardVerifierCacheForTest() + }) + + for i := 0; i <= voteRewardVerifierCacheCapacity; i++ { + epoch := baseEpoch + uint64(i) + installTestVoteRewardValidator(epoch, int64(i+2)) + _, hit, err := cachedVoteRewardVerifiers.get(epoch, 88) + require.NoError(t, err) + assert.False(t, hit) + } + + cachedVoteRewardVerifiers.mu.Lock() + entryCount := len(cachedVoteRewardVerifiers.entries) + identityCount := len(cachedVoteRewardVerifiers.epochIdentity) + cachedVoteRewardVerifiers.mu.Unlock() + assert.Equal(t, voteRewardVerifierCacheCapacity, entryCount) + assert.Equal(t, voteRewardVerifierCacheCapacity+1, identityCount) + + // The evicted epoch can be rebuilt only from the exact immutable material. + _, hit, err := cachedVoteRewardVerifiers.get(baseEpoch, 88) + require.NoError(t, err) + assert.False(t, hit) +} + +func installTestVoteRewardValidator(epoch uint64, scalar int64) { + stakes, voteAccounts := testVoteRewardValidatorMaterial(epoch, scalar) + global.PutEpochStakes(epoch, stakes, voteAccounts, 100) +} + +func testVoteRewardValidatorMaterial( + epoch uint64, + scalar int64, +) (map[solana.PublicKey]uint64, map[solana.PublicKey]*epochstakes.VoteAccount) { + var voteAccount solana.PublicKey + voteAccount[0] = byte(epoch) + var nodePubkey solana.PublicKey + nodePubkey[0] = byte(epoch >> 8) + nodePubkey[1] = byte(scalar) + + var blsPubkey bls12381.G1Affine + blsPubkey.ScalarMultiplicationBase(big.NewInt(scalar)) + compressed := blsPubkey.Bytes() + return map[solana.PublicKey]uint64{ + voteAccount: 100, + }, map[solana.PublicKey]*epochstakes.VoteAccount{ + voteAccount: { + NodePubkey: nodePubkey, + BlsPubkeyCompressed: &compressed, + }, + } +} diff --git a/pkg/rewardcerts/final_cert.go b/pkg/rewardcerts/final_cert.go index 523c045f..c6218f0e 100644 --- a/pkg/rewardcerts/final_cert.go +++ b/pkg/rewardcerts/final_cert.go @@ -102,7 +102,32 @@ func ValidateBlockFinalCertificate( return nil, fmt.Errorf("configure validator set: %w", err) } verifier.SetShredVersion(shredVersion) + return validateDecodedBlockFinalCertificate(fc, validatorSet, verifier) +} + +// ValidateDecodedBlockFinalCertificateWithVerifier verifies an already +// strictly-decoded footer certificate with immutable validator material +// installed in verifier. Exact aggregate-signature verification is retained. +func ValidateDecodedBlockFinalCertificateWithVerifier( + fc FinalCertificate, + epoch uint64, + verifier *alpenglow.CertificateVerifier, +) (*ValidatedFinalCert, error) { + if verifier == nil { + return nil, fmt.Errorf("missing certificate verifier") + } + validatorSet, ok := verifier.ValidatorSetForEpoch(epoch) + if !ok { + return nil, fmt.Errorf("certificate verifier has no validator set for epoch %d", epoch) + } + return validateDecodedBlockFinalCertificate(fc, validatorSet, verifier) +} +func validateDecodedBlockFinalCertificate( + fc FinalCertificate, + validatorSet alpenglow.ValidatorSet, + verifier *alpenglow.CertificateVerifier, +) (*ValidatedFinalCert, error) { signers := make(map[solana.PublicKey]struct{}) if fc.NotarAggregate != nil { notarSig, err := decompressRewardCertSignature(fc.NotarAggregate.Signature) diff --git a/pkg/rewardcerts/validated.go b/pkg/rewardcerts/validated.go index 305a4969..0db1caa2 100644 --- a/pkg/rewardcerts/validated.go +++ b/pkg/rewardcerts/validated.go @@ -2,6 +2,7 @@ package rewardcerts import ( "fmt" + "time" bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" "github.com/Overclock-Validator/mithril/pkg/alpenglow" @@ -14,58 +15,136 @@ type ValidatedRewardCert struct { Validators map[solana.PublicKey]struct{} } +// RewardCertificateValidationTimings separates the two independent reward +// certificate decode-and-BLS-verify paths. +type RewardCertificateValidationTimings struct { + Skip time.Duration + Notar time.Duration +} + +type rewardCertificateVerifierProvider func() (alpenglow.ValidatorSet, *alpenglow.CertificateVerifier, error) + // ValidateRewardCertificates verifies footer reward certs and returns participating vote accounts. func ValidateRewardCertificates(currentSlot uint64, skipRaw, notarRaw []byte, validatorSet alpenglow.ValidatorSet, shredVersion uint16) (*ValidatedRewardCert, error) { + validated, _, err := validateRewardCertificates(currentSlot, skipRaw, notarRaw, func() (alpenglow.ValidatorSet, *alpenglow.CertificateVerifier, error) { + verifier := alpenglow.NewCertificateVerifier() + if err := verifier.SetValidatorSet(validatorSet); err != nil { + return alpenglow.ValidatorSet{}, nil, fmt.Errorf("configure validator set: %w", err) + } + verifier.SetShredVersion(shredVersion) + return validatorSet, verifier, nil + }, false) + return validated, err +} + +// ValidateRewardCertificatesWithVerifier verifies reward certs with immutable +// validator material already installed in verifier. It still decodes and +// performs exact BLS verification for every supplied certificate. +func ValidateRewardCertificatesWithVerifier( + currentSlot uint64, + skipRaw, notarRaw []byte, + epoch uint64, + verifier *alpenglow.CertificateVerifier, + measureTimings bool, +) (*ValidatedRewardCert, RewardCertificateValidationTimings, error) { + if verifier == nil { + return nil, RewardCertificateValidationTimings{}, fmt.Errorf("missing certificate verifier") + } + return validateRewardCertificates(currentSlot, skipRaw, notarRaw, func() (alpenglow.ValidatorSet, *alpenglow.CertificateVerifier, error) { + validatorSet, ok := verifier.ValidatorSetForEpoch(epoch) + if !ok { + return alpenglow.ValidatorSet{}, nil, fmt.Errorf("certificate verifier has no validator set for epoch %d", epoch) + } + return validatorSet, verifier, nil + }, measureTimings) +} + +func validateRewardCertificates( + currentSlot uint64, + skipRaw, notarRaw []byte, + verifierProvider rewardCertificateVerifierProvider, + measureTimings bool, +) (*ValidatedRewardCert, RewardCertificateValidationTimings, error) { + var timings RewardCertificateValidationTimings var skipCert *SkipRewardCertificate var notarCert *NotarRewardCertificate if len(skipRaw) > 0 { + var start time.Time + if measureTimings { + start = time.Now() + } decoded, err := DecodeSkipRewardCertificate(skipRaw) + if measureTimings { + timings.Skip += time.Since(start) + } if err != nil { - return nil, fmt.Errorf("decode skip reward cert: %w", err) + return nil, timings, fmt.Errorf("decode skip reward cert: %w", err) } skipCert = &decoded } if len(notarRaw) > 0 { + var start time.Time + if measureTimings { + start = time.Now() + } decoded, err := DecodeNotarRewardCertificate(notarRaw) + if measureTimings { + timings.Notar += time.Since(start) + } if err != nil { - return nil, fmt.Errorf("decode notar reward cert: %w", err) + return nil, timings, fmt.Errorf("decode notar reward cert: %w", err) } notarCert = &decoded } rewardSlot, err := extractRewardSlot(currentSlot, skipCert, notarCert) if err != nil { - return nil, err + return nil, timings, err } if rewardSlot == nil { - return nil, nil + return nil, timings, nil } - validators := make(map[solana.PublicKey]struct{}) - verifier := alpenglow.NewCertificateVerifier() - if err := verifier.SetValidatorSet(validatorSet); err != nil { - return nil, fmt.Errorf("configure validator set: %w", err) + validatorSet, verifier, err := verifierProvider() + if err != nil { + return nil, timings, err } - verifier.SetShredVersion(shredVersion) + validators := make(map[solana.PublicKey]struct{}) if skipCert != nil { - if err := verifyAndCollectRewardCert(verifier, validatorSet, skipCertToCertificate(*skipCert), validators); err != nil { - return nil, fmt.Errorf("verify skip reward cert: %w", err) + var start time.Time + if measureTimings { + start = time.Now() + } + err := verifyAndCollectRewardCert(verifier, validatorSet, skipCertToCertificate(*skipCert), validators) + if measureTimings { + timings.Skip += time.Since(start) + } + if err != nil { + return nil, timings, fmt.Errorf("verify skip reward cert: %w", err) } } if notarCert != nil { - if err := verifyAndCollectRewardCert(verifier, validatorSet, notarCertToCertificate(*notarCert), validators); err != nil { - return nil, fmt.Errorf("verify notar reward cert: %w", err) + var start time.Time + if measureTimings { + start = time.Now() + } + err := verifyAndCollectRewardCert(verifier, validatorSet, notarCertToCertificate(*notarCert), validators) + if measureTimings { + timings.Notar += time.Since(start) + } + if err != nil { + return nil, timings, fmt.Errorf("verify notar reward cert: %w", err) } } if len(validators) == 0 { - return nil, nil + return nil, timings, nil } return &ValidatedRewardCert{ RewardSlot: *rewardSlot, Validators: validators, - }, nil + }, timings, nil } func extractRewardSlot(currentSlot uint64, skip *SkipRewardCertificate, notar *NotarRewardCertificate) (*uint64, error) { diff --git a/pkg/rewardcerts/validated_verifier_test.go b/pkg/rewardcerts/validated_verifier_test.go new file mode 100644 index 00000000..d92e4616 --- /dev/null +++ b/pkg/rewardcerts/validated_verifier_test.go @@ -0,0 +1,114 @@ +package rewardcerts + +import ( + "testing" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRewardCertificatesWithVerifierRetainsExactBLSChecks(t *testing.T) { + const rewardSlot = uint64(123) + vote := testSignedVote(t, alpenglow.NewSkipVote(rewardSlot), 0, testBLSKeys(t, 1)[0]) + builder := NewBuilder(DefaultBuilderConfig()) + builder.AddVote(vote) + raw := builder.BuildForLeaderSlot(rewardSlot + SlotsForReward).Skip + require.NotEmpty(t, raw) + + validatorSet, _ := testValidatorSetForVotes(t, []alpenglow.VoteMessage{vote}) + verifier := alpenglow.NewCertificateVerifier() + require.NoError(t, verifier.SetValidatorSet(validatorSet)) + verifier.SetShredVersion(0) + + validated, timings, err := ValidateRewardCertificatesWithVerifier( + rewardSlot+SlotsForReward, + raw, + nil, + validatorSet.Epoch, + verifier, + true, + ) + require.NoError(t, err) + require.NotNil(t, validated) + assert.Contains(t, validated.Validators, validatorSet.Validators[0].VoteAccount) + assert.Positive(t, timings.Skip) + + tampered, err := DecodeSkipRewardCertificate(raw) + require.NoError(t, err) + tampered.Signature[0] ^= 1 + tamperedRaw, err := EncodeSkipRewardCertificate(tampered) + require.NoError(t, err) + _, _, err = ValidateRewardCertificatesWithVerifier( + rewardSlot+SlotsForReward, + tamperedRaw, + nil, + validatorSet.Epoch, + verifier, + true, + ) + require.Error(t, err) + + wrongVersionVerifier := alpenglow.NewCertificateVerifier() + require.NoError(t, wrongVersionVerifier.SetValidatorSet(validatorSet)) + wrongVersionVerifier.SetShredVersion(1) + _, _, err = ValidateRewardCertificatesWithVerifier( + rewardSlot+SlotsForReward, + raw, + nil, + validatorSet.Epoch, + wrongVersionVerifier, + true, + ) + require.Error(t, err) +} + +func TestValidateDecodedBlockFinalCertificateWithVerifierRetainsExactBLSChecks(t *testing.T) { + const slot = uint64(456) + var blockID solana.Hash + blockID[0] = 9 + + vote := testSignedVote(t, alpenglow.NewNotarizationVote(slot, blockID), 0, testBLSKeys(t, 1)[0]) + validatorSet, _ := testValidatorSetForVotes(t, []alpenglow.VoteMessage{vote}) + verifier := alpenglow.NewCertificateVerifier() + require.NoError(t, verifier.SetValidatorSet(validatorSet)) + verifier.SetShredVersion(0) + + var signature bls12381.G2Affine + _, err := signature.SetBytes(vote.Signature) + require.NoError(t, err) + finalCertificate := FinalCertificate{ + Slot: slot, + BlockID: blockID, + FinalAggregate: VotesAggregateWire{ + Signature: signature.Bytes(), + Bitmap: mustSignerBitmapBase2(t, 1, 0), + }, + } + + validated, err := ValidateDecodedBlockFinalCertificateWithVerifier( + finalCertificate, + validatorSet.Epoch, + verifier, + ) + require.NoError(t, err) + require.NotNil(t, validated) + assert.Contains(t, validated.Signers, validatorSet.Validators[0].VoteAccount) + + tampered := finalCertificate + tampered.BlockID[0] ^= 1 + _, err = ValidateDecodedBlockFinalCertificateWithVerifier(tampered, validatorSet.Epoch, verifier) + require.Error(t, err) + + wrongVersionVerifier := alpenglow.NewCertificateVerifier() + require.NoError(t, wrongVersionVerifier.SetValidatorSet(validatorSet)) + wrongVersionVerifier.SetShredVersion(1) + _, err = ValidateDecodedBlockFinalCertificateWithVerifier( + finalCertificate, + validatorSet.Epoch, + wrongVersionVerifier, + ) + require.Error(t, err) +} diff --git a/pkg/statsd/replay_diagnostics_test.go b/pkg/statsd/replay_diagnostics_test.go new file mode 100644 index 00000000..ee01fbee --- /dev/null +++ b/pkg/statsd/replay_diagnostics_test.go @@ -0,0 +1,155 @@ +package statsd + +import ( + "strings" + "testing" + "time" + + mithrilmetrics "github.com/Overclock-Validator/mithril/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type replayHistogramState struct { + count uint64 + sum float64 +} + +func readReplayHistogram( + t *testing.T, + metric Metric, + label string, +) replayHistogramState { + t.Helper() + observer, err := metricsCollection.histograms[metric].GetMetricWithLabelValues(label) + require.NoError(t, err) + dtoMetric := &dto.Metric{} + require.NoError(t, observer.(prometheus.Metric).Write(dtoMetric)) + return replayHistogramState{ + count: dtoMetric.GetHistogram().GetSampleCount(), + sum: dtoMetric.GetHistogram().GetSampleSum(), + } +} + +func readReplayCounter(t *testing.T, metric Metric, label string) float64 { + t.Helper() + counter, err := metricsCollection.counters[metric].GetMetricWithLabelValues(label) + require.NoError(t, err) + dtoMetric := &dto.Metric{} + require.NoError(t, counter.Write(dtoMetric)) + return dtoMetric.GetCounter().GetValue() +} + +func TestSendBlockReplayDiagnosticMetricsUsesSecondsAndExactMappings(t *testing.T) { + var replay mithrilmetrics.BlockReplay + timingCases := []struct { + metric Metric + label string + timing *mithrilmetrics.Timing + duration time.Duration + }{ + {AlpenglowVoteRewards, "replay_block", &replay.AlpenglowVoteRewards, 11 * time.Millisecond}, + {VoteRewardValidatorPreparation, "replay_block", &replay.VoteRewardDetails.ValidatorPreparation, 12 * time.Millisecond}, + {VoteRewardSkipCertificateValidation, "replay_block", &replay.VoteRewardDetails.SkipCertificateValidation, 13 * time.Millisecond}, + {VoteRewardNotarCertificateValidation, "replay_block", &replay.VoteRewardDetails.NotarCertificateValidation, 14 * time.Millisecond}, + {VoteRewardFinalCertificateDecode, "replay_block", &replay.VoteRewardDetails.FinalCertificateDecode, 15 * time.Millisecond}, + {VoteRewardFinalCertificateValidation, "replay_block", &replay.VoteRewardDetails.FinalCertificateValidation, 16 * time.Millisecond}, + {VoteRewardStatePreparation, "replay_block", &replay.VoteRewardDetails.StatePreparation, 17 * time.Millisecond}, + {VoteRewardAccountMutation, "replay_block", &replay.VoteRewardDetails.AccountMutation, 18 * time.Millisecond}, + {TxUpdateAccountsDuration, "replay_tx_sum", &replay.TxUpdateAccounts, 19 * time.Millisecond}, + {TxPublishRecordWritableAcct, "replay_tx_sum", &replay.TxPublishRecordWritableAcct, 20 * time.Millisecond}, + {TxPublishTouchedAccountState, "replay_tx_sum", &replay.TxPublishTouchedAccountState, 21 * time.Millisecond}, + {TxPublishStakeVoteBookkeeping, "replay_tx_sum", &replay.TxPublishStakeVoteBookkeeping, 22 * time.Millisecond}, + {TxFailedUpdateAccounts, "replay_tx_sum", &replay.TxFailedUpdateAccounts, 23 * time.Millisecond}, + {TxFailedPublicationPreparation, "replay_tx_sum", &replay.TxFailedPublicationPreparation, 24 * time.Millisecond}, + {TxFailedPayerPublication, "replay_tx_sum", &replay.TxFailedPayerPublication, 25 * time.Millisecond}, + {TxFailedNoncePublication, "replay_tx_sum", &replay.TxFailedNoncePublication, 26 * time.Millisecond}, + } + + histogramBefore := make(map[Metric]replayHistogramState, len(timingCases)) + for _, testCase := range timingCases { + require.True(t, strings.HasSuffix(testCase.metric.String(), "_duration_seconds")) + assert.Equal(t, turbinePipelineDurationBuckets, MetricToBuckets[testCase.metric]) + histogramBefore[testCase.metric] = readReplayHistogram(t, testCase.metric, testCase.label) + testCase.timing.AddTiming(testCase.duration) + } + + replay.VoteRewardDetails.ValidatorCacheHits = 2 + replay.VoteRewardDetails.ValidatorCacheMisses = 3 + replay.VoteRewardDetails.RewardValidators = 4 + replay.VoteRewardDetails.FinalSigners = 5 + replay.VoteRewardDetails.VoteAccountsUpdated = 6 + replay.TxPublicationTouchedAccounts = 7 + replay.TxPublicationTouchedAccountBytes = 8 + counterCases := []struct { + metric Metric + label string + delta float64 + }{ + {VoteRewardValidatorCacheHits, "replay_block", 2}, + {VoteRewardValidatorCacheMisses, "replay_block", 3}, + {VoteRewardValidators, "replay_block", 4}, + {VoteRewardFinalSigners, "replay_block", 5}, + {VoteRewardAccountsUpdated, "replay_block", 6}, + {TxPublicationTouchedAccounts, "replay_tx_sum", 7}, + {TxPublicationTouchedAccountBytes, "replay_tx_sum", 8}, + } + counterBefore := make(map[Metric]float64, len(counterCases)) + for _, testCase := range counterCases { + counterBefore[testCase.metric] = readReplayCounter(t, testCase.metric, testCase.label) + } + + SendBlockReplayMetrics(replay) + + for _, testCase := range timingCases { + after := readReplayHistogram(t, testCase.metric, testCase.label) + before := histogramBefore[testCase.metric] + assert.Equal(t, before.count+1, after.count, testCase.metric.String()) + assert.InDelta(t, testCase.duration.Seconds(), after.sum-before.sum, 1e-12, testCase.metric.String()) + } + for _, testCase := range counterCases { + after := readReplayCounter(t, testCase.metric, testCase.label) + assert.Equal(t, counterBefore[testCase.metric]+testCase.delta, after, testCase.metric.String()) + } +} + +func TestSendBlockReplayDiagnosticMetricsSkipsAbsentOptionalTimings(t *testing.T) { + timingCases := []struct { + metric Metric + label string + }{ + {AlpenglowVoteRewards, "replay_block"}, + {VoteRewardValidatorPreparation, "replay_block"}, + {VoteRewardSkipCertificateValidation, "replay_block"}, + {VoteRewardNotarCertificateValidation, "replay_block"}, + {VoteRewardFinalCertificateDecode, "replay_block"}, + {VoteRewardFinalCertificateValidation, "replay_block"}, + {VoteRewardStatePreparation, "replay_block"}, + {VoteRewardAccountMutation, "replay_block"}, + {TxUpdateAccountsDuration, "replay_tx_sum"}, + {TxPublishRecordWritableAcct, "replay_tx_sum"}, + {TxPublishTouchedAccountState, "replay_tx_sum"}, + {TxPublishStakeVoteBookkeeping, "replay_tx_sum"}, + {TxFailedUpdateAccounts, "replay_tx_sum"}, + {TxFailedPublicationPreparation, "replay_tx_sum"}, + {TxFailedPayerPublication, "replay_tx_sum"}, + {TxFailedNoncePublication, "replay_tx_sum"}, + } + before := make(map[Metric]replayHistogramState, len(timingCases)) + for _, testCase := range timingCases { + before[testCase.metric] = readReplayHistogram(t, testCase.metric, testCase.label) + } + + SendBlockReplayMetrics(mithrilmetrics.BlockReplay{}) + + for _, testCase := range timingCases { + assert.Equal( + t, + before[testCase.metric], + readReplayHistogram(t, testCase.metric, testCase.label), + testCase.metric.String(), + ) + } +} diff --git a/pkg/statsd/statsd.go b/pkg/statsd/statsd.go index 9860636a..ca2a26a3 100644 --- a/pkg/statsd/statsd.go +++ b/pkg/statsd/statsd.go @@ -29,15 +29,29 @@ type Metric struct { func (m Metric) String() string { return m.name } var ( - PreprocessBlock = Metric{"preprocess_block"} - TxLoop = Metric{"tx_loop"} - LoadBlockAccounts = Metric{"load_block_accounts"} - RunIncinerator = Metric{"run_incinerator"} - Reward = Metric{"reward"} - Rent = Metric{"rent"} - BlockUpdateAccounts = Metric{"block_update_accounts"} - AccountsDeltaHash = Metric{"accounts_delta_hash"} - BankHash = Metric{"bank_hash"} + PreprocessBlock = Metric{"preprocess_block"} + TxLoop = Metric{"tx_loop"} + LoadBlockAccounts = Metric{"load_block_accounts"} + RunIncinerator = Metric{"run_incinerator"} + Reward = Metric{"reward"} + Rent = Metric{"rent"} + BlockUpdateAccounts = Metric{"block_update_accounts"} + AccountsDeltaHash = Metric{"accounts_delta_hash"} + BankHash = Metric{"bank_hash"} + AlpenglowVoteRewards = Metric{"alpenglow_vote_rewards_duration_seconds"} + + VoteRewardValidatorPreparation = Metric{"vote_reward_validator_preparation_duration_seconds"} + VoteRewardSkipCertificateValidation = Metric{"vote_reward_skip_certificate_validation_duration_seconds"} + VoteRewardNotarCertificateValidation = Metric{"vote_reward_notar_certificate_validation_duration_seconds"} + VoteRewardFinalCertificateDecode = Metric{"vote_reward_final_certificate_decode_duration_seconds"} + VoteRewardFinalCertificateValidation = Metric{"vote_reward_final_certificate_validation_duration_seconds"} + VoteRewardStatePreparation = Metric{"vote_reward_state_preparation_duration_seconds"} + VoteRewardAccountMutation = Metric{"vote_reward_account_mutation_duration_seconds"} + VoteRewardValidatorCacheHits = Metric{"vote_reward_validator_cache_hits"} + VoteRewardValidatorCacheMisses = Metric{"vote_reward_validator_cache_misses"} + VoteRewardValidators = Metric{"vote_reward_validators"} + VoteRewardFinalSigners = Metric{"vote_reward_final_signers"} + VoteRewardAccountsUpdated = Metric{"vote_reward_accounts_updated"} InstructionsAndAccountMetasFromTx = Metric{"instructions_and_account_metas_from_tx"} ComputeBudgetExecutionInstructions = Metric{"compute_budget_execution_instructions"} @@ -50,6 +64,16 @@ var ( PostTxRentStates = Metric{"post_tx_rent_states"} PostBalanceDivergenceCheck = Metric{"post_balance_divergence_check"} TxUpdateAccounts = Metric{"tx_update_accounts"} + TxUpdateAccountsDuration = Metric{"tx_update_accounts_duration_seconds"} + TxPublishRecordWritableAcct = Metric{"tx_publish_record_writable_acct_duration_seconds"} + TxPublishTouchedAccountState = Metric{"tx_publish_touched_account_state_duration_seconds"} + TxPublishStakeVoteBookkeeping = Metric{"tx_publish_stake_vote_bookkeeping_duration_seconds"} + TxPublicationTouchedAccounts = Metric{"tx_publication_touched_accounts"} + TxPublicationTouchedAccountBytes = Metric{"tx_publication_touched_account_bytes"} + TxFailedUpdateAccounts = Metric{"tx_failed_update_accounts_duration_seconds"} + TxFailedPublicationPreparation = Metric{"tx_failed_publication_preparation_duration_seconds"} + TxFailedPayerPublication = Metric{"tx_failed_payer_publication_duration_seconds"} + TxFailedNoncePublication = Metric{"tx_failed_nonce_publication_duration_seconds"} GetNextIxCtx = Metric{"get_next_ix_ctx"} NextIxCtxConfigure = Metric{"next_ix_ctx_configure"} @@ -117,15 +141,29 @@ var ( // making these public so they can be used in tests and validate that if there is metrics there are corresponding types and labels var MetricToType = map[Metric]metricType{ - PreprocessBlock: TimingT, - TxLoop: TimingT, - LoadBlockAccounts: TimingT, - RunIncinerator: TimingT, - Reward: TimingT, - Rent: TimingT, - BlockUpdateAccounts: TimingT, - AccountsDeltaHash: TimingT, - BankHash: TimingT, + PreprocessBlock: TimingT, + TxLoop: TimingT, + LoadBlockAccounts: TimingT, + RunIncinerator: TimingT, + Reward: TimingT, + Rent: TimingT, + BlockUpdateAccounts: TimingT, + AccountsDeltaHash: TimingT, + BankHash: TimingT, + AlpenglowVoteRewards: TimingT, + + VoteRewardValidatorPreparation: TimingT, + VoteRewardSkipCertificateValidation: TimingT, + VoteRewardNotarCertificateValidation: TimingT, + VoteRewardFinalCertificateDecode: TimingT, + VoteRewardFinalCertificateValidation: TimingT, + VoteRewardStatePreparation: TimingT, + VoteRewardAccountMutation: TimingT, + VoteRewardValidatorCacheHits: CountT, + VoteRewardValidatorCacheMisses: CountT, + VoteRewardValidators: CountT, + VoteRewardFinalSigners: CountT, + VoteRewardAccountsUpdated: CountT, InstructionsAndAccountMetasFromTx: TimingT, ComputeBudgetExecutionInstructions: TimingT, @@ -138,6 +176,16 @@ var MetricToType = map[Metric]metricType{ PostTxRentStates: TimingT, PostBalanceDivergenceCheck: TimingT, TxUpdateAccounts: TimingT, + TxUpdateAccountsDuration: TimingT, + TxPublishRecordWritableAcct: TimingT, + TxPublishTouchedAccountState: TimingT, + TxPublishStakeVoteBookkeeping: TimingT, + TxPublicationTouchedAccounts: CountT, + TxPublicationTouchedAccountBytes: CountT, + TxFailedUpdateAccounts: TimingT, + TxFailedPublicationPreparation: TimingT, + TxFailedPayerPublication: TimingT, + TxFailedNoncePublication: TimingT, GetNextIxCtx: TimingT, NextIxCtxConfigure: TimingT, @@ -200,15 +248,29 @@ var MetricToType = map[Metric]metricType{ Slot: GaugeT, } var MetricToLabels = map[Metric][]string{ - PreprocessBlock: {"phase"}, - TxLoop: {"phase"}, - LoadBlockAccounts: {"phase"}, - RunIncinerator: {"phase"}, - Reward: {"phase"}, - Rent: {"phase"}, - BlockUpdateAccounts: {"phase"}, - AccountsDeltaHash: {"phase"}, - BankHash: {"phase"}, + PreprocessBlock: {"phase"}, + TxLoop: {"phase"}, + LoadBlockAccounts: {"phase"}, + RunIncinerator: {"phase"}, + Reward: {"phase"}, + Rent: {"phase"}, + BlockUpdateAccounts: {"phase"}, + AccountsDeltaHash: {"phase"}, + BankHash: {"phase"}, + AlpenglowVoteRewards: {"phase"}, + + VoteRewardValidatorPreparation: {"phase"}, + VoteRewardSkipCertificateValidation: {"phase"}, + VoteRewardNotarCertificateValidation: {"phase"}, + VoteRewardFinalCertificateDecode: {"phase"}, + VoteRewardFinalCertificateValidation: {"phase"}, + VoteRewardStatePreparation: {"phase"}, + VoteRewardAccountMutation: {"phase"}, + VoteRewardValidatorCacheHits: {"phase"}, + VoteRewardValidatorCacheMisses: {"phase"}, + VoteRewardValidators: {"phase"}, + VoteRewardFinalSigners: {"phase"}, + VoteRewardAccountsUpdated: {"phase"}, InstructionsAndAccountMetasFromTx: {"phase"}, ComputeBudgetExecutionInstructions: {"phase"}, @@ -221,6 +283,16 @@ var MetricToLabels = map[Metric][]string{ PostTxRentStates: {"phase"}, PostBalanceDivergenceCheck: {"phase"}, TxUpdateAccounts: {"phase"}, + TxUpdateAccountsDuration: {"phase"}, + TxPublishRecordWritableAcct: {"phase"}, + TxPublishTouchedAccountState: {"phase"}, + TxPublishStakeVoteBookkeeping: {"phase"}, + TxPublicationTouchedAccounts: {"phase"}, + TxPublicationTouchedAccountBytes: {"phase"}, + TxFailedUpdateAccounts: {"phase"}, + TxFailedPublicationPreparation: {"phase"}, + TxFailedPayerPublication: {"phase"}, + TxFailedNoncePublication: {"phase"}, GetNextIxCtx: {"phase"}, NextIxCtxConfigure: {"phase"}, @@ -308,6 +380,22 @@ var MetricToBuckets = map[Metric][]float64{ TurbineTransactionParse: turbinePipelineDurationBuckets, TurbineTransactionSigverify: turbinePipelineDurationBuckets, TurbineReplayAdmission: turbinePipelineDurationBuckets, + AlpenglowVoteRewards: turbinePipelineDurationBuckets, + VoteRewardValidatorPreparation: turbinePipelineDurationBuckets, + VoteRewardSkipCertificateValidation: turbinePipelineDurationBuckets, + VoteRewardNotarCertificateValidation: turbinePipelineDurationBuckets, + VoteRewardFinalCertificateDecode: turbinePipelineDurationBuckets, + VoteRewardFinalCertificateValidation: turbinePipelineDurationBuckets, + VoteRewardStatePreparation: turbinePipelineDurationBuckets, + VoteRewardAccountMutation: turbinePipelineDurationBuckets, + TxUpdateAccountsDuration: turbinePipelineDurationBuckets, + TxPublishRecordWritableAcct: turbinePipelineDurationBuckets, + TxPublishTouchedAccountState: turbinePipelineDurationBuckets, + TxPublishStakeVoteBookkeeping: turbinePipelineDurationBuckets, + TxFailedUpdateAccounts: turbinePipelineDurationBuckets, + TxFailedPublicationPreparation: turbinePipelineDurationBuckets, + TxFailedPayerPublication: turbinePipelineDurationBuckets, + TxFailedNoncePublication: turbinePipelineDurationBuckets, } type Prometheusmetrics struct { @@ -432,6 +520,13 @@ func Duration(m Metric, duration time.Duration, labels []string) error { return nil } +func sendReplayDuration(metric Metric, timing mithrilmetrics.Timing, labels []string) { + if timing.Count == 0 { + return + } + _ = Duration(metric, time.Duration(timing.SumNanoseconds), labels) +} + func SendBlockReplayMetrics(r mithrilmetrics.BlockReplay) { blockLatency := "replay_block" txLatency := "replay_tx_sum" @@ -447,6 +542,19 @@ func SendBlockReplayMetrics(r mithrilmetrics.BlockReplay) { Timing(BlockUpdateAccounts, r.BlockUpdateAccounts.SumNanoseconds, []string{blockLatency}) Timing(AccountsDeltaHash, r.AccountsDeltaHash.SumNanoseconds, []string{blockLatency}) Timing(BankHash, r.BankHash.SumNanoseconds, []string{blockLatency}) + sendReplayDuration(AlpenglowVoteRewards, r.AlpenglowVoteRewards, []string{blockLatency}) + sendReplayDuration(VoteRewardValidatorPreparation, r.VoteRewardDetails.ValidatorPreparation, []string{blockLatency}) + sendReplayDuration(VoteRewardSkipCertificateValidation, r.VoteRewardDetails.SkipCertificateValidation, []string{blockLatency}) + sendReplayDuration(VoteRewardNotarCertificateValidation, r.VoteRewardDetails.NotarCertificateValidation, []string{blockLatency}) + sendReplayDuration(VoteRewardFinalCertificateDecode, r.VoteRewardDetails.FinalCertificateDecode, []string{blockLatency}) + sendReplayDuration(VoteRewardFinalCertificateValidation, r.VoteRewardDetails.FinalCertificateValidation, []string{blockLatency}) + sendReplayDuration(VoteRewardStatePreparation, r.VoteRewardDetails.StatePreparation, []string{blockLatency}) + sendReplayDuration(VoteRewardAccountMutation, r.VoteRewardDetails.AccountMutation, []string{blockLatency}) + Count(VoteRewardValidatorCacheHits, int64(r.VoteRewardDetails.ValidatorCacheHits), []string{blockLatency}) + Count(VoteRewardValidatorCacheMisses, int64(r.VoteRewardDetails.ValidatorCacheMisses), []string{blockLatency}) + Count(VoteRewardValidators, int64(r.VoteRewardDetails.RewardValidators), []string{blockLatency}) + Count(VoteRewardFinalSigners, int64(r.VoteRewardDetails.FinalSigners), []string{blockLatency}) + Count(VoteRewardAccountsUpdated, int64(r.VoteRewardDetails.VoteAccountsUpdated), []string{blockLatency}) Timing(InstructionsAndAccountMetasFromTx, r.InstructionsAndAccountMetasFromTx.SumNanoseconds, []string{txLatency}) Timing(ComputeBudgetExecutionInstructions, r.ComputeBudgetExecutionInstructions.SumNanoseconds, []string{txLatency}) Timing(AccountsFromTx, r.AccountsFromTx.SumNanoseconds, []string{txLatency}) @@ -458,6 +566,16 @@ func SendBlockReplayMetrics(r mithrilmetrics.BlockReplay) { Timing(PostTxRentStates, r.PostTxRentStates.SumNanoseconds, []string{txLatency}) Timing(PostBalanceDivergenceCheck, r.PostBalanceDivergenceCheck.SumNanoseconds, []string{txLatency}) Timing(TxUpdateAccounts, r.TxUpdateAccounts.SumNanoseconds, []string{txLatency}) + sendReplayDuration(TxUpdateAccountsDuration, r.TxUpdateAccounts, []string{txLatency}) + sendReplayDuration(TxPublishRecordWritableAcct, r.TxPublishRecordWritableAcct, []string{txLatency}) + sendReplayDuration(TxPublishTouchedAccountState, r.TxPublishTouchedAccountState, []string{txLatency}) + sendReplayDuration(TxPublishStakeVoteBookkeeping, r.TxPublishStakeVoteBookkeeping, []string{txLatency}) + Count(TxPublicationTouchedAccounts, int64(r.TxPublicationTouchedAccounts), []string{txLatency}) + Count(TxPublicationTouchedAccountBytes, int64(r.TxPublicationTouchedAccountBytes), []string{txLatency}) + sendReplayDuration(TxFailedUpdateAccounts, r.TxFailedUpdateAccounts, []string{txLatency}) + sendReplayDuration(TxFailedPublicationPreparation, r.TxFailedPublicationPreparation, []string{txLatency}) + sendReplayDuration(TxFailedPayerPublication, r.TxFailedPayerPublication, []string{txLatency}) + sendReplayDuration(TxFailedNoncePublication, r.TxFailedNoncePublication, []string{txLatency}) Timing(GetNextIxCtx, r.GetNextIxCtx.SumNanoseconds, []string{ixLatency}) Timing(NextIxCtxConfigure, r.NextIxCtxConfigure.SumNanoseconds, []string{ixLatency}) Timing(IxPush, r.IxPush.SumNanoseconds, []string{ixLatency}) diff --git a/scripts/replay_timings_viewer.py b/scripts/replay_timings_viewer.py index 6c76fc60..a24345ea 100644 --- a/scripts/replay_timings_viewer.py +++ b/scripts/replay_timings_viewer.py @@ -34,7 +34,14 @@ def _(): # Flatten the structure. flat_record = {} for k, v in record.items(): - if type(v) == dict and "SumNanoseconds" in v: + if k == "VoteRewardDetails" and type(v) == dict: + for detail_key, detail_value in v.items(): + flat_key = f"VoteReward{detail_key}" + if type(detail_value) == dict and "SumNanoseconds" in detail_value: + flat_record[flat_key] = detail_value["SumNanoseconds"] / 1_000_000 + else: + flat_record[flat_key] = detail_value + elif type(v) == dict and "SumNanoseconds" in v: flat_record[k] = v["SumNanoseconds"] / 1_000_000 else: flat_record[k] = v @@ -54,26 +61,227 @@ def _(): @app.cell(hide_code=True) def _(alt, latency_records, mo): - block_chart = ( + # These three bars are exact, disjoint wall-clock intervals. SlotReplay is + # drawn as a line because it is their inclusive total and must not be + # stacked with them. + wall_components = ( alt.Chart(alt.InlineData(values=latency_records)) .transform_fold( [ "PreprocessBlock", + "ProcessBlock", + "PostProcessBlock", + ], + as_=["Phase", "Latency"], + ) + .mark_bar() + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Latency (ms)"), + color="Phase:N", + tooltip=["Slot:Q", "Phase:N", "Latency:Q"], + ) + ) + slot_total = ( + alt.Chart(alt.InlineData(values=latency_records)) + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("SlotReplay:Q", title="Latency (ms)"), + tooltip=["Slot:Q", "SlotReplay:Q"], + ) + ) + wall_chart = alt.layer(wall_components, slot_total).properties( + title="Exact slot wall time (SlotReplay line equals the stacked phases)" + ) + mo.ui.altair_chart(wall_chart) + return + + +@app.cell(hide_code=True) +def _(alt, latency_records, mo): + # Only disjoint top-level ProcessBlock phases are stacked. Planner build + # and dispatch are nested within TxLoop, so they are overlaid as lines. + # SignatureVerificationJoin is only the final blocking wait; the existing + # Sigverify metric is summed worker time that overlaps these wall phases. + process_components = ( + alt.Chart(alt.InlineData(values=latency_records)) + .transform_fold( + [ + "TransactionExecutionPlan", + "TransactionStatusValidation", + "DependencyPlannerPreparation", "LoadBlockAccounts", + "SlotCtxSetup", "TxLoop", "Reward", "Rent", "RunIncinerator", + "AlpenglowFooterClock", + "AlpenglowVoteRewards", + "CompileWritableAndModifiedAccts", + "EnsureParentAccountsForModified", + "BankHash", + "AlpenglowFooterVerification", "BlockUpdateAccounts", + "TransactionStatusCommit", + "SignatureVerificationJoin", + ], + as_=["Phase", "Latency"], + ) + .mark_bar() + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Latency (ms)"), + color="Phase:N", + tooltip=["Slot:Q", "Phase:N", "Latency:Q"], + ) + ) + process_total = ( + alt.Chart(alt.InlineData(values=latency_records)) + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("ProcessBlock:Q", title="Latency (ms)"), + tooltip=["Slot:Q", "ProcessBlock:Q"], + ) + ) + planner_detail = ( + alt.Chart(alt.InlineData(values=latency_records)) + .transform_fold( + ["DependencyPlannerBuild", "DependencyPlannerDispatch"], + as_=["Nested timer", "Latency"], + ) + .mark_line(point=True, strokeDash=[5, 3]) + .encode( + x="SlotCount:O", + y=alt.Y("Latency:Q", title="Latency (ms)"), + color="Nested timer:N", + tooltip=["Slot:Q", "Nested timer:N", "Latency:Q"], + ) + ) + process_chart = ( + alt.layer(process_components, process_total, planner_detail) + .resolve_scale(color="independent") + .properties( + title="ProcessBlock detail (black total; dashed planner timers are nested in TxLoop)" + ) + ) + mo.ui.altair_chart(process_chart) + return + + +@app.cell(hide_code=True) +def _(alt, latency_records, mo): + # These sub-phases are disjoint, but do not cover every bookkeeping action. + # Keep PostProcessBlock as an overlaid total so the residual remains visible. + post_components = ( + alt.Chart(alt.InlineData(values=latency_records)) + .transform_fold( + ["TransactionStatusView", "ChainTipUpdate", "ResumeContext"], + as_=["Phase", "Latency"], + ) + .mark_bar() + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Latency (ms)"), + color="Phase:N", + tooltip=["Slot:Q", "Phase:N", "Latency:Q"], + ) + ) + post_total = ( + alt.Chart(alt.InlineData(values=latency_records)) + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("PostProcessBlock:Q", title="Latency (ms)"), + tooltip=["Slot:Q", "PostProcessBlock:Q"], + ) + ) + post_chart = alt.layer(post_components, post_total).properties( + title="PostProcessBlock detail (inclusive total shown as black line)" + ) + mo.ui.altair_chart(post_chart) + return + + +@app.cell(hide_code=True) +def _(alt, latency_records, mo): + # AccountsDeltaHash and the LtHash phases are feature-dependent alternatives; + # BankHashFinalize follows either path. BankHash is their inclusive total. + bankhash_components = ( + alt.Chart(alt.InlineData(values=latency_records)) + .transform_fold( + [ "AccountsDeltaHash", - "BankHash", + "LtHashDedupe", + "LtHashWorkerCompute", + "LtHashPartialReduce", + "BankHashFinalize", ], as_=["Phase", "Latency"], ) .mark_bar() - .encode(x="SlotCount:O", y="Latency:Q", color="Phase:N") + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Latency (ms)"), + color="Phase:N", + tooltip=["Slot:Q", "Phase:N", "Latency:Q"], + ) + ) + bankhash_total = ( + alt.Chart(alt.InlineData(values=latency_records)) + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("BankHash:Q", title="Latency (ms)"), + tooltip=["Slot:Q", "BankHash:Q"], + ) + ) + bankhash_chart = alt.layer(bankhash_components, bankhash_total).properties( + title="BankHash/LtHash detail (BankHash inclusive total shown as black line)" + ) + mo.ui.altair_chart(bankhash_chart) + return + + +@app.cell(hide_code=True) +def _(alt, latency_records, mo): + reward_components = ( + alt.Chart(alt.InlineData(values=latency_records)) + .transform_fold( + [ + "VoteRewardValidatorPreparation", + "VoteRewardSkipCertificateValidation", + "VoteRewardNotarCertificateValidation", + "VoteRewardFinalCertificateDecode", + "VoteRewardFinalCertificateValidation", + "VoteRewardStatePreparation", + "VoteRewardAccountMutation", + ], + as_=["Phase", "Latency"], + ) + .mark_bar() + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Latency (ms)"), + color="Phase:N", + tooltip=["Slot:Q", "Phase:N", "Latency:Q"], + ) + ) + reward_total = ( + alt.Chart(alt.InlineData(values=latency_records)) + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("AlpenglowVoteRewards:Q", title="Latency (ms)"), + tooltip=["Slot:Q", "AlpenglowVoteRewards:Q"], + ) + ) + reward_chart = alt.layer(reward_components, reward_total).properties( + title="AlpenglowVoteRewards detail (inclusive total shown as black line)" ) - mo.ui.altair_chart(block_chart) + mo.ui.altair_chart(reward_chart) return @@ -94,6 +302,7 @@ def _(alt, latency_records, mo): "PostTxRentStates", "PostBalanceDivergenceCheck", "TxUpdateAccounts", + "TxFailedUpdateAccounts", ], as_=["Phase", "Latency"], ) @@ -104,6 +313,97 @@ def _(alt, latency_records, mo): return +@app.cell(hide_code=True) +def _(alt, latency_records, mo): + publication_source = alt.Chart( + alt.InlineData(values=latency_records) + ).transform_calculate( + TouchedMiB="datum.TxPublicationTouchedAccountBytes / 1048576", + TouchedMicrosPerAccount="datum.TxPublicationTouchedAccounts > 0 ? datum.TxPublishTouchedAccountState * 1000 / datum.TxPublicationTouchedAccounts : 0", + ) + publication_components = ( + publication_source + .transform_fold( + [ + "TxPublishRecordWritableAcct", + "TxPublishTouchedAccountState", + "TxPublishStakeVoteBookkeeping", + ], + as_=["Phase", "Latency"], + ) + .mark_bar() + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Summed transaction latency (ms)"), + color="Phase:N", + tooltip=[ + "Slot:Q", + "Phase:N", + "Latency:Q", + "TxPublicationTouchedAccounts:Q", + alt.Tooltip("TouchedMiB:Q", format=".2f"), + alt.Tooltip("TouchedMicrosPerAccount:Q", format=".3f"), + ], + ) + ) + publication_total = ( + publication_source + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("TxUpdateAccounts:Q", title="Summed transaction latency (ms)"), + tooltip=[ + "Slot:Q", + "TxUpdateAccounts:Q", + "TxPublicationTouchedAccounts:Q", + alt.Tooltip("TouchedMiB:Q", format=".2f"), + alt.Tooltip("TouchedMicrosPerAccount:Q", format=".3f"), + ], + ) + ) + publication_chart = alt.layer(publication_components, publication_total).properties( + title="Successful transaction publication (inclusive total shown as black line)" + ) + mo.ui.altair_chart(publication_chart) + return + + +@app.cell(hide_code=True) +def _(alt, latency_records, mo): + failed_components = ( + alt.Chart(alt.InlineData(values=latency_records)) + .transform_fold( + [ + "TxFailedPublicationPreparation", + "TxFailedPayerPublication", + "TxFailedNoncePublication", + ], + as_=["Phase", "Latency"], + ) + .mark_bar() + .encode( + x=alt.X("SlotCount:O", title="Replayed slot index"), + y=alt.Y("Latency:Q", title="Summed transaction latency (ms)"), + color="Phase:N", + tooltip=["Slot:Q", "Phase:N", "Latency:Q"], + ) + ) + failed_total = ( + alt.Chart(alt.InlineData(values=latency_records)) + .mark_line(point=True, color="black") + .encode( + x="SlotCount:O", + y=alt.Y("TxFailedUpdateAccounts:Q", title="Summed transaction latency (ms)"), + tooltip=["Slot:Q", "TxFailedUpdateAccounts:Q"], + ) + ) + failed_chart = alt.layer(failed_components, failed_total).properties( + title="Failed transaction publication (inclusive total shown as black line)" + ) + mo.ui.altair_chart(failed_chart) + return + + @app.cell(hide_code=True) def _(alt, latency_records, mo): ix_chart = ( From 6df01a26ccc996fc28d330dbd018f52b6e961f34 Mon Sep 17 00:00:00 2001 From: smcio Date: Sat, 25 Jul 2026 01:13:59 +0200 Subject: [PATCH 3/5] speed up transaction publication --- pkg/accounts/accounts.go | 52 +++ pkg/accounts/mem_accounts.go | 18 ++ pkg/accounts/overlay.go | 186 +++++++++-- pkg/accounts/overlay_benchmark_test.go | 130 ++++++++ .../transaction_account_batch_test.go | 295 ++++++++++++++++++ pkg/replay/block.go | 76 +++-- pkg/replay/commit.go | 6 +- pkg/replay/lean_writable_fastpath_test.go | 27 ++ pkg/replay/publication_capacity_test.go | 75 +++++ pkg/replay/publication_concurrency_test.go | 163 ++++++++++ pkg/replay/topsort_planner.go | 35 ++- pkg/replay/transaction.go | 22 +- pkg/sealevel/execution_ctx.go | 25 +- 13 files changed, 1023 insertions(+), 87 deletions(-) create mode 100644 pkg/accounts/overlay_benchmark_test.go create mode 100644 pkg/accounts/transaction_account_batch_test.go create mode 100644 pkg/replay/publication_capacity_test.go create mode 100644 pkg/replay/publication_concurrency_test.go diff --git a/pkg/accounts/accounts.go b/pkg/accounts/accounts.go index 0e89ad3d..d3f01be9 100644 --- a/pkg/accounts/accounts.go +++ b/pkg/accounts/accounts.go @@ -1,7 +1,9 @@ package accounts import ( + "fmt" "io" + "math" "github.com/Overclock-Validator/mithril/pkg/base58" bin "github.com/gagliardetto/binary" @@ -16,6 +18,56 @@ type Accounts interface { GetAccountWithoutLock(pubkey solana.PublicKey) (*Account, error) } +func validateTransactionAccountBatch(accountStates []*Account, touched []bool) error { + if len(accountStates) != len(touched) { + return fmt.Errorf("account states/touched length mismatch: %d != %d", len(accountStates), len(touched)) + } + for idx, acct := range accountStates { + if touched[idx] && acct == nil { + return fmt.Errorf("touched account state at index %d is nil", idx) + } + } + return nil +} + +// SetTransactionAccounts publishes touched transaction states in message order, +// canonicalizing zero-lamport states as tombstones. Built-in stores batch their +// synchronization; other Accounts implementations retain the per-key fallback. +func SetTransactionAccounts(store Accounts, accountStates []*Account, touched []bool) error { + if err := validateTransactionAccountBatch(accountStates, touched); err != nil { + return err + } + switch builtInStore := store.(type) { + case MemAccounts: + builtInStore.setTransactionAccounts(accountStates, touched) + return nil + case *MemAccounts: + builtInStore.setTransactionAccounts(accountStates, touched) + return nil + case *OverlayAccounts: + builtInStore.setTransactionAccounts(accountStates, touched) + return nil + } + for idx, acct := range accountStates { + if !touched[idx] { + continue + } + storedAcct := transactionAccountForStorage(acct) + key := [32]byte(storedAcct.Key) + if err := store.SetAccount(&key, storedAcct); err != nil { + return err + } + } + return nil +} + +func transactionAccountForStorage(acct *Account) *Account { + if acct.Lamports == 0 { + return &Account{Key: acct.Key, RentEpoch: math.MaxUint64} + } + return acct +} + type Account struct { Slot uint64 Key solana.PublicKey diff --git a/pkg/accounts/mem_accounts.go b/pkg/accounts/mem_accounts.go index c977b624..4158be55 100644 --- a/pkg/accounts/mem_accounts.go +++ b/pkg/accounts/mem_accounts.go @@ -52,6 +52,24 @@ func (m MemAccounts) SetAccount(pubkey *[32]byte, acct *Account) error { return nil } +func (m MemAccounts) SetTransactionAccounts(accountStates []*Account, touched []bool) error { + if err := validateTransactionAccountBatch(accountStates, touched); err != nil { + return err + } + m.setTransactionAccounts(accountStates, touched) + return nil +} + +func (m MemAccounts) setTransactionAccounts(accountStates []*Account, touched []bool) { + m.mu.Lock() + defer m.mu.Unlock() + for idx, acct := range accountStates { + if touched[idx] { + m.Map[acct.Key] = transactionAccountForStorage(acct) + } + } +} + func (m MemAccounts) SetAccountWithoutLock(pubkey solana.PublicKey, acct *Account) error { m.Map[pubkey] = acct return nil diff --git a/pkg/accounts/overlay.go b/pkg/accounts/overlay.go index 9807bba1..db5cc475 100644 --- a/pkg/accounts/overlay.go +++ b/pkg/accounts/overlay.go @@ -1,67 +1,193 @@ package accounts import ( + "hash/maphash" "maps" "sync" "github.com/gagliardetto/solana-go" ) +const ( + overlayMaxShardCount = 128 + overlayTargetEntriesPerShard = 64 +) + +// Padding keeps adjacent shard locks off the same cache line. The exact mutex +// size is architecture-dependent, so a full line is deliberately conservative. +type overlayAccountShard struct { + mu sync.RWMutex + delta map[[32]byte]*Account + _ [64]byte +} + // OverlayAccounts is a branch-local MVCC overlay over a parent account set: writes // go to an in-memory delta, reads fall back to the never-mutated parent. type OverlayAccounts struct { - mu sync.RWMutex - delta map[[32]byte]*Account - parent Accounts + shards []overlayAccountShard + shardMask uint64 + shardCapacity int + hashSeed maphash.Seed + parent Accounts } func NewOverlayAccounts(parent Accounts) *OverlayAccounts { + return NewOverlayAccountsWithLen(parent, 0) +} + +func NewOverlayAccountsWithLen(parent Accounts, length int) *OverlayAccounts { + return NewOverlayAccountsWithSizing(parent, length, length) +} + +// NewOverlayAccountsWithSizing sizes lock sharding from the number of keys that +// may be accessed, while sizing lazy delta maps from the expected write set. +func NewOverlayAccountsWithSizing(parent Accounts, keyCount, writeCapacity int) *OverlayAccounts { + shardCount := overlayShardCount(keyCount) + shardCapacity := 0 + if writeCapacity > 0 { + shardCapacity = (writeCapacity-1)/shardCount + 1 + } return &OverlayAccounts{ - delta: make(map[[32]byte]*Account), - parent: parent, + shards: make([]overlayAccountShard, shardCount), + shardMask: uint64(shardCount - 1), + shardCapacity: shardCapacity, + hashSeed: maphash.MakeSeed(), + parent: parent, + } +} + +func overlayShardCount(length int) int { + requested := 0 + if length > 0 { + requested = (length-1)/overlayTargetEntriesPerShard + 1 + } + shardCount := 1 + for shardCount < requested && shardCount < overlayMaxShardCount { + shardCount <<= 1 + } + return shardCount +} + +func (o *OverlayAccounts) shardForKey(pubkey [32]byte) *overlayAccountShard { + if len(o.shards) == 1 { + return &o.shards[0] + } + shardIdx := maphash.Comparable(o.hashSeed, pubkey) & o.shardMask + return &o.shards[shardIdx] +} + +// setAccountOnShard stores an account while the caller holds the shard write +// lock, or during quiescent construction through SetAccountWithoutLock. +func (o *OverlayAccounts) setAccountOnShard(shard *overlayAccountShard, pubkey [32]byte, acct *Account) { + if shard.delta == nil { + shard.delta = make(map[[32]byte]*Account, o.shardCapacity) } + shard.delta[pubkey] = acct } func (o *OverlayAccounts) GetAccount(pubkey *[32]byte) (*Account, error) { - o.mu.RLock() - defer o.mu.RUnlock() - if acct, ok := o.delta[*pubkey]; ok { + shard := o.shardForKey(*pubkey) + shard.mu.RLock() + if acct, ok := shard.delta[*pubkey]; ok { + shard.mu.RUnlock() return acct, nil } - // Lock order is always overlay -> parent, so holding RLock across the - // parent read closes the delta/parent race without risking a deadlock. - return o.parent.GetAccount(pubkey) + // Lock order is always overlay shard -> parent, so holding RLock across + // the parent read closes the same-key delta/parent race without coupling + // unrelated shards. + acct, err := o.parent.GetAccount(pubkey) + shard.mu.RUnlock() + return acct, err } func (o *OverlayAccounts) GetAccountWithoutLock(pubkey solana.PublicKey) (*Account, error) { - if acct, ok := o.delta[pubkey]; ok { + shard := o.shardForKey(pubkey) + if acct, ok := shard.delta[pubkey]; ok { return acct, nil } return o.parent.GetAccountWithoutLock(pubkey) } func (o *OverlayAccounts) SetAccount(pubkey *[32]byte, acct *Account) error { - o.mu.Lock() - o.delta[*pubkey] = acct - o.mu.Unlock() + shard := o.shardForKey(*pubkey) + shard.mu.Lock() + o.setAccountOnShard(shard, *pubkey, acct) + shard.mu.Unlock() return nil } +func (o *OverlayAccounts) SetTransactionAccounts(accountStates []*Account, touched []bool) error { + if err := validateTransactionAccountBatch(accountStates, touched); err != nil { + return err + } + o.setTransactionAccounts(accountStates, touched) + return nil +} + +func (o *OverlayAccounts) setTransactionAccounts(accountStates []*Account, touched []bool) { + if len(o.shards) == 1 { + shard := &o.shards[0] + shard.mu.Lock() + defer shard.mu.Unlock() + for idx, acct := range accountStates { + if touched[idx] { + o.setAccountOnShard(shard, acct.Key, transactionAccountForStorage(acct)) + } + } + return + } + + // Publish one key at a time, matching the old visibility contract while + // allowing scheduler-independent keys to proceed through separate shards. + for idx, acct := range accountStates { + if !touched[idx] { + continue + } + shard := o.shardForKey(acct.Key) + shard.mu.Lock() + o.setAccountOnShard(shard, acct.Key, transactionAccountForStorage(acct)) + shard.mu.Unlock() + } +} + +// SetAccountWithoutLock is reserved for quiescent construction. func (o *OverlayAccounts) SetAccountWithoutLock(pubkey solana.PublicKey, acct *Account) error { - o.delta[pubkey] = acct + shard := o.shardForKey(pubkey) + o.setAccountOnShard(shard, pubkey, acct) return nil } -// AllAccounts returns the parent set with this branch's delta applied on top — a -// point-in-time view (delta snapshotted under lock), not consistent under concurrent writes. +func (o *OverlayAccounts) lockAllShardsForRead() int { + totalAccounts := 0 + for idx := range o.shards { + o.shards[idx].mu.RLock() + totalAccounts += len(o.shards[idx].delta) + } + return totalAccounts +} + +func (o *OverlayAccounts) unlockAllShardsForRead() { + for idx := len(o.shards) - 1; idx >= 0; idx-- { + o.shards[idx].mu.RUnlock() + } +} + +func (o *OverlayAccounts) snapshotDelta() map[[32]byte]*Account { + totalAccounts := o.lockAllShardsForRead() + deltaCopy := make(map[[32]byte]*Account, totalAccounts) + for idx := range o.shards { + maps.Copy(deltaCopy, o.shards[idx].delta) + } + o.unlockAllShardsForRead() + return deltaCopy +} + +// AllAccounts returns the parent set with this branch's delta applied on top. func (o *OverlayAccounts) AllAccounts() []*Account { - o.mu.RLock() - deltaCopy := make(map[[32]byte]*Account, len(o.delta)) - maps.Copy(deltaCopy, o.delta) - o.mu.RUnlock() + deltaCopy := o.snapshotDelta() - // Merge outside the overlay lock: parent set first, then delta shadows it. - merged := make(map[[32]byte]*Account) + // Merge outside the overlay locks: parent set first, then delta shadows it. + merged := make(map[[32]byte]*Account, len(deltaCopy)) for _, acct := range o.parent.AllAccounts() { merged[[32]byte(acct.Key)] = acct } @@ -76,12 +202,14 @@ func (o *OverlayAccounts) AllAccounts() []*Account { // DeltaAccounts returns the accounts changed on this overlay (the branch diff). // Multi-branch (#14) promote-winner path; not used by the current linear tip. func (o *OverlayAccounts) DeltaAccounts() []*Account { - o.mu.RLock() - defer o.mu.RUnlock() - out := make([]*Account, 0, len(o.delta)) - for _, acct := range o.delta { - out = append(out, acct) + totalAccounts := o.lockAllShardsForRead() + out := make([]*Account, 0, totalAccounts) + for idx := range o.shards { + for _, acct := range o.shards[idx].delta { + out = append(out, acct) + } } + o.unlockAllShardsForRead() return out } diff --git a/pkg/accounts/overlay_benchmark_test.go b/pkg/accounts/overlay_benchmark_test.go new file mode 100644 index 00000000..0b6936f8 --- /dev/null +++ b/pkg/accounts/overlay_benchmark_test.go @@ -0,0 +1,130 @@ +package accounts + +import ( + "encoding/binary" + "sync" + "sync/atomic" + "testing" + + "github.com/gagliardetto/solana-go" +) + +const overlayPublicationBenchmarkBatchCount = 1 << 15 + +type overlayPublicationBenchmarkBatch struct { + accountStates [2]*Account + touched [2]bool +} + +type overlayPublicationBenchmarkStore interface { + GetAccount(pubkey *[32]byte) (*Account, error) + SetTransactionAccounts(accountStates []*Account, touched []bool) error +} + +type globalOverlayBenchmarkStore struct { + mu sync.RWMutex + delta map[[32]byte]*Account + parent Accounts +} + +func (o *globalOverlayBenchmarkStore) GetAccount(pubkey *[32]byte) (*Account, error) { + o.mu.RLock() + if acct, ok := o.delta[*pubkey]; ok { + o.mu.RUnlock() + return acct, nil + } + acct, err := o.parent.GetAccount(pubkey) + o.mu.RUnlock() + return acct, err +} + +func (o *globalOverlayBenchmarkStore) SetTransactionAccounts(accountStates []*Account, touched []bool) error { + if err := validateTransactionAccountBatch(accountStates, touched); err != nil { + return err + } + o.mu.Lock() + for idx, acct := range accountStates { + if touched[idx] { + o.delta[acct.Key] = transactionAccountForStorage(acct) + } + } + o.mu.Unlock() + return nil +} + +func overlayPublicationBenchmarkKey(n uint64) solana.PublicKey { + var key solana.PublicKey + binary.LittleEndian.PutUint64(key[:8], n) + binary.LittleEndian.PutUint64(key[8:16], n*0x9e3779b97f4a7c15) + binary.LittleEndian.PutUint64(key[16:24], n^0xa0761d6478bd642f) + binary.LittleEndian.PutUint64(key[24:], n*0xe7037ed1a0b428db) + return key +} + +func overlayPublicationBenchmarkFixture(b *testing.B) (MemAccounts, []overlayPublicationBenchmarkBatch) { + parent := NewMemAccountsWithLen(overlayPublicationBenchmarkBatchCount * 2) + batches := make([]overlayPublicationBenchmarkBatch, overlayPublicationBenchmarkBatchCount) + for batchIdx := range batches { + for accountIdx := range batches[batchIdx].accountStates { + key := overlayPublicationBenchmarkKey(uint64(batchIdx*2 + accountIdx + 1)) + acct := &Account{Key: key, Lamports: uint64(batchIdx + 1)} + if err := parent.SetAccountWithoutLock(key, acct); err != nil { + b.Fatal(err) + } + batches[batchIdx].accountStates[accountIdx] = acct + batches[batchIdx].touched[accountIdx] = true + } + } + return parent, batches +} + +func BenchmarkOverlayAccountsParallelReadPublish(b *testing.B) { + parent, batches := overlayPublicationBenchmarkFixture(b) + benchmarks := []struct { + name string + new func() overlayPublicationBenchmarkStore + }{ + { + name: "global", + new: func() overlayPublicationBenchmarkStore { + return &globalOverlayBenchmarkStore{ + delta: make(map[[32]byte]*Account, overlayPublicationBenchmarkBatchCount*2), + parent: parent, + } + }, + }, + { + name: "sharded", + new: func() overlayPublicationBenchmarkStore { + return NewOverlayAccountsWithLen(parent, overlayPublicationBenchmarkBatchCount*2) + }, + }, + } + + for _, benchmark := range benchmarks { + b.Run(benchmark.name, func(b *testing.B) { + overlay := benchmark.new() + var nextLane atomic.Uint64 + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + lane := nextLane.Add(1) - 1 + batchIdx := int((lane * 4099) & (overlayPublicationBenchmarkBatchCount - 1)) + for pb.Next() { + batch := &batches[batchIdx] + for _, acct := range batch.accountStates { + key := [32]byte(acct.Key) + if _, err := overlay.GetAccount(&key); err != nil { + panic(err) + } + } + if err := overlay.SetTransactionAccounts(batch.accountStates[:], batch.touched[:]); err != nil { + panic(err) + } + batchIdx = (batchIdx + 1) & (overlayPublicationBenchmarkBatchCount - 1) + } + }) + b.ReportMetric(2, "accounts/op") + }) + } +} diff --git a/pkg/accounts/transaction_account_batch_test.go b/pkg/accounts/transaction_account_batch_test.go new file mode 100644 index 00000000..2bf82e3e --- /dev/null +++ b/pkg/accounts/transaction_account_batch_test.go @@ -0,0 +1,295 @@ +package accounts + +import ( + "encoding/binary" + "errors" + "math" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func transactionBatchTestKey(n uint64) solana.PublicKey { + var key solana.PublicKey + binary.LittleEndian.PutUint64(key[:8], n) + binary.LittleEndian.PutUint64(key[8:16], n*0x9e3779b97f4a7c15) + return key +} + +type fallbackTransactionAccountStore struct { + MemAccounts + setOrder []solana.PublicKey + failKey *solana.PublicKey +} + +func (s *fallbackTransactionAccountStore) SetAccount(pubkey *[32]byte, acct *Account) error { + key := solana.PublicKey(*pubkey) + s.setOrder = append(s.setOrder, key) + if s.failKey != nil && key == *s.failKey { + return errors.New("fallback set failed") + } + return s.MemAccounts.SetAccount(pubkey, acct) +} + +var _ Accounts = (*fallbackTransactionAccountStore)(nil) + +func TestSetTransactionAccountsFallback(t *testing.T) { + key1 := transactionBatchTestKey(1) + key2 := transactionBatchTestKey(2) + key3 := transactionBatchTestKey(3) + store := &fallbackTransactionAccountStore{MemAccounts: NewMemAccounts()} + + require.NoError(t, SetTransactionAccounts(store, []*Account{ + {Key: key1, Lamports: 11}, + {Key: key2, Lamports: 22}, + {Key: key3, Lamports: 0, Data: []byte{9}}, + }, []bool{true, false, true})) + assert.Equal(t, []solana.PublicKey{key1, key3}, store.setOrder) + + _, err := store.GetAccount((*[32]byte)(&key2)) + require.Error(t, err, "untouched fallback state must not be stored") + tombstone, err := store.GetAccount((*[32]byte)(&key3)) + require.NoError(t, err) + assert.Zero(t, tombstone.Lamports) + assert.Equal(t, uint64(math.MaxUint64), tombstone.RentEpoch) + assert.Empty(t, tombstone.Data) + + store.setOrder = nil + store.failKey = &key2 + err = SetTransactionAccounts(store, []*Account{ + {Key: key1, Lamports: 12}, + {Key: key2, Lamports: 23}, + }, []bool{true, true}) + require.EqualError(t, err, "fallback set failed") + assert.Equal(t, []solana.PublicKey{key1, key2}, store.setOrder) +} + +func TestSetTransactionAccountsSemantics(t *testing.T) { + for _, test := range []struct { + name string + overlay bool + }{ + {name: "memory"}, + {name: "overlay", overlay: true}, + } { + t.Run(test.name, func(t *testing.T) { + key1 := transactionBatchTestKey(1) + key2 := transactionBatchTestKey(2) + key3 := transactionBatchTestKey(3) + parent := NewMemAccountsWithLen(3) + require.NoError(t, parent.SetAccountWithoutLock(key1, &Account{Key: key1, Lamports: 1})) + require.NoError(t, parent.SetAccountWithoutLock(key2, &Account{Key: key2, Lamports: 2})) + require.NoError(t, parent.SetAccountWithoutLock(key3, &Account{Key: key3, Lamports: 3})) + + var store Accounts = parent + if test.overlay { + store = NewOverlayAccountsWithLen(parent, overlayTargetEntriesPerShard+1) + } + + states := []*Account{ + {Key: key1, Lamports: 11, Data: []byte{1, 2, 3}}, + {Key: key2, Lamports: 22}, + {Key: key3, Lamports: 0, Data: []byte{9}, Owner: [32]byte{7}}, + } + require.NoError(t, SetTransactionAccounts(store, states, []bool{true, false, true})) + + got1, err := store.GetAccount((*[32]byte)(&key1)) + require.NoError(t, err) + assert.Same(t, states[0], got1) + assert.Equal(t, uint64(11), got1.Lamports) + + got2, err := store.GetAccount((*[32]byte)(&key2)) + require.NoError(t, err) + assert.Equal(t, uint64(2), got2.Lamports, "untouched account must not be overwritten") + + got3, err := store.GetAccount((*[32]byte)(&key3)) + require.NoError(t, err) + assert.Equal(t, key3, got3.Key) + assert.Zero(t, got3.Lamports) + assert.Equal(t, uint64(math.MaxUint64), got3.RentEpoch) + assert.Empty(t, got3.Data) + assert.Zero(t, got3.Owner) + + require.Error(t, SetTransactionAccounts(store, []*Account{{Key: key1, Lamports: 99}}, nil)) + still11, err := store.GetAccount((*[32]byte)(&key1)) + require.NoError(t, err) + assert.Equal(t, uint64(11), still11.Lamports, "length mismatch must not partially mutate") + + require.Error(t, SetTransactionAccounts(store, []*Account{{Key: key1, Lamports: 99}, nil}, []bool{true, true})) + still11, err = store.GetAccount((*[32]byte)(&key1)) + require.NoError(t, err) + assert.Equal(t, uint64(11), still11.Lamports, "nil touched state must not partially mutate") + + require.NoError(t, SetTransactionAccounts(store, + []*Account{{Key: key1, Lamports: 12}, {Key: key1, Lamports: 13}}, + []bool{true, true}, + )) + last, err := store.GetAccount((*[32]byte)(&key1)) + require.NoError(t, err) + assert.Equal(t, uint64(13), last.Lamports, "duplicate key must retain last-write-wins ordering") + + if test.overlay { + parentValue, err := parent.GetAccount((*[32]byte)(&key1)) + require.NoError(t, err) + assert.Equal(t, uint64(1), parentValue.Lamports, "overlay publication must not mutate its parent") + } + }) + } +} + +func TestOverlayShardCountScalesWithCapacity(t *testing.T) { + parent := NewMemAccounts() + assert.Len(t, NewOverlayAccountsWithLen(parent, 0).shards, 1) + assert.Len(t, NewOverlayAccountsWithLen(parent, overlayTargetEntriesPerShard).shards, 1) + assert.Len(t, NewOverlayAccountsWithLen(parent, overlayTargetEntriesPerShard+1).shards, 2) + assert.Len(t, NewOverlayAccountsWithLen(parent, 1<<20).shards, overlayMaxShardCount) + sized := NewOverlayAccountsWithSizing(parent, 1<<20, overlayMaxShardCount) + assert.Len(t, sized.shards, overlayMaxShardCount) + assert.Equal(t, 1, sized.shardCapacity) +} + +func TestOverlayConcurrentTransactionPublicationAndSnapshots(t *testing.T) { + const ( + writers = 16 + batchesPerWriter = 128 + accountsPerBatch = 2 + ) + totalAccounts := writers * batchesPerWriter * accountsPerBatch + parent := NewMemAccounts() + overlay := NewOverlayAccountsWithLen(parent, totalAccounts) + + start := make(chan struct{}) + firstWritePublished := make(chan struct{}) + firstSnapshotTaken := make(chan struct{}) + stopSnapshots := make(chan struct{}) + var snapshotCount atomic.Uint64 + var snapshotWG sync.WaitGroup + snapshotWG.Add(1) + go func() { + defer snapshotWG.Done() + <-firstWritePublished + _ = overlay.DeltaAccounts() + _ = overlay.AllAccounts() + snapshotCount.Add(1) + close(firstSnapshotTaken) + for { + select { + case <-stopSnapshots: + return + default: + _ = overlay.DeltaAccounts() + _ = overlay.AllAccounts() + snapshotCount.Add(1) + } + } + }() + + var writersWG sync.WaitGroup + writersWG.Add(writers) + for writer := range writers { + go func(writer int) { + defer writersWG.Done() + <-start + for batchIdx := range batchesPerWriter { + first := uint64((writer*batchesPerWriter+batchIdx)*accountsPerBatch + 1) + accountStates := []*Account{ + {Key: transactionBatchTestKey(first), Lamports: first}, + {Key: transactionBatchTestKey(first + 1), Lamports: first + 1}, + } + if err := overlay.SetTransactionAccounts(accountStates, []bool{true, true}); err != nil { + panic(err) + } + if writer == 0 && batchIdx == 0 { + close(firstWritePublished) + <-firstSnapshotTaken + } + } + }(writer) + } + close(start) + writersWG.Wait() + close(stopSnapshots) + snapshotWG.Wait() + assert.NotZero(t, snapshotCount.Load(), "snapshot must complete before every writer returns") + + delta := overlay.DeltaAccounts() + require.Len(t, delta, totalAccounts) + for _, acct := range delta { + assert.Equal(t, binary.LittleEndian.Uint64(acct.Key[:8]), acct.Lamports) + } +} + +type blockingOverlayParent struct { + MemAccounts + started chan struct{} + release chan struct{} + once sync.Once +} + +func (p *blockingOverlayParent) GetAccount(_ *[32]byte) (*Account, error) { + p.once.Do(func() { close(p.started) }) + <-p.release + return nil, errors.New("missing account") +} + +func TestOverlayParentMissDoesNotBlockOtherShard(t *testing.T) { + parent := &blockingOverlayParent{ + MemAccounts: NewMemAccounts(), + started: make(chan struct{}), + release: make(chan struct{}), + } + overlay := NewOverlayAccountsWithLen(parent, 1<<14) + keyA := transactionBatchTestKey(1) + shardA := overlay.shardForKey(keyA) + + var keyB solana.PublicKey + for candidate := uint64(2); ; candidate++ { + keyB = transactionBatchTestKey(candidate) + if overlay.shardForKey(keyB) != shardA { + break + } + } + + getDone := make(chan struct{}) + go func() { + _, _ = overlay.GetAccount((*[32]byte)(&keyA)) + close(getDone) + }() + <-parent.started + + released := false + defer func() { + if !released { + close(parent.release) + } + }() + + otherShardDone := make(chan struct{}) + go func() { + if err := overlay.SetAccount((*[32]byte)(&keyB), &Account{Key: keyB, Lamports: 2}); err != nil { + panic(err) + } + close(otherShardDone) + }() + select { + case <-otherShardDone: + case <-time.After(time.Second): + t.Fatal("different-shard write blocked behind unrelated parent miss") + } + accountB, err := overlay.GetAccount((*[32]byte)(&keyB)) + require.NoError(t, err) + assert.Equal(t, uint64(2), accountB.Lamports) + + close(parent.release) + released = true + select { + case <-getDone: + case <-time.After(time.Second): + t.Fatal("parent fallback read did not finish") + } +} diff --git a/pkg/replay/block.go b/pkg/replay/block.go index ab2c6a96..e6c2fb25 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -348,34 +348,59 @@ func ResolveAddrTableLookupsForTx(ctx context.Context, accountsDb *accountsdb.Ac return tx.Message.ResolveLookups() } -func extractAndDedupeBlockAccts(block *b.Block) []solana.PublicKey { +const transactionPublicationNonTransactionSlack = 8 +const expectedTouchedAccountsPerTransaction = 2 + +func extractAndDedupeBlockAccts(block *b.Block) ([]solana.PublicKey, int) { var numPubkeys int for _, tx := range block.Transactions { numPubkeys += len(tx.Message.AccountKeys) } numPubkeys += len(block.UpdatedAccts) - - pubkeyMap := make(map[solana.PublicKey]struct{}, numPubkeys) + pubkeyMap := make(map[solana.PublicKey]bool, numPubkeys) for _, tx := range block.Transactions { - for _, pk := range tx.Message.AccountKeys { - pubkeyMap[pk] = struct{}{} + numStaticAccounts, numWritableLookupAccounts := messageAccountLayout(&tx.Message) + for idx, pk := range tx.Message.AccountKeys { + if messageAccountIsWritable(&tx.Message, idx, numStaticAccounts, numWritableLookupAccounts) { + pubkeyMap[pk] = true + } else if _, exists := pubkeyMap[pk]; !exists { + pubkeyMap[pk] = false + } } } for _, pk := range block.UpdatedAccts { - pubkeyMap[pk] = struct{}{} + pubkeyMap[pk] = true } pubkeys := make([]solana.PublicKey, len(pubkeyMap)) i := 0 - for pk := range pubkeyMap { + writableAccountCount := 0 + for pk, writable := range pubkeyMap { pubkeys[i] = pk i++ + if writable { + writableAccountCount++ + } } - return pubkeys + return pubkeys, writableAccountCount +} + +func publicationMapCapacity(block *b.Block, uniqueWritableAccounts int, alpenglow bool) int { + // This is an allocation hint, not a shard-count bound. Cap speculative + // transaction capacity at the observed transfer workload's touch rate so + // failure-heavy blocks do not eagerly allocate for every declared writable + // key; maps grow naturally for higher-fanout successful transactions. + expectedTransactionTouches := min(uniqueWritableAccounts, len(block.Transactions)*expectedTouchedAccountsPerTransaction) + capacity := expectedTransactionTouches + len(block.EpochUpdatedAccts) + transactionPublicationNonTransactionSlack + if alpenglow { + // Certificate-driven vote writes are absent from transaction message keys. + capacity += len(block.EpochStakesPerVoteAcct) + } + return capacity } func isNativeProgram(pubkey solana.PublicKey) bool { @@ -466,16 +491,17 @@ func validatePartitionedRewardsResume(slot uint64, epochRewards *sealevel.Sysvar ) } -func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.Block, epochSchedule *sealevel.SysvarEpochSchedule, alpenglowClock bool) (accounts.Accounts, accounts.Accounts, error) { +func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.Block, epochSchedule *sealevel.SysvarEpochSchedule, alpenglowClock bool) (accounts.Accounts, accounts.Accounts, int, error) { phaseStart := time.Now() err := resolveAddrTableLookups(accountsDb, block) metrics.GlobalBlockReplay.AccountLoader.AddressTableLookups.AddTimingSince(phaseStart) if err != nil { - return nil, nil, err + return nil, nil, 0, err } phaseStart = time.Now() - dedupedAccts := extractAndDedupeBlockAccts(block) + dedupedAccts, uniqueWritableAccounts := extractAndDedupeBlockAccts(block) + publicationCapacity := publicationMapCapacity(block, uniqueWritableAccounts, alpenglowClock) metrics.GlobalBlockReplay.AccountLoader.DedupeBlockAccounts.AddTimingSince(phaseStart) ctx := context.Background() phaseStart = time.Now() @@ -483,22 +509,22 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B metrics.GlobalBlockReplay.AccountLoader.SourceBatch.AddTimingSince(phaseStart) recordAccountLoaderBatchStats(&metrics.GlobalBlockReplay.AccountLoader, batchStats) if err != nil { - return nil, nil, err + return nil, nil, 0, err } phaseStart = time.Now() - numAccts := uint64(len(slotAccts)) - metrics.GlobalBlockReplay.AccountLoader.ParentAccounts = numAccts - parentAccts := accounts.NewMemAccountsWithLen(numAccts) + numAccts := len(slotAccts) + metrics.GlobalBlockReplay.AccountLoader.ParentAccounts = uint64(numAccts) + parentAccts := accounts.NewMemAccountsWithLen(uint64(numAccts)) for _, acct := range slotAccts { if err = parentAccts.SetAccountWithoutLock(acct.Key, acct); err != nil { - return nil, nil, err + return nil, nil, 0, err } } // accts is a branch-local overlay over the pristine parent snapshot; execution // copy-on-writes, so parentAccts stays pristine for LtHash "before" values. - accts := accounts.NewOverlayAccounts(parentAccts) + accts := accounts.NewOverlayAccountsWithSizing(parentAccts, numAccts, publicationCapacity) metrics.GlobalBlockReplay.AccountLoader.ParentMapBuild.AddTimingSince(phaseStart) phaseStart = time.Now() @@ -758,7 +784,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B } metrics.GlobalBlockReplay.AccountLoader.SysvarUpdates.AddTimingSince(phaseStart) - return accts, parentAccts, nil + return accts, parentAccts, publicationCapacity, nil } func recordAccountLoaderBatchStats(dst *metrics.AccountLoader, src accountsdb.BatchReadStats) { @@ -3226,7 +3252,11 @@ func EncodeSlotHashes(sysvar *sealevel.SysvarSlotHashes) []state.SlotHashEntry { return result } -func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Accounts, acctsDb *accountsdb.AccountsDb, tail unrootedState) *sealevel.SlotCtx { +func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Accounts, acctsDb *accountsdb.AccountsDb, tail unrootedState, accountMapCapacity int) *sealevel.SlotCtx { + writableMapCapacity := accountMapCapacity + if block.Features != nil && block.Features.IsActive(features.RemoveAccountsDeltaHash) { + writableMapCapacity = 0 + } slotCtx := &sealevel.SlotCtx{ Accounts: accts, ParentAccts: parentAccts, @@ -3238,8 +3268,8 @@ func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Ac NumSignatures: block.NumSignatures, AcctMapsMu: &sync.Mutex{}, - ModifiedAccts: make(map[solana.PublicKey]bool), - WritableAccts: make(map[solana.PublicKey]bool), + ModifiedAccts: make(map[solana.PublicKey]bool, accountMapCapacity), + WritableAccts: make(map[solana.PublicKey]bool, writableMapCapacity), Blockhash: block.Blockhash, LastBlockhash: block.LastBlockhash, @@ -3768,7 +3798,7 @@ func ProcessBlock( if tail != nil { blockSrc = tail } - accts, parentAccts, err := loadBlockAccountsAndUpdateSysvars(blockSrc, block, epochSchedule, alpenglowClock) + accts, parentAccts, accountMapCapacity, err := loadBlockAccountsAndUpdateSysvars(blockSrc, block, epochSchedule, alpenglowClock) loadAcctsRegion.End() if err != nil { panic(fmt.Sprintf("unable to load slot accounts and update sysvars: %s", err)) @@ -3776,7 +3806,7 @@ func ProcessBlock( metrics.GlobalBlockReplay.LoadBlockAccounts.AddTimingSince(start) slotCtxSetupStart := time.Now() - slotCtx := newSlotCtx(block, accts, parentAccts, acctsDb, tail) + slotCtx := newSlotCtx(block, accts, parentAccts, acctsDb, tail, accountMapCapacity) slotCtx.TraceCtx = ctx slotCtx.NumSignatures = executionPlan.processedSignatures metrics.GlobalBlockReplay.SlotCtxSetup.AddTimingSince(slotCtxSetupStart) diff --git a/pkg/replay/commit.go b/pkg/replay/commit.go index 81ee0ea1..0af6b696 100644 --- a/pkg/replay/commit.go +++ b/pkg/replay/commit.go @@ -18,14 +18,12 @@ func applySuccessfulTransactionState(slotCtx *sealevel.SlotCtx, execCtx *sealeve } recordMetrics := slotCtx != nil && slotCtx.Replay - if executionResult != nil { + if executionResult != nil && !accountsDeltaHashRemoved(slotCtx) { var writableStart time.Time if recordMetrics { writableStart = time.Now() } - for _, pk := range executionResult.WritableAccounts { - slotCtx.RecordWritableAcct(pk) - } + slotCtx.RecordWritableAccts(executionResult.WritableAccounts) if recordMetrics { metrics.GlobalBlockReplay.TxPublishRecordWritableAcct.AddTimingSince(writableStart) } diff --git a/pkg/replay/lean_writable_fastpath_test.go b/pkg/replay/lean_writable_fastpath_test.go index 5bcd4d79..0975280f 100644 --- a/pkg/replay/lean_writable_fastpath_test.go +++ b/pkg/replay/lean_writable_fastpath_test.go @@ -39,11 +39,35 @@ func TestLeanWritableFastPathOmitsExecutionResultWhenADHDisabled(t *testing.T) { assert.Nil(t, output.ExecutionResult, "ADH-disabled lean replay must not materialize writable result collections") require.NoError(t, ApplySuccessfulTransaction(slotCtx, output)) + assert.Empty(t, slotCtx.WritableAccts, "ADH-removed lean publication must not build the unused writable set") destAfter, err := slotCtx.GetAccount(txfixture.DestPubkey()) require.NoError(t, err) assert.Greater(t, destAfter.Lamports, destBefore.Lamports, "omitting the result must not skip touched-account publication") } +func TestRecordModifiedAcctOnlyTracksLegacyWritableState(t *testing.T) { + key := txfixture.DestPubkey() + + t.Run("accounts delta hash removed", func(t *testing.T) { + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) + + slotCtx.RecordModifiedAcct(key) + assert.Contains(t, slotCtx.ModifiedAccts, key) + assert.NotContains(t, slotCtx.WritableAccts, key) + }) + + t.Run("legacy accounts delta hash", func(t *testing.T) { + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + + slotCtx.RecordModifiedAcct(key) + assert.Contains(t, slotCtx.ModifiedAccts, key) + assert.Contains(t, slotCtx.WritableAccts, key) + }) +} + func TestLeanWritableFastPathPreservesExecutionResultWhenADHEnabled(t *testing.T) { slotCtx, cleanup := newCommitTestSlotCtx() defer cleanup() @@ -120,6 +144,9 @@ func TestLeanWritableFastPathKeepsRichResultWhenADHDisabled(t *testing.T) { assert.NotEmpty(t, output.ExecutionResult.AccountUpdates) assert.NotEmpty(t, output.ExecutionResult.WritableAccounts) assert.NotEmpty(t, output.ExecutionResult.WritableAccountSet) + require.NoError(t, ApplySuccessfulTransaction(slotCtx, output)) + assert.Empty(t, slotCtx.WritableAccts, "ADH-removed rich publication must not build the unused writable set") + assert.Contains(t, slotCtx.ModifiedAccts, txfixture.DestPubkey()) } func TestApplySuccessfulTransactionLegacyRecordsWritableUntouchedAccount(t *testing.T) { diff --git a/pkg/replay/publication_capacity_test.go b/pkg/replay/publication_capacity_test.go new file mode 100644 index 00000000..a171b7fa --- /dev/null +++ b/pkg/replay/publication_capacity_test.go @@ -0,0 +1,75 @@ +package replay + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtractAndDedupeBlockAcctsPublicationCapacity(t *testing.T) { + payer := publicationConcurrencyTestKey(1) + firstWritable := publicationConcurrencyTestKey(2) + secondWritable := publicationConcurrencyTestKey(3) + readonly := publicationConcurrencyTestKey(4) + updated := publicationConcurrencyTestKey(5) + lookupProgram := publicationConcurrencyTestKey(6) + dynamicReadonly := publicationConcurrencyTestKey(7) + tableID := publicationConcurrencyTestKey(8) + + newTransaction := func(writable solana.PublicKey) *solana.Transaction { + return &solana.Transaction{Message: solana.Message{ + Header: solana.MessageHeader{ + NumRequiredSignatures: 1, + NumReadonlyUnsignedAccounts: 1, + }, + AccountKeys: []solana.PublicKey{payer, writable, readonly}, + }} + } + + resolvedTransaction := &solana.Transaction{Message: solana.Message{ + Header: solana.MessageHeader{ + NumRequiredSignatures: 1, + NumReadonlyUnsignedAccounts: 1, + }, + AccountKeys: []solana.PublicKey{payer, lookupProgram}, + }} + resolvedTransaction.Message.SetAddressTableLookups([]solana.MessageAddressTableLookup{{ + AccountKey: tableID, + WritableIndexes: []byte{0}, + ReadonlyIndexes: []byte{1}, + }}) + require.NoError(t, resolvedTransaction.Message.SetAddressTables(map[solana.PublicKey]solana.PublicKeySlice{ + tableID: {readonly, dynamicReadonly}, + })) + require.NoError(t, resolvedTransaction.Message.ResolveLookups()) + assert.ElementsMatch(t, []solana.PublicKey{payer, readonly}, messageWritableAccounts(&resolvedTransaction.Message)) + + block := &b.Block{ + Transactions: []*solana.Transaction{ + newTransaction(firstWritable), + newTransaction(secondWritable), + resolvedTransaction, + }, + UpdatedAccts: []solana.PublicKey{updated}, + EpochUpdatedAccts: make([]*accounts.Account, 2), + EpochStakesPerVoteAcct: make(map[solana.PublicKey]uint64, 100), + } + for idx := uint64(0); idx < 100; idx++ { + block.EpochStakesPerVoteAcct[publicationConcurrencyTestKey(100+idx)] = idx + } + + pubkeys, uniqueWritableAccounts := extractAndDedupeBlockAccts(block) + require.Len(t, pubkeys, 7) + assert.ElementsMatch(t, []solana.PublicKey{ + payer, firstWritable, secondWritable, readonly, updated, lookupProgram, dynamicReadonly, + }, pubkeys) + assert.Equal(t, 5, uniqueWritableAccounts, "readonly key must upgrade to writable across messages") + nonTransactionCapacity := len(block.EpochUpdatedAccts) + transactionPublicationNonTransactionSlack + assert.Equal(t, 5+nonTransactionCapacity, publicationMapCapacity(block, uniqueWritableAccounts, false)) + assert.Equal(t, 5+nonTransactionCapacity+len(block.EpochStakesPerVoteAcct), publicationMapCapacity(block, uniqueWritableAccounts, true)) + assert.Equal(t, len(block.Transactions)*expectedTouchedAccountsPerTransaction+nonTransactionCapacity, publicationMapCapacity(block, 20, false)) +} diff --git a/pkg/replay/publication_concurrency_test.go b/pkg/replay/publication_concurrency_test.go new file mode 100644 index 00000000..7e3fd7e6 --- /dev/null +++ b/pkg/replay/publication_concurrency_test.go @@ -0,0 +1,163 @@ +package replay + +import ( + "encoding/binary" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func publicationConcurrencyTestKey(n uint64) solana.PublicKey { + var key solana.PublicKey + binary.LittleEndian.PutUint64(key[:8], n) + binary.LittleEndian.PutUint64(key[8:16], n*0x9e3779b97f4a7c15) + return key +} + +func TestConcurrentDisjointTransactionPublication(t *testing.T) { + const ( + workers = 16 + batchesPerWorker = 128 + accountsPerBatch = 2 + ) + totalAccounts := workers * batchesPerWorker * accountsPerBatch + feats := features.NewFeaturesDefault() + feats.EnableFeature(features.RemoveAccountsDeltaHash, 0) + + parent := accounts.NewMemAccounts() + overlay := accounts.NewOverlayAccountsWithLen(parent, totalAccounts) + slotCtx := &sealevel.SlotCtx{ + Accounts: overlay, + Features: feats, + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool, totalAccounts), + WritableAccts: make(map[solana.PublicKey]bool), + } + + start := make(chan struct{}) + errs := make(chan error, workers) + var wg sync.WaitGroup + wg.Add(workers) + for worker := range workers { + go func(worker int) { + defer wg.Done() + <-start + for batchIdx := range batchesPerWorker { + first := uint64((worker*batchesPerWorker+batchIdx)*accountsPerBatch + 1) + accountStates := []*accounts.Account{ + {Key: publicationConcurrencyTestKey(first), Lamports: first}, + {Key: publicationConcurrencyTestKey(first + 1), Lamports: first + 1}, + } + execCtx := &sealevel.ExecutionCtx{ + Features: *feats, + TransactionContext: &sealevel.TransactionCtx{ + Accounts: sealevel.TransactionAccounts{ + Accounts: accountStates, + Touched: []bool{true, true}, + }, + }, + } + if err := applySuccessfulTransactionState(slotCtx, execCtx, nil); err != nil { + errs <- err + return + } + } + }(worker) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + assert.Empty(t, slotCtx.WritableAccts, "ADH-removed replay must not build the unused writable set") + require.Len(t, slotCtx.ModifiedAccts, totalAccounts) + require.Len(t, overlay.DeltaAccounts(), totalAccounts) + for key := range slotCtx.ModifiedAccts { + acct, err := overlay.GetAccount((*[32]byte)(&key)) + require.NoError(t, err) + assert.Equal(t, binary.LittleEndian.Uint64(key[:8]), acct.Lamports) + } +} + +func TestConcurrentDisjointLegacyTransactionPublication(t *testing.T) { + const ( + workers = 8 + batchesPerWorker = 64 + accountsPerBatch = 2 + ) + totalAccounts := workers * batchesPerWorker * accountsPerBatch + feats := features.NewFeaturesDefault() + + parent := accounts.NewMemAccounts() + overlay := accounts.NewOverlayAccountsWithLen(parent, totalAccounts) + slotCtx := &sealevel.SlotCtx{ + Accounts: overlay, + Features: feats, + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool, totalAccounts), + WritableAccts: make(map[solana.PublicKey]bool, totalAccounts), + } + + start := make(chan struct{}) + errs := make(chan error, workers) + var wg sync.WaitGroup + wg.Add(workers) + for worker := range workers { + go func(worker int) { + defer wg.Done() + <-start + for batchIdx := range batchesPerWorker { + first := uint64((worker*batchesPerWorker+batchIdx)*accountsPerBatch + 1) + firstKey := publicationConcurrencyTestKey(first) + secondKey := publicationConcurrencyTestKey(first + 1) + accountStates := []*accounts.Account{ + {Key: firstKey, Lamports: first}, + {Key: secondKey, Lamports: first + 1}, + } + execCtx := &sealevel.ExecutionCtx{ + Features: *feats, + TransactionContext: &sealevel.TransactionCtx{ + Accounts: sealevel.TransactionAccounts{ + Accounts: accountStates, + Touched: []bool{true, true}, + }, + }, + } + executionResult := &TransactionExecutionResult{ + WritableAccounts: []solana.PublicKey{firstKey, secondKey}, + WritableAccountSet: map[solana.PublicKey]struct{}{ + firstKey: {}, + secondKey: {}, + }, + } + if err := applySuccessfulTransactionState(slotCtx, execCtx, executionResult); err != nil { + errs <- err + return + } + } + }(worker) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + require.Len(t, slotCtx.WritableAccts, totalAccounts) + require.Len(t, slotCtx.ModifiedAccts, totalAccounts) + require.Len(t, overlay.DeltaAccounts(), totalAccounts) + for key := range slotCtx.ModifiedAccts { + acct, err := overlay.GetAccount((*[32]byte)(&key)) + require.NoError(t, err) + assert.Equal(t, binary.LittleEndian.Uint64(key[:8]), acct.Lamports) + } +} diff --git a/pkg/replay/topsort_planner.go b/pkg/replay/topsort_planner.go index 0caa3c88..da5b9db6 100644 --- a/pkg/replay/topsort_planner.go +++ b/pkg/replay/topsort_planner.go @@ -25,28 +25,33 @@ func canDeriveAccountsFromMessage(t *solana.Transaction) bool { return !t.Message.IsVersioned() || t.Message.AddressTableLookups.NumLookups() == 0 } -func messageWritableAccounts(msg *solana.Message) []solana.PublicKey { - numStaticAccounts := len(msg.AccountKeys) - numWritableLookupAccounts := 0 +func messageAccountLayout(msg *solana.Message) (numStaticAccounts, numWritableLookupAccounts int) { + numStaticAccounts = len(msg.AccountKeys) if msg.IsResolved() { numStaticAccounts -= msg.NumLookups() numWritableLookupAccounts = msg.GetAddressTableLookups().NumWritableLookups() } + return numStaticAccounts, numWritableLookupAccounts +} + +func messageAccountIsWritable(msg *solana.Message, idx, numStaticAccounts, numWritableLookupAccounts int) bool { + switch { + case idx >= numStaticAccounts: + return idx-numStaticAccounts < numWritableLookupAccounts + case idx >= int(msg.Header.NumRequiredSignatures): + numUnsignedWritable := (numStaticAccounts - int(msg.Header.NumRequiredSignatures)) - int(msg.Header.NumReadonlyUnsignedAccounts) + return idx-int(msg.Header.NumRequiredSignatures) < numUnsignedWritable + default: + return idx < int(msg.Header.NumRequiredSignatures-msg.Header.NumReadonlySignedAccounts) + } +} + +func messageWritableAccounts(msg *solana.Message) []solana.PublicKey { + numStaticAccounts, numWritableLookupAccounts := messageAccountLayout(msg) accounts := make([]solana.PublicKey, 0, len(msg.AccountKeys)) for idx, account := range msg.AccountKeys { - isWritable := false - switch { - case idx >= numStaticAccounts: - isWritable = idx-numStaticAccounts < numWritableLookupAccounts - case idx >= int(msg.Header.NumRequiredSignatures): - numUnsignedWritable := (numStaticAccounts - int(msg.Header.NumRequiredSignatures)) - int(msg.Header.NumReadonlyUnsignedAccounts) - isWritable = idx-int(msg.Header.NumRequiredSignatures) < numUnsignedWritable - default: - isWritable = idx < int(msg.Header.NumRequiredSignatures-msg.Header.NumReadonlySignedAccounts) - } - - if isWritable { + if messageAccountIsWritable(msg, idx, numStaticAccounts, numWritableLookupAccounts) { accounts = append(accounts, account) } } diff --git a/pkg/replay/transaction.go b/pkg/replay/transaction.go index e3ba94a0..0d15b2c6 100644 --- a/pkg/replay/transaction.go +++ b/pkg/replay/transaction.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "errors" "fmt" - "math" "runtime/trace" "strings" "sync" @@ -211,26 +210,19 @@ type transactionPublicationStats struct { func handleModifiedAccounts(slotCtx *sealevel.SlotCtx, execCtx *sealevel.ExecutionCtx) transactionPublicationStats { // update account states in slotCtx for all accounts 'touched' during the tx's execution + transactionAccounts := &execCtx.TransactionContext.Accounts var stats transactionPublicationStats - for idx, newAcctState := range execCtx.TransactionContext.Accounts.Accounts { - if execCtx.TransactionContext.Accounts.Touched[idx] { + for idx, newAcctState := range transactionAccounts.Accounts { + if transactionAccounts.Touched[idx] { // Track touched account stats for profiling stats.touchedAccounts++ stats.touchedAccountBytes += uint64(len(newAcctState.Data)) - - // clean up accounts closed during the tx (garbage collection) - if newAcctState.Lamports == 0 { - newAcctState = &accounts.Account{Key: newAcctState.Key, RentEpoch: math.MaxUint64} - } - - err := slotCtx.SetAccount(newAcctState.Key, newAcctState) - if err != nil { - panic(fmt.Sprintf("unable to set slot account for %s to update state: %s", newAcctState.Key, err)) - } - slotCtx.RecordModifiedAcct(newAcctState.Key) - //mlog.Log.Debugf("modified account %s after tx", newAcctState.Key) } } + if err := accounts.SetTransactionAccounts(slotCtx.Accounts, transactionAccounts.Accounts, transactionAccounts.Touched); err != nil { + panic(fmt.Sprintf("unable to publish transaction account states: %s", err)) + } + slotCtx.RecordModifiedAccountStates(transactionAccounts.Accounts, transactionAccounts.Touched) // Record touched stats for clone optimization profiling TxAcctsTouched.Add(stats.touchedAccounts) diff --git a/pkg/sealevel/execution_ctx.go b/pkg/sealevel/execution_ctx.go index cba69036..d9d6feec 100644 --- a/pkg/sealevel/execution_ctx.go +++ b/pkg/sealevel/execution_ctx.go @@ -469,12 +469,35 @@ func (slotCtx *SlotCtx) SetAccount(pubkey solana.PublicKey, acct *accounts.Accou func (slotCtx *SlotCtx) RecordModifiedAcct(pubkey solana.PublicKey) { slotCtx.AcctMapsMu.Lock() defer slotCtx.AcctMapsMu.Unlock() - slotCtx.WritableAccts[pubkey] = true + if slotCtx.Features == nil || !slotCtx.Features.IsActive(features.RemoveAccountsDeltaHash) { + slotCtx.WritableAccts[pubkey] = true + } slotCtx.ModifiedAccts[pubkey] = true } +func (slotCtx *SlotCtx) RecordModifiedAccountStates(accountStates []*accounts.Account, touched []bool) { + if len(accountStates) != len(touched) { + panic("account states/touched length mismatch") + } + slotCtx.AcctMapsMu.Lock() + defer slotCtx.AcctMapsMu.Unlock() + for idx, acct := range accountStates { + if touched[idx] { + slotCtx.ModifiedAccts[acct.Key] = true + } + } +} + func (slotCtx *SlotCtx) RecordWritableAcct(pubkey solana.PublicKey) { slotCtx.AcctMapsMu.Lock() defer slotCtx.AcctMapsMu.Unlock() slotCtx.WritableAccts[pubkey] = true } + +func (slotCtx *SlotCtx) RecordWritableAccts(pubkeys []solana.PublicKey) { + slotCtx.AcctMapsMu.Lock() + defer slotCtx.AcctMapsMu.Unlock() + for _, pubkey := range pubkeys { + slotCtx.WritableAccts[pubkey] = true + } +} From b0239a4e32b034c3e6a5e0e7b9baf4eb36a6f457 Mon Sep 17 00:00:00 2001 From: smcio Date: Sat, 25 Jul 2026 08:59:27 +0200 Subject: [PATCH 4/5] various small perf enhancements --- cmd/mithril/node/node.go | 7 +- pkg/accountsdb/accountsdb.go | 159 ++++++++++++++++++--- pkg/accountsdb/batch.go | 64 ++++++--- pkg/accountsdb/batch_test.go | 84 +++++++++++ pkg/accountsdb/fold.go | 34 +++-- pkg/accountsdb/fold_test.go | 47 ++++++ pkg/accountsdb/segment.go | 76 ++++++++++ pkg/bankhash/bankhash.go | 12 +- pkg/bankhash/lthash.go | 12 +- pkg/bankhash/lthash_optimized_test.go | 7 + pkg/bankhash/lthash_test.go | 12 ++ pkg/metrics/metrics.go | 46 ++++-- pkg/replay/block.go | 82 +++++++++-- pkg/replay/leader_finalize.go | 66 ++++++++- pkg/replay/lean_writable_fastpath_test.go | 28 ++++ pkg/replay/promotion.go | 47 +++++- pkg/replay/publication_concurrency_test.go | 16 ++- pkg/sealevel/execution_ctx.go | 22 ++- 18 files changed, 725 insertions(+), 96 deletions(-) diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index 43d91e77..43b2faaf 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -3834,7 +3834,10 @@ func addTransactionStatusManifestRef(keep map[string]*state.TransactionStatusChe if len(manifest.ResumeCtx) == 0 { return fmt.Errorf("fold manifest through slot %d carries no resume context", manifest.ThroughSlot) } - var ctx state.ResumeContext + var ctx struct { + Slot uint64 `json:"slot"` + TransactionStatusCheckpoint *state.TransactionStatusCheckpointRef `json:"transaction_status_checkpoint,omitempty"` + } if err := json.Unmarshal(manifest.ResumeCtx, &ctx); err != nil { return fmt.Errorf("decode fold manifest context through slot %d: %w", manifest.ThroughSlot, err) } @@ -3875,7 +3878,7 @@ func retainedTransactionStatusCheckpointRefs(accountsDb *accountsdb.AccountsDb, start = len(headers) - int(retainCount) } for _, header := range headers[start:] { - manifest, err := accountsdb.ReadSegmentManifest(header.Path) + manifest, err := accountsdb.ReadSegmentManifestContext(header.Path) if err != nil { return nil, fmt.Errorf("read in-horizon fold manifest %s: %w", header.Path, err) } diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index 47e76b90..49714f73 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -14,6 +14,7 @@ import ( "runtime/trace" "sync" "sync/atomic" + "time" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/addresses" @@ -37,12 +38,14 @@ type AccountsDb struct { // the write side; hot-path program cache operations take the read side. programCacheMu sync.RWMutex - // readCacheEpochMu makes the authoritative index flip and its cache refresh - // one publication boundary. Batch readers retain the epoch captured with - // their Pebble snapshot and may only admit decoded values while it still - // matches, preventing an old appendvec read from overwriting a newer fold. + // readCacheEpochMu protects the cache epoch and the pending fold view. + // CommitBatch publishes its immutable newest-wins union here before the + // Pebble index commit. Readers use that view for changed keys while the + // index and caches catch up, so the expensive durable commit does not hold + // this mutex and old snapshot reads still cannot publish stale cache bytes. readCacheEpochMu sync.RWMutex readCacheEpoch uint64 + pendingFold map[[32]byte]dedupedVersion commonAdmission *commonCacheAdmission batchHooks batchReadTestHooks @@ -317,6 +320,27 @@ func (accountsDb *AccountsDb) RemoveProgramFromCache(pubkey solana.PublicKey) { accountsDb.ProgramCache.Delete(pubkey) } +// AccountReadStats is one single-key read's exact wall-time decomposition. +// The sysvar loader records these separately from its decode/update work so a +// fold publication wait cannot hide inside the broad SysvarUpdates timer. +type AccountReadStats struct { + WorkingSetLookupNanoseconds uint64 + CloneNanoseconds uint64 + AppendVecPinWaitNanoseconds uint64 + InProgressNanoseconds uint64 + ReadCacheEpochWaitNanoseconds uint64 + CacheLookupNanoseconds uint64 + IndexAndAppendVecReadNanoseconds uint64 + CachePublicationWaitNanoseconds uint64 + CachePublicationNanoseconds uint64 + WorkingSetHit bool + InProgressHit bool + PendingFoldHit bool + CacheHit bool + DurableRead bool + CachePublicationEpochRejected bool +} + func (accountsDb *AccountsDb) GetAccount(slot uint64, pubkey solana.PublicKey) (*accounts.Account, error) { if accountsDb == nil { return nil, ErrNoAccount @@ -330,6 +354,26 @@ func (accountsDb *AccountsDb) GetAccount(slot uint64, pubkey solana.PublicKey) ( return accountsDb.getStoredAccountPinned(slot, pubkey) } +func (accountsDb *AccountsDb) GetAccountWithStats(slot uint64, pubkey solana.PublicKey) (*accounts.Account, AccountReadStats, error) { + var stats AccountReadStats + if accountsDb == nil { + return nil, stats, ErrNoAccount + } + start := time.Now() + accountsDb.appendVecReadMu.RLock() + stats.AppendVecPinWaitNanoseconds = uint64(time.Since(start).Nanoseconds()) + defer accountsDb.appendVecReadMu.RUnlock() + start = time.Now() + accts := accountsDb.getStoreInProgressAccounts([]solana.PublicKey{pubkey}) + stats.InProgressNanoseconds = uint64(time.Since(start).Nanoseconds()) + if accts[0] != nil { + stats.InProgressHit = true + return accts[0], stats, nil + } + acct, err := accountsDb.getStoredAccountPinnedWithStats(slot, pubkey, &stats) + return acct, stats, err +} + func (accountsDb *AccountsDb) getStoredAccount(slot uint64, pubkey solana.PublicKey) (*accounts.Account, error) { accountsDb.appendVecReadMu.RLock() defer accountsDb.appendVecReadMu.RUnlock() @@ -350,6 +394,46 @@ func (accountsDb *AccountsDb) getStoredAccountPinned(slot uint64, pubkey solana. r.End() defer trace.StartRegion(context.Background(), "GetStoredAccountDisk").End() + acct, err := accountsDb.readIndexedAccount(pubkey) + if err == ErrNoAccount { + return nil, ErrNoAccount + } + if err != nil { + if acct, err = accountsDb.readIndexedAccount(pubkey); err != nil { + if err == ErrNoAccount { + return nil, ErrNoAccount + } + return nil, fmt.Errorf("accountsdb: read %s failed after retry: %w", pubkey, err) + } + } + accountsDb.cacheReadAccount(pubkey, acct, cacheEpoch) + return acct, nil +} + +func (accountsDb *AccountsDb) getStoredAccountPinnedWithStats(slot uint64, pubkey solana.PublicKey, stats *AccountReadStats) (*accounts.Account, error) { + if accountsDb.Index == nil { + return nil, ErrNoAccount + } + r := trace.StartRegion(context.Background(), "GetStoredAccountCache") + waitStart := time.Now() + accountsDb.readCacheEpochMu.RLock() + stats.ReadCacheEpochWaitNanoseconds += uint64(time.Since(waitStart).Nanoseconds()) + cacheStart := time.Now() + cachedAcct, hasAcct, pending := accountsDb.getCachedAccountLocked(pubkey) + cacheEpoch := accountsDb.readCacheEpoch + accountsDb.readCacheEpochMu.RUnlock() + stats.CacheLookupNanoseconds += uint64(time.Since(cacheStart).Nanoseconds()) + if hasAcct { + stats.PendingFoldHit = pending + stats.CacheHit = !pending + r.End() + return cachedAcct, nil + } + r.End() + + defer trace.StartRegion(context.Background(), "GetStoredAccountDisk").End() + stats.DurableRead = true + readStart := time.Now() // One-shot retry preserves the single-account path's existing tolerance for // an externally removed or stale index location. The appendvec reader pin @@ -366,8 +450,13 @@ func (accountsDb *AccountsDb) getStoredAccountPinned(slot uint64, pubkey solana. return nil, fmt.Errorf("accountsdb: read %s failed after retry: %w", pubkey, err) } } + stats.IndexAndAppendVecReadNanoseconds += uint64(time.Since(readStart).Nanoseconds()) - accountsDb.cacheReadAccount(pubkey, acct, cacheEpoch) + publicationStart := time.Now() + waitNanoseconds, rejected := accountsDb.cacheReadAccountWithStats(pubkey, acct, cacheEpoch) + stats.CachePublicationNanoseconds += uint64(time.Since(publicationStart).Nanoseconds()) + stats.CachePublicationWaitNanoseconds += waitNanoseconds + stats.CachePublicationEpochRejected = rejected return acct, nil } @@ -375,37 +464,55 @@ func (accountsDb *AccountsDb) getStoredAccountPinned(slot uint64, pubkey solana. func (accountsDb *AccountsDb) getCachedAccount(pubkey solana.PublicKey) (*accounts.Account, bool) { accountsDb.readCacheEpochMu.RLock() defer accountsDb.readCacheEpochMu.RUnlock() - return accountsDb.getCachedAccountLocked(pubkey) + acct, ok, _ := accountsDb.getCachedAccountLocked(pubkey) + return acct, ok } func (accountsDb *AccountsDb) getCachedAccountAndEpoch(pubkey solana.PublicKey) (*accounts.Account, bool, uint64) { accountsDb.readCacheEpochMu.RLock() defer accountsDb.readCacheEpochMu.RUnlock() - acct, ok := accountsDb.getCachedAccountLocked(pubkey) + acct, ok, _ := accountsDb.getCachedAccountLocked(pubkey) return acct, ok, accountsDb.readCacheEpoch } // getCachedAccountLocked requires readCacheEpochMu to be held for reading or // writing. Keeping a whole batch's cache probes under one epoch makes its // cache results coherent with the Pebble snapshot created at that boundary. -func (accountsDb *AccountsDb) getCachedAccountLocked(pubkey solana.PublicKey) (*accounts.Account, bool) { +func (accountsDb *AccountsDb) getCachedAccountLocked(pubkey solana.PublicKey) (*accounts.Account, bool, bool) { + if version, ok := accountsDb.pendingFold[[32]byte(pubkey)]; ok { + return version.acct, true, true + } if acct, ok := accountsDb.VoteAcctCache.Get(pubkey); ok { - return acct, true + return acct, true, false } - return accountsDb.CommonAcctsCache.Get(pubkey) + acct, ok := accountsDb.CommonAcctsCache.Get(pubkey) + return acct, ok, false } func (accountsDb *AccountsDb) cacheReadAccount(pubkey solana.PublicKey, acct *accounts.Account, expectedEpoch uint64) { accountsDb.readCacheEpochMu.RLock() defer accountsDb.readCacheEpochMu.RUnlock() - if expectedEpoch != accountsDb.readCacheEpoch { - return + accountsDb.cacheReadAccountLocked(pubkey, acct, expectedEpoch) +} + +func (accountsDb *AccountsDb) cacheReadAccountWithStats(pubkey solana.PublicKey, acct *accounts.Account, expectedEpoch uint64) (uint64, bool) { + waitStart := time.Now() + accountsDb.readCacheEpochMu.RLock() + waitNanoseconds := uint64(time.Since(waitStart).Nanoseconds()) + defer accountsDb.readCacheEpochMu.RUnlock() + return waitNanoseconds, !accountsDb.cacheReadAccountLocked(pubkey, acct, expectedEpoch) +} + +func (accountsDb *AccountsDb) cacheReadAccountLocked(pubkey solana.PublicKey, acct *accounts.Account, expectedEpoch uint64) bool { + if expectedEpoch != accountsDb.readCacheEpoch || accountsDb.pendingFoldContainsLocked(pubkey) { + return false } if solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr { accountsDb.VoteAcctCache.Set(pubkey, acct) } else { accountsDb.CommonAcctsCache.Set(pubkey, acct) } + return true } type batchCacheAdmission uint8 @@ -428,25 +535,32 @@ func (accountsDb *AccountsDb) cacheBatchReadAccount( acct *accounts.Account, admitCommon bool, expectedEpoch uint64, -) batchCacheAdmission { +) (batchCacheAdmission, uint64) { if hook := accountsDb.batchHooks.beforeCacheAdmission; hook != nil { hook(pubkey) } + waitStart := time.Now() accountsDb.readCacheEpochMu.RLock() + waitNanoseconds := uint64(time.Since(waitStart).Nanoseconds()) defer accountsDb.readCacheEpochMu.RUnlock() - if expectedEpoch != accountsDb.readCacheEpoch { - return batchCacheEpochRejected + if expectedEpoch != accountsDb.readCacheEpoch || accountsDb.pendingFoldContainsLocked(pubkey) { + return batchCacheEpochRejected, waitNanoseconds } if solana.PublicKeyFromBytes(acct.Owner[:]) == addresses.VoteProgramAddr { if accountsDb.VoteAcctCache.Set(pubkey, acct) { - return batchCacheVote + return batchCacheVote, waitNanoseconds } - return batchCacheVoteSkipped + return batchCacheVoteSkipped, waitNanoseconds } if admitCommon && accountsDb.CommonAcctsCache.Set(pubkey, acct) { - return batchCacheCommon + return batchCacheCommon, waitNanoseconds } - return batchCacheCommonSkipped + return batchCacheCommonSkipped, waitNanoseconds +} + +func (accountsDb *AccountsDb) pendingFoldContainsLocked(pubkey solana.PublicKey) bool { + _, ok := accountsDb.pendingFold[[32]byte(pubkey)] + return ok } // readIndexedAccount performs one index-fetch + file-read attempt. @@ -576,10 +690,13 @@ func (accountsDb *AccountsDb) refreshReadCaches(accts []*accounts.Account) { accountsDb.readCacheEpochMu.Lock() defer accountsDb.readCacheEpochMu.Unlock() accountsDb.readCacheEpoch++ - accountsDb.refreshReadCachesLocked(accts) + accountsDb.refreshReadCacheEntries(accts) } -func (accountsDb *AccountsDb) refreshReadCachesLocked(accts []*accounts.Account) { +// refreshReadCacheEntries uses only concurrent-safe ordinary Otter operations. +// CommitBatch may call it without readCacheEpochMu while pendingFold masks every +// changed key; Clear remains confined to resetReadCachesLocked. +func (accountsDb *AccountsDb) refreshReadCacheEntries(accts []*accounts.Account) { for _, acct := range accts { if acct == nil { continue diff --git a/pkg/accountsdb/batch.go b/pkg/accountsdb/batch.go index 30d576e2..77c259a5 100644 --- a/pkg/accountsdb/batch.go +++ b/pkg/accountsdb/batch.go @@ -68,6 +68,7 @@ type BatchReadStats struct { WorkingSetHits uint64 InProgressHits uint64 + PendingFoldHits uint64 CacheHits uint64 IndexHits uint64 IndexMisses uint64 @@ -88,15 +89,17 @@ type BatchReadStats struct { DecodedAccountBytes uint64 PlaceholderObjects uint64 - WorkingSetLookupNanoseconds uint64 - InProgressNanoseconds uint64 - AppendVecPinWaitNanoseconds uint64 - CacheLookupNanoseconds uint64 - AdmissionFilterNanoseconds uint64 - IndexLookupNanoseconds uint64 - ReadPlanningNanoseconds uint64 - AppendVecReadNanoseconds uint64 - CachePublicationNanoseconds uint64 + WorkingSetLookupNanoseconds uint64 + InProgressNanoseconds uint64 + AppendVecPinWaitNanoseconds uint64 + ReadCacheEpochWaitNanoseconds uint64 + CacheLookupNanoseconds uint64 + AdmissionFilterNanoseconds uint64 + IndexLookupNanoseconds uint64 + ReadPlanningNanoseconds uint64 + AppendVecReadNanoseconds uint64 + CachePublicationWaitNanoseconds uint64 + CachePublicationNanoseconds uint64 } type batchChunkReadStats struct { @@ -149,13 +152,15 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 phaseStart = time.Now() out := db.getStoreInProgressAccounts(pks) stats.InProgressNanoseconds = uint64(time.Since(phaseStart).Nanoseconds()) - phaseStart = time.Now() cold := make([]int, 0, len(pks)) var indexSnapshot *pebble.Snapshot var cacheEpoch uint64 var admission *commonCacheAdmission var snapshotSetupNanoseconds uint64 + waitStart := time.Now() db.readCacheEpochMu.RLock() + stats.ReadCacheEpochWaitNanoseconds = uint64(time.Since(waitStart).Nanoseconds()) + phaseStart = time.Now() cacheEpoch = db.readCacheEpoch admission = db.commonAdmission for i, pk := range pks { @@ -163,8 +168,12 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 stats.InProgressHits++ continue } - if acct, ok := db.getCachedAccountLocked(pk); ok { - stats.CacheHits++ + if acct, ok, pending := db.getCachedAccountLocked(pk); ok { + if pending { + stats.PendingFoldHits++ + } else { + stats.CacheHits++ + } if acct == nil || acct.Lamports == 0 { stats.PlaceholderObjects++ } @@ -304,15 +313,17 @@ func (db *AccountsDb) getAccountsBatchWithStats(ctx context.Context, slot uint64 publicationJobs = append(publicationJobs, idx) } publicationResults := make([]batchCacheAdmission, len(publicationJobs)) + publicationWaits := make([]uint64, len(publicationJobs)) err = runBatchWorkers(ctx, len(publicationJobs), func(job int) error { idx := publicationJobs[job] acct := decodedForCache[idx] - publicationResults[job] = db.cacheBatchReadAccount( + publicationResults[job], publicationWaits[job] = db.cacheBatchReadAccount( pks[idx], acct, admitCommon[idx], cacheEpoch, ) return nil }) - for _, result := range publicationResults { + for idx, result := range publicationResults { + stats.CachePublicationWaitNanoseconds += publicationWaits[idx] switch result { case batchCacheVote: stats.VoteCacheAdmissions++ @@ -417,13 +428,19 @@ func resolveBatchAccountLocationsIterators( workerCount = min(len(cold), max(1, workerCount)) g, workerCtx := errgroup.WithContext(ctx) + upperBounds := make([][32]byte, workerCount) for worker := range workerCount { start := len(cold) * worker / workerCount end := len(cold) * (worker + 1) / workerCount g.Go(func() (retErr error) { - iter, err := snapshot.NewIterWithContext(workerCtx, &pebble.IterOptions{ - KeyTypes: pebble.IterKeyTypePointsOnly, - }) + options := pebble.IterOptions{ + KeyTypes: pebble.IterKeyTypePointsOnly, + LowerBound: pks[cold[start]][:], + } + if nextPubkey(pks[cold[end-1]], &upperBounds[worker]) { + options.UpperBound = upperBounds[worker][:] + } + iter, err := snapshot.NewIterWithContext(workerCtx, &options) if err != nil { return fmt.Errorf("create account index iterator: %w", err) } @@ -469,6 +486,19 @@ func resolveBatchAccountLocationsIterators( return locations, found, nil } +// nextPubkey returns the smallest 32-byte key strictly greater than key. A +// false result means key is the maximal all-0xff value and needs no upper bound. +func nextPubkey(key solana.PublicKey, out *[32]byte) bool { + *out = key + for idx := len(out) - 1; idx >= 0; idx-- { + out[idx]++ + if out[idx] != 0 { + return true + } + } + return false +} + func readBatchAccountAt(file *os.File, path string, location batchAccountLocation) (*accounts.Account, error) { if location.entry.Offset > math.MaxInt64 { return nil, fmt.Errorf("account offset %d overflows int64", location.entry.Offset) diff --git a/pkg/accountsdb/batch_test.go b/pkg/accountsdb/batch_test.go index 709b5a5a..2eafa856 100644 --- a/pkg/accountsdb/batch_test.go +++ b/pkg/accountsdb/batch_test.go @@ -178,6 +178,26 @@ func TestGetAccountsBatchIteratorExactMatchOrderAndInputImmutability(t *testing. assert.Equal(t, out[count+2].Lamports, out[count+3].Lamports) } +func TestNextPubkey(t *testing.T) { + key := solana.PublicKey{1, 2, 3} + var next [32]byte + require.True(t, nextPubkey(key, &next)) + assert.Equal(t, byte(1), next[0]) + assert.Equal(t, byte(1), next[len(next)-1]) + + key = solana.PublicKey{} + key[len(key)-2] = 7 + key[len(key)-1] = 0xff + require.True(t, nextPubkey(key, &next)) + assert.Equal(t, byte(8), next[len(next)-2]) + assert.Zero(t, next[len(next)-1]) + + for idx := range key { + key[idx] = 0xff + } + assert.False(t, nextPubkey(key, &next)) +} + func TestGetAccountsBatchRejectsCancelledContextAndMalformedIndex(t *testing.T) { db, _ := newFoldTestDb(t) defer db.CloseDb() @@ -250,6 +270,70 @@ func TestBatchCacheAdmissionCannotOverwriteNewerFold(t *testing.T) { } } +func TestFoldPublicationServesNewValuesWithoutBlockingReaders(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + + old := foldAcct(1, 10, []byte{1}) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 100, Delta: []*accounts.Account{old}}}, 100, nil, nil) + require.NoError(t, err) + require.True(t, db.CommonAcctsCache.Set(old.Key, old)) + + publicationStarted := make(chan struct{}) + releaseCommit := make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(releaseCommit) }) + db.foldHooks.afterPublicationStart = func() { + close(publicationStarted) + <-releaseCommit + } + + updated := foldAcct(1, 20, []byte{2}) + commitDone := make(chan error, 1) + go func() { + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 101, Delta: []*accounts.Account{updated}}}, 101, nil, nil) + commitDone <- err + }() + select { + case <-publicationStarted: + case <-time.After(5 * time.Second): + t.Fatal("fold did not publish its pending view") + } + + type singleResult struct { + acct *accounts.Account + stats AccountReadStats + err error + } + singleDone := make(chan singleResult, 1) + go func() { + acct, stats, err := db.GetAccountWithStats(101, old.Key) + singleDone <- singleResult{acct: acct, stats: stats, err: err} + }() + select { + case result := <-singleDone: + require.NoError(t, result.err) + assert.Equal(t, uint64(20), result.acct.Lamports) + assert.True(t, result.stats.PendingFoldHit) + case <-time.After(time.Second): + t.Fatal("single account read blocked behind index commit") + } + + batch, stats, err := db.GetAccountsBatchSharedWithStats(context.Background(), 101, []solana.PublicKey{old.Key}) + require.NoError(t, err) + require.Len(t, batch, 1) + assert.Equal(t, uint64(20), batch[0].Lamports) + assert.Equal(t, uint64(1), stats.PendingFoldHits) + assert.Zero(t, stats.IndexHits) + + releaseOnce.Do(func() { close(releaseCommit) }) + require.NoError(t, <-commitDone) + db.foldHooks.afterPublicationStart = nil + got, err := db.GetAccount(101, old.Key) + require.NoError(t, err) + assert.Equal(t, uint64(20), got.Lamports) +} + func TestRefreshReadCachesIsScanResistantAndCoherent(t *testing.T) { db, _ := newFoldTestDb(t) defer db.CloseDb() diff --git a/pkg/accountsdb/fold.go b/pkg/accountsdb/fold.go index 07f75ed2..096a59a1 100644 --- a/pkg/accountsdb/fold.go +++ b/pkg/accountsdb/fold.go @@ -69,10 +69,11 @@ func (db *AccountsDb) readFoldMeta() (foldMeta, bool, error) { // foldTestHooks fire between CommitBatch stages so tests can inject crashes // (each hook may panic) at every point of the crash matrix. type foldTestHooks struct { - afterSegmentFsync func() - afterManifestRename func() - beforeIndexCommit func() - afterIndexCommit func() + afterSegmentFsync func() + afterManifestRename func() + beforeIndexCommit func() + afterPublicationStart func() + afterIndexCommit func() } func fire(h func()) { @@ -279,17 +280,32 @@ func (db *AccountsDb) CommitBatch( live = append(live, union[k].acct) } - // (7) The index epoch flip: entries + meta in one batch. Hold the read-cache - // publication lock through the refresh so an older batch read can neither - // observe a mixed index/cache view nor admit stale bytes after this commit. + // (7) Publish the immutable changed-key view and advance the cache epoch + // under a short lock. While pendingFold is installed, readers resolve every + // changed key from the new union; unchanged keys are identical on both sides + // of the atomic Pebble commit. This lets the expensive index fsync and cache + // refresh proceed without blocking account loaders. fire(db.foldHooks.beforeIndexCommit) db.readCacheEpochMu.Lock() + db.pendingFold = union + db.readCacheEpoch++ + db.readCacheEpochMu.Unlock() + fire(db.foldHooks.afterPublicationStart) + if err := db.applyManifestToIndex(manifest); err != nil { + db.readCacheEpochMu.Lock() + db.pendingFold = nil db.readCacheEpochMu.Unlock() return BatchCommitResult{}, err } - db.readCacheEpoch++ - db.refreshReadCachesLocked(live) + + // Old readers captured the preceding epoch and therefore cannot publish + // stale values. New readers use pendingFold for changed keys until these + // ordinary concurrent-safe cache operations finish. + db.refreshReadCacheEntries(live) + + db.readCacheEpochMu.Lock() + db.pendingFold = nil db.readCacheEpochMu.Unlock() fire(db.foldHooks.afterIndexCommit) diff --git a/pkg/accountsdb/fold_test.go b/pkg/accountsdb/fold_test.go index 6a3eb750..fbdff0ad 100644 --- a/pkg/accountsdb/fold_test.go +++ b/pkg/accountsdb/fold_test.go @@ -141,6 +141,15 @@ func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) { require.NoError(t, err) assert.Equal(t, m, got) + contextOnly, err := ReadSegmentManifestContext(segmentManifestPath(dir, 130, 42)) + require.NoError(t, err) + assert.Equal(t, m.Kind, contextOnly.Kind) + assert.Equal(t, m.BatchSeq, contextOnly.BatchSeq) + assert.Equal(t, m.ThroughSlot, contextOnly.ThroughSlot) + assert.Equal(t, m.ResumeCtx, contextOnly.ResumeCtx) + assert.Empty(t, contextOnly.Records, "retention scan must not allocate or decode account records") + assert.Empty(t, contextOnly.Bankhashes, "retention scan only needs the resume context") + // Flip one byte -> torn. path := segmentManifestPath(dir, 130, 42) data, err := os.ReadFile(path) @@ -151,6 +160,44 @@ func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) { assert.ErrorIs(t, err, ErrTornManifest) } +func BenchmarkReadSegmentManifestContext(b *testing.B) { + dir := b.TempDir() + const recordCount = 100_000 + records := make([]ManifestRecord, recordCount) + for idx := range records { + binary.LittleEndian.PutUint64(records[idx].Pubkey[:8], uint64(idx+1)) + records[idx].Offset = uint64(idx * 128) + } + manifest := &SegmentManifest{ + Version: segManifestVersion, + Kind: ManifestKindFold, + BatchSeq: 1, + ThroughSlot: 100, + FileId: 1, + Records: records, + ResumeCtx: []byte(`{"slot":100}`), + } + require.NoError(b, WriteSegmentManifest(dir, manifest)) + path := segmentManifestPath(dir, manifest.ThroughSlot, manifest.FileId) + + b.Run("full", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := ReadSegmentManifest(path); err != nil { + b.Fatal(err) + } + } + }) + b.Run("context-only", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := ReadSegmentManifestContext(path); err != nil { + b.Fatal(err) + } + } + }) +} + // Crash-point injection across the commit protocol. Each stage panics once, // the store reopens, RecoverFoldState runs, and the outcome must match the // crash matrix: before the manifest rename the batch never happened (orphan diff --git a/pkg/accountsdb/segment.go b/pkg/accountsdb/segment.go index 16429837..9359f784 100644 --- a/pkg/accountsdb/segment.go +++ b/pkg/accountsdb/segment.go @@ -338,6 +338,82 @@ func ReadSegmentManifest(path string) (*SegmentManifest, error) { return m, nil } +// ReadSegmentManifestContext reads only the fixed prefix and resume context, +// then structurally validates the declared record tail against the file size. +// It deliberately does not scan records or validate the trailing CRC. This is +// for advisory retention bookkeeping, where a malformed context fails closed; +// recovery, rewind, and index mutation must continue using ReadSegmentManifest. +func ReadSegmentManifestContext(path string) (*SegmentManifest, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, err + } + + const manifestPrefixSize = 4 + 4 + 1 + 5*8 + 4 + var prefix [manifestPrefixSize]byte + if _, err := io.ReadFull(f, prefix[:]); err != nil { + return nil, ErrTornManifest + } + d := &manifestDecoder{data: prefix[:]} + m := &SegmentManifest{} + if err := decodeManifestPrefix(d, m); err != nil { + return nil, err + } + + var u32 [4]byte + if _, err := io.ReadFull(f, u32[:]); err != nil { + return nil, ErrTornManifest + } + nBank := uint64(binary.LittleEndian.Uint32(u32[:])) + if nBank > uint64(info.Size())/40 { + return nil, ErrTornManifest + } + if _, err := f.Seek(int64(nBank*40), io.SeekCurrent); err != nil { + return nil, ErrTornManifest + } + if _, err := io.ReadFull(f, u32[:]); err != nil { + return nil, ErrTornManifest + } + ctxLen := uint64(binary.LittleEndian.Uint32(u32[:])) + current, err := f.Seek(0, io.SeekCurrent) + if err != nil || ctxLen > uint64(info.Size()) || uint64(current) > uint64(info.Size())-ctxLen { + return nil, ErrTornManifest + } + m.ResumeCtx = make([]byte, int(ctxLen)) + if _, err := io.ReadFull(f, m.ResumeCtx); err != nil { + return nil, ErrTornManifest + } + + var u64 [8]byte + if _, err := io.ReadFull(f, u64[:]); err != nil { + return nil, ErrTornManifest + } + nRecords := binary.LittleEndian.Uint64(u64[:]) + tailStart, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return nil, ErrTornManifest + } + if nRecords > ^uint64(0)/manifestRecordSize { + return nil, ErrTornManifest + } + recordsBytes := nRecords * manifestRecordSize + if recordsBytes > ^uint64(0)-4 || + tailStart < 0 || + uint64(tailStart) > ^uint64(0)-recordsBytes-4 { + return nil, ErrTornManifest + } + expectedSize := uint64(tailStart) + recordsBytes + 4 + if expectedSize != uint64(info.Size()) { + return nil, ErrTornManifest + } + return m, nil +} + // readManifestHeader parses only the fixed prefix — no CRC validation. func readManifestHeader(path string) (ManifestHeader, error) { f, err := os.Open(path) diff --git a/pkg/bankhash/bankhash.go b/pkg/bankhash/bankhash.go index 4d9e02ce..c2fc4d59 100644 --- a/pkg/bankhash/bankhash.go +++ b/pkg/bankhash/bankhash.go @@ -16,6 +16,16 @@ import ( ) func CalculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Account, modifiedAccts []*accounts.Account, parentBankHash [32]byte, numSigs uint64, blockHash [32]byte) []byte { + return calculateBankHash(slotCtx, writableAccts, modifiedAccts, false, parentBankHash, numSigs, blockHash) +} + +// CalculateBankHashUniqueModified skips LtHash's defensive key dedupe when the +// caller supplies an already-unique OverlayAccounts delta. +func CalculateBankHashUniqueModified(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Account, modifiedAccts []*accounts.Account, parentBankHash [32]byte, numSigs uint64, blockHash [32]byte) []byte { + return calculateBankHash(slotCtx, writableAccts, modifiedAccts, true, parentBankHash, numSigs, blockHash) +} + +func calculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Account, modifiedAccts []*accounts.Account, modifiedAcctsUnique bool, parentBankHash [32]byte, numSigs uint64, blockHash [32]byte) []byte { adhEnabled := !slotCtx.Features.IsActive(features.RemoveAccountsDeltaHash) ltHashEnabled := slotCtx.Features.IsActive(features.AccountsLtHash) @@ -32,7 +42,7 @@ func CalculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Acco } if ltHashEnabled { - updateAcctsLtHash(slotCtx, modifiedAccts) + updateAcctsLtHash(slotCtx, modifiedAccts, modifiedAcctsUnique) } var finalizeStart time.Time diff --git a/pkg/bankhash/lthash.go b/pkg/bankhash/lthash.go index 4cbf4656..6bcaa60e 100644 --- a/pkg/bankhash/lthash.go +++ b/pkg/bankhash/lthash.go @@ -15,8 +15,8 @@ import ( "github.com/Overclock-Validator/mithril/pkg/sealevel" ) -func updateAcctsLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account) { - deltaLtHash := calculateDeltaLtHash(slotCtx, modifiedAccts) +func updateAcctsLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account, inputUnique bool) { + deltaLtHash := calculateDeltaLtHashInternal(slotCtx, modifiedAccts, inputUnique) slotCtx.AcctsLtHash.Add(deltaLtHash) if ltDebug && (ltDebugSlot == 0 || ltDebugSlot == slotCtx.Slot) { dumpPerAcctDeltas(slotCtx, modifiedAccts) @@ -45,6 +45,10 @@ func dumpPerAcctDeltas(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Acco } func calculateDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account) *lthash.LtHash { + return calculateDeltaLtHashInternal(slotCtx, modifiedAccts, false) +} + +func calculateDeltaLtHashInternal(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account, inputUnique bool) *lthash.LtHash { // Dedupe by key (keep the last/newest value): an account that appears more // than once (e.g. both rent-collected and modified, or a sysvar collected via // multiple paths) would otherwise have its delta counted multiple times, @@ -61,7 +65,9 @@ func calculateDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.A metrics.GlobalBlockReplay.LtHashNewDataBytes = 0 dedupeStart = time.Now() } - modifiedAccts = dedupeModifiedAccts(modifiedAccts) + if !inputUnique { + modifiedAccts = dedupeModifiedAccts(modifiedAccts) + } if recordMetrics { metrics.GlobalBlockReplay.LtHashDedupe.AddTimingSince(dedupeStart) metrics.GlobalBlockReplay.LtHashUniqueAccounts = uint64(len(modifiedAccts)) diff --git a/pkg/bankhash/lthash_optimized_test.go b/pkg/bankhash/lthash_optimized_test.go index 0394f8af..f55a9eda 100644 --- a/pkg/bankhash/lthash_optimized_test.go +++ b/pkg/bankhash/lthash_optimized_test.go @@ -364,6 +364,13 @@ func benchmarkCalculateDeltaLtHash(b *testing.B, accountCount int) { benchmarkDeltaLtHashByte = result.Hash()[0] } }) + b.Run("worker_partials_unique_input", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + result := calculateDeltaLtHashInternal(ctx, modified, true) + benchmarkDeltaLtHashByte = result.Hash()[0] + } + }) b.Run("legacy_per_account", func(b *testing.B) { b.ReportAllocs() for range b.N { diff --git a/pkg/bankhash/lthash_test.go b/pkg/bankhash/lthash_test.go index b42b0dd6..1135fc49 100644 --- a/pkg/bankhash/lthash_test.go +++ b/pkg/bankhash/lthash_test.go @@ -110,6 +110,18 @@ func TestDedupePreventsLtHashDoubleCount(t *testing.T) { } } +func TestUniqueModifiedFastPathMatchesDefensiveDedupe(t *testing.T) { + key := solana.PublicKey{9} + old := &accounts.Account{Key: key, Lamports: 100, Data: []byte{1}, Owner: [32]byte{7}} + changed := &accounts.Account{Key: key, Lamports: 200, Data: []byte{2}, Owner: [32]byte{7}} + + defensive := calculateDeltaLtHashInternal(ctxWithParent(t, key, old), []*accounts.Account{changed}, false) + unique := calculateDeltaLtHashInternal(ctxWithParent(t, key, old), []*accounts.Account{changed}, true) + if !bytes.Equal(defensive.Hash(), unique.Hash()) { + t.Fatal("unique modified-account fast path changed the LtHash delta") + } +} + // PROOF of keep-LAST at the real LtHash level: when a key appears twice with different // values, the delta must reflect the LAST (newest) value, not the first (stale) one. func TestDedupeKeepsLastValueInDelta(t *testing.T) { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 1b6296fd..39f91a99 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -29,15 +29,34 @@ type AccountLoader struct { ParentMapBuild Timing SysvarUpdates Timing - WorkingSetLookup Timing - InProgressLookup Timing - AppendVecPinWait Timing - CacheLookup Timing - AdmissionFilter Timing - IndexLookup Timing - ReadPlanning Timing - AppendVecRead Timing - CachePublication Timing + SysvarClockRead Timing + SysvarSlotHashesRead Timing + SysvarRecentBlockhashesRead Timing + SysvarSlotHistoryRead Timing + SysvarStakeHistoryRead Timing + SysvarLastRestartSlotRead Timing + + WorkingSetLookup Timing + InProgressLookup Timing + AppendVecPinWait Timing + ReadCacheEpochWait Timing + CacheLookup Timing + AdmissionFilter Timing + IndexLookup Timing + ReadPlanning Timing + AppendVecRead Timing + CachePublicationWait Timing + CachePublication Timing + + SysvarWorkingSetLookup Timing + SysvarClone Timing + SysvarAppendVecPinWait Timing + SysvarInProgressLookup Timing + SysvarReadCacheEpochWait Timing + SysvarCacheLookup Timing + SysvarIndexAndAppendVecRead Timing + SysvarCachePublicationWait Timing + SysvarCachePublication Timing RequestedKeys uint64 DurableKeys uint64 @@ -45,6 +64,7 @@ type AccountLoader struct { WorkingSetHits uint64 InProgressHits uint64 + PendingFoldHits uint64 CacheHits uint64 IndexHits uint64 IndexMisses uint64 @@ -64,6 +84,14 @@ type AccountLoader struct { DecodedAccountObjects uint64 DecodedAccountBytes uint64 PlaceholderObjects uint64 + + SysvarReads uint64 + SysvarWorkingSetHits uint64 + SysvarInProgressHits uint64 + SysvarPendingFoldHits uint64 + SysvarCacheHits uint64 + SysvarDurableReads uint64 + SysvarCachePublicationEpochRejects uint64 } // TurbineIngress is the exact per-slot pre-replay pipeline decomposition. diff --git a/pkg/replay/block.go b/pkg/replay/block.go index e6c2fb25..2f2a4436 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -491,6 +491,45 @@ func validatePartitionedRewardsResume(slot uint64, epochRewards *sealevel.Sysvar ) } +func loadSysvarAccount(source blockAccountSource, slot uint64, pubkey solana.PublicKey, timing *metrics.Timing) (*accounts.Account, error) { + start := time.Now() + acct, stats, err := getAccountWithStats(source, slot, pubkey) + timing.AddTimingSince(start) + recordSysvarAccountReadStats(&metrics.GlobalBlockReplay.AccountLoader, stats) + return acct, err +} + +func recordSysvarAccountReadStats(dst *metrics.AccountLoader, src accountsdb.AccountReadStats) { + dst.SysvarReads++ + dst.SysvarWorkingSetLookup.AddTiming(time.Duration(src.WorkingSetLookupNanoseconds)) + dst.SysvarClone.AddTiming(time.Duration(src.CloneNanoseconds)) + dst.SysvarAppendVecPinWait.AddTiming(time.Duration(src.AppendVecPinWaitNanoseconds)) + dst.SysvarInProgressLookup.AddTiming(time.Duration(src.InProgressNanoseconds)) + dst.SysvarReadCacheEpochWait.AddTiming(time.Duration(src.ReadCacheEpochWaitNanoseconds)) + dst.SysvarCacheLookup.AddTiming(time.Duration(src.CacheLookupNanoseconds)) + dst.SysvarIndexAndAppendVecRead.AddTiming(time.Duration(src.IndexAndAppendVecReadNanoseconds)) + dst.SysvarCachePublicationWait.AddTiming(time.Duration(src.CachePublicationWaitNanoseconds)) + dst.SysvarCachePublication.AddTiming(time.Duration(src.CachePublicationNanoseconds)) + if src.WorkingSetHit { + dst.SysvarWorkingSetHits++ + } + if src.InProgressHit { + dst.SysvarInProgressHits++ + } + if src.PendingFoldHit { + dst.SysvarPendingFoldHits++ + } + if src.CacheHit { + dst.SysvarCacheHits++ + } + if src.DurableRead { + dst.SysvarDurableReads++ + } + if src.CachePublicationEpochRejected { + dst.SysvarCachePublicationEpochRejects++ + } +} + func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.Block, epochSchedule *sealevel.SysvarEpochSchedule, alpenglowClock bool) (accounts.Accounts, accounts.Accounts, int, error) { phaseStart := time.Now() err := resolveAddrTableLookups(accountsDb, block) @@ -544,7 +583,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B // resume it is the restored Clock as of the last rooted slot, which durable may not match. clockAcct = sealevel.SysvarCache.Clock.Acct.Clone() } else { - clockAcct, err = accountsDb.GetAccount(block.Slot, sealevel.SysvarClockAddr) + clockAcct, err = loadSysvarAccount(accountsDb, block.Slot, sealevel.SysvarClockAddr, &metrics.GlobalBlockReplay.AccountLoader.SysvarClockRead) if err != nil { panic("unable to retrieve clock sysvar when updating clock") } @@ -581,7 +620,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B // update and cache SlotHashes sysvar { - slotHashesAcct, err := accountsDb.GetAccount(block.Slot, sealevel.SysvarSlotHashesAddr) + slotHashesAcct, err := loadSysvarAccount(accountsDb, block.Slot, sealevel.SysvarSlotHashesAddr, &metrics.GlobalBlockReplay.AccountLoader.SysvarSlotHashesRead) if err != nil { panic("unable to retrieve slothashes sysvar from acctsdb") } @@ -631,7 +670,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B // cache RecentBlockhashes sysvar { - recentBlockhashesAcct, err := accountsDb.GetAccount(block.Slot, sealevel.SysvarRecentBlockHashesAddr) + recentBlockhashesAcct, err := loadSysvarAccount(accountsDb, block.Slot, sealevel.SysvarRecentBlockHashesAddr, &metrics.GlobalBlockReplay.AccountLoader.SysvarRecentBlockhashesRead) if err != nil { panic("unable to get recentblockhashes") } @@ -678,7 +717,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B // cache SlotHistory sysvar { - slotHistoryAcct, err := accountsDb.GetAccount(block.Slot, sealevel.SysvarSlotHistoryAddr) + slotHistoryAcct, err := loadSysvarAccount(accountsDb, block.Slot, sealevel.SysvarSlotHistoryAddr, &metrics.GlobalBlockReplay.AccountLoader.SysvarSlotHistoryRead) if err != nil { panic("unable to get slothistory") } @@ -702,7 +741,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B // cache StakeHistory sysvar { - stakeHistoryAcct, err := accountsDb.GetAccount(block.Slot, sealevel.SysvarStakeHistoryAddr) + stakeHistoryAcct, err := loadSysvarAccount(accountsDb, block.Slot, sealevel.SysvarStakeHistoryAddr, &metrics.GlobalBlockReplay.AccountLoader.SysvarStakeHistoryRead) if err != nil { panic("unable to get stakehistory") } @@ -743,7 +782,7 @@ func loadBlockAccountsAndUpdateSysvars(accountsDb blockAccountSource, block *b.B // cache LastRestartSlot sysvar { - lastRestartSlotAcct, err := accountsDb.GetAccount(block.Slot, sealevel.SysvarLastRestartSlotAddr) + lastRestartSlotAcct, err := loadSysvarAccount(accountsDb, block.Slot, sealevel.SysvarLastRestartSlotAddr, &metrics.GlobalBlockReplay.AccountLoader.SysvarLastRestartSlotRead) if err != nil { panic("unable to get last restart slot sysvar acct") } @@ -792,6 +831,7 @@ func recordAccountLoaderBatchStats(dst *metrics.AccountLoader, src accountsdb.Ba dst.DurableKeys = src.DurableKeys dst.WorkingSetHits = src.WorkingSetHits dst.InProgressHits = src.InProgressHits + dst.PendingFoldHits = src.PendingFoldHits dst.CacheHits = src.CacheHits dst.IndexHits = src.IndexHits dst.IndexMisses = src.IndexMisses @@ -812,11 +852,13 @@ func recordAccountLoaderBatchStats(dst *metrics.AccountLoader, src accountsdb.Ba dst.WorkingSetLookup.AddTiming(time.Duration(src.WorkingSetLookupNanoseconds)) dst.InProgressLookup.AddTiming(time.Duration(src.InProgressNanoseconds)) dst.AppendVecPinWait.AddTiming(time.Duration(src.AppendVecPinWaitNanoseconds)) + dst.ReadCacheEpochWait.AddTiming(time.Duration(src.ReadCacheEpochWaitNanoseconds)) dst.CacheLookup.AddTiming(time.Duration(src.CacheLookupNanoseconds)) dst.AdmissionFilter.AddTiming(time.Duration(src.AdmissionFilterNanoseconds)) dst.IndexLookup.AddTiming(time.Duration(src.IndexLookupNanoseconds)) dst.ReadPlanning.AddTiming(time.Duration(src.ReadPlanningNanoseconds)) dst.AppendVecRead.AddTiming(time.Duration(src.AppendVecReadNanoseconds)) + dst.CachePublicationWait.AddTiming(time.Duration(src.CachePublicationWaitNanoseconds)) dst.CachePublication.AddTiming(time.Duration(src.CachePublicationNanoseconds)) } @@ -3161,6 +3203,10 @@ func runIncinerator(slotCtx *sealevel.SlotCtx) { func compileWritableAndModifiedAccts(slotCtx *sealevel.SlotCtx, block *b.Block, rentAccts []*accounts.Account) ([]*accounts.Account, []*accounts.Account) { adhRemoved := accountsDeltaHashRemoved(slotCtx) + sysvarAccts := collectAndUpdateSysvarAcctsForAdh(slotCtx) + if modifiedAccts, ok := uniqueOverlayModifiedAccounts(slotCtx, block.EpochUpdatedAccts, rentAccts, sysvarAccts); ok { + return nil, modifiedAccts + } var writableAccts []*accounts.Account var alreadyAdded map[solana.PublicKey]bool if !adhRemoved { @@ -3211,7 +3257,6 @@ func compileWritableAndModifiedAccts(slotCtx *sealevel.SlotCtx, block *b.Block, } modifiedAccts = append(modifiedAccts, rentAccts...) - sysvarAccts := collectAndUpdateSysvarAcctsForAdh(slotCtx) if !adhRemoved { writableAccts = append(writableAccts, sysvarAccts...) } @@ -3257,6 +3302,14 @@ func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Ac if block.Features != nil && block.Features.IsActive(features.RemoveAccountsDeltaHash) { writableMapCapacity = 0 } + _, overlayBacked := accts.(*accounts.OverlayAccounts) + modifiedAccountsFromDelta := overlayBacked && + block.Features != nil && + block.Features.IsActive(features.RemoveAccountsDeltaHash) + modifiedMapCapacity := accountMapCapacity + if modifiedAccountsFromDelta { + modifiedMapCapacity = 0 + } slotCtx := &sealevel.SlotCtx{ Accounts: accts, ParentAccts: parentAccts, @@ -3267,9 +3320,10 @@ func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Ac FeeRateGovernor: block.FeeRateGovernor, NumSignatures: block.NumSignatures, - AcctMapsMu: &sync.Mutex{}, - ModifiedAccts: make(map[solana.PublicKey]bool, accountMapCapacity), - WritableAccts: make(map[solana.PublicKey]bool, writableMapCapacity), + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool, modifiedMapCapacity), + WritableAccts: make(map[solana.PublicKey]bool, writableMapCapacity), + ModifiedAccountsFromDelta: modifiedAccountsFromDelta, Blockhash: block.Blockhash, LastBlockhash: block.LastBlockhash, @@ -3876,7 +3930,7 @@ func ProcessBlock( writableAccts, modifiedAccts := compileWritableAndModifiedAccts(slotCtx, block, rentAccts) metrics.GlobalBlockReplay.CompileWritableAndModifiedAccts.AddTimingSince(start) start = time.Now() - ensureParentsErr := ensureParentAccountsForModified(slotCtx) + ensureParentsErr := ensureParentAccountsForModified(slotCtx, modifiedAccts) metrics.GlobalBlockReplay.EnsureParentAccountsForModified.AddTimingSince(start) if ensureParentsErr != nil { return nil, ensureParentsErr @@ -3884,7 +3938,11 @@ func ProcessBlock( start = time.Now() setReplayStage("bankhash") - slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) + if slotCtx.ModifiedAccountsFromDelta { + slotCtx.FinalBankhash = bankhash.CalculateBankHashUniqueModified(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) + } else { + slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) + } metrics.GlobalBlockReplay.BankHash.AddTimingSince(start) if alpenglowClock { footerVerificationStart := time.Now() diff --git a/pkg/replay/leader_finalize.go b/pkg/replay/leader_finalize.go index 0ee99534..13b74cf0 100644 --- a/pkg/replay/leader_finalize.go +++ b/pkg/replay/leader_finalize.go @@ -144,11 +144,15 @@ func CommitLeaderSlot(in CommitLeaderInput) (*sealevel.SlotCtx, error) { if err := finishLeaderSysvars(slotCtx, block); err != nil { return nil, err } - if err := ensureParentAccountsForModified(slotCtx); err != nil { + writable, modified := compileLeaderAccounts(slotCtx, block, rentAccts) + if err := ensureParentAccountsForModified(slotCtx, modified); err != nil { return nil, err } - writable, modified := compileLeaderAccounts(slotCtx, block, rentAccts) - slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writable, modified, block.ParentBankhash, block.NumSignatures, block.Blockhash) + if slotCtx.ModifiedAccountsFromDelta { + slotCtx.FinalBankhash = bankhash.CalculateBankHashUniqueModified(slotCtx, writable, modified, block.ParentBankhash, block.NumSignatures, block.Blockhash) + } else { + slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writable, modified, block.ParentBankhash, block.NumSignatures, block.Blockhash) + } copy(block.ExpectedBankhash[:], slotCtx.FinalBankhash) block.HasExpectedBankhash = true return slotCtx, nil @@ -217,11 +221,15 @@ func withData(acct *accounts.Account, data []byte) *accounts.Account { return out } -func ensureParentAccountsForModified(slotCtx *sealevel.SlotCtx) error { +func ensureParentAccountsForModified(slotCtx *sealevel.SlotCtx, modified []*accounts.Account) error { if slotCtx.Features == nil || !slotCtx.Features.IsActive(features.AccountsLtHash) { return nil } - for key := range slotCtx.ModifiedAccts { + for _, modifiedAcct := range modified { + if modifiedAcct == nil { + continue + } + key := modifiedAcct.Key if _, err := slotCtx.GetParentAccount(key); err == nil { continue } @@ -236,7 +244,55 @@ func ensureParentAccountsForModified(slotCtx *sealevel.SlotCtx) error { return nil } +func uniqueOverlayModifiedAccounts(slotCtx *sealevel.SlotCtx, extras ...[]*accounts.Account) ([]*accounts.Account, bool) { + if !slotCtx.ModifiedAccountsFromDelta { + return nil, false + } + overlay, ok := slotCtx.Accounts.(*accounts.OverlayAccounts) + if !ok { + return nil, false + } + delta := overlay.DeltaAccounts() + modified := delta[:0] + for _, acct := range delta { + if acct != nil { + modified = append(modified, acct) + } + } + extraCount := 0 + for _, group := range extras { + extraCount += len(group) + } + if extraCount == 0 { + return modified, true + } + + byKey := make(map[solana.PublicKey]int, len(modified)+extraCount) + for idx, acct := range modified { + if acct != nil { + byKey[acct.Key] = idx + } + } + for _, group := range extras { + for _, acct := range group { + if acct == nil { + continue + } + if idx, exists := byKey[acct.Key]; exists { + modified[idx] = acct + continue + } + byKey[acct.Key] = len(modified) + modified = append(modified, acct) + } + } + return modified, true +} + func compileLeaderAccounts(slotCtx *sealevel.SlotCtx, block *b.Block, rentAccts []*accounts.Account) ([]*accounts.Account, []*accounts.Account) { + if modified, ok := uniqueOverlayModifiedAccounts(slotCtx, block.EpochUpdatedAccts, rentAccts); ok { + return nil, modified + } adhRemoved := accountsDeltaHashRemoved(slotCtx) var writable []*accounts.Account var seenWritable map[solana.PublicKey]struct{} diff --git a/pkg/replay/lean_writable_fastpath_test.go b/pkg/replay/lean_writable_fastpath_test.go index 0975280f..08d72d70 100644 --- a/pkg/replay/lean_writable_fastpath_test.go +++ b/pkg/replay/lean_writable_fastpath_test.go @@ -364,6 +364,34 @@ func TestCompileLeaderAccountsGatesWritableListOnADH(t *testing.T) { } } +func TestOverlayModifiedFastPathAvoidsGlobalMapAndKeepsUniqueNewestValues(t *testing.T) { + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + slotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) + overlay := accounts.NewOverlayAccounts(slotCtx.Accounts) + slotCtx.Accounts = overlay + slotCtx.ModifiedAccountsFromDelta = true + + firstKey := solana.PublicKey{0xa1} + first := &accounts.Account{Key: firstKey, Lamports: 10} + require.NoError(t, slotCtx.SetAccount(firstKey, first)) + slotCtx.RecordModifiedAcct(firstKey) + slotCtx.RecordModifiedAccountStates([]*accounts.Account{first}, []bool{true}) + assert.Empty(t, slotCtx.ModifiedAccts, "overlay-backed replay must not contend on the legacy modified map") + + replacement := &accounts.Account{Key: firstKey, Lamports: 20} + second := &accounts.Account{Key: solana.PublicKey{0xa2}, Lamports: 30} + modified, ok := uniqueOverlayModifiedAccounts(slotCtx, []*accounts.Account{replacement, second, nil}) + require.True(t, ok) + require.Len(t, modified, 2) + byKey := make(map[solana.PublicKey]*accounts.Account, len(modified)) + for _, acct := range modified { + byKey[acct.Key] = acct + } + assert.Same(t, replacement, byKey[firstKey]) + assert.Same(t, second, byKey[second.Key]) +} + func TestCompileWritableAndModifiedAcctsGatesWritableListOnADH(t *testing.T) { for _, tc := range []struct { name string diff --git a/pkg/replay/promotion.go b/pkg/replay/promotion.go index 599d9b5c..d41802d9 100644 --- a/pkg/replay/promotion.go +++ b/pkg/replay/promotion.go @@ -68,6 +68,22 @@ type measuredSharedBlockAccountSource interface { GetAccountsBatchSharedWithStats(ctx context.Context, slot uint64, pks []solana.PublicKey) ([]*accounts.Account, accountsdb.BatchReadStats, error) } +type measuredBlockAccountSource interface { + GetAccountWithStats(slot uint64, pubkey solana.PublicKey) (*accounts.Account, accountsdb.AccountReadStats, error) +} + +func getAccountWithStats(source blockAccountSource, slot uint64, pubkey solana.PublicKey) (*accounts.Account, accountsdb.AccountReadStats, error) { + if measured, ok := source.(measuredBlockAccountSource); ok { + return measured.GetAccountWithStats(slot, pubkey) + } + start := time.Now() + acct, err := source.GetAccount(slot, pubkey) + return acct, accountsdb.AccountReadStats{ + IndexAndAppendVecReadNanoseconds: uint64(time.Since(start).Nanoseconds()), + DurableRead: true, + }, err +} + func getAccountsBatchShared(ctx context.Context, source blockAccountSource, slot uint64, pks []solana.PublicKey) ([]*accounts.Account, error) { out, _, err := getAccountsBatchSharedWithStats(ctx, source, slot, pks) return out, err @@ -149,20 +165,43 @@ func (t *unrootedTail) SetTransactionStatusCheckpointHooks(hooks TransactionStat // (rooted) value read at slot. func (t *unrootedTail) GetAccount(slot uint64, pubkey solana.PublicKey) (*accounts.Account, error) { if a, ok := t.overlay.Lookup([32]byte(pubkey)); ok { - // Callers receive a mutable account, never the WorkingSet's retained - // historical value. Fee and reward paths legitimately mutate values - // returned by this method. return a.Clone(), nil } a, err := t.durable.GetAccount(slot, pubkey) if err != nil || a == nil { return a, err } + return a.Clone(), nil +} + +func (t *unrootedTail) GetAccountWithStats(slot uint64, pubkey solana.PublicKey) (*accounts.Account, accountsdb.AccountReadStats, error) { + var stats accountsdb.AccountReadStats + lookupStart := time.Now() + if a, ok := t.overlay.Lookup([32]byte(pubkey)); ok { + stats.WorkingSetLookupNanoseconds = uint64(time.Since(lookupStart).Nanoseconds()) + stats.WorkingSetHit = true + // Callers receive a mutable account, never the WorkingSet's retained + // historical value. Fee and reward paths legitimately mutate values + // returned by this method. + cloneStart := time.Now() + acct := a.Clone() + stats.CloneNanoseconds = uint64(time.Since(cloneStart).Nanoseconds()) + return acct, stats, nil + } + stats.WorkingSetLookupNanoseconds = uint64(time.Since(lookupStart).Nanoseconds()) + a, durableStats, err := getAccountWithStats(t.durable, slot, pubkey) + durableStats.WorkingSetLookupNanoseconds += stats.WorkingSetLookupNanoseconds + if err != nil || a == nil { + return a, durableStats, err + } // AccountsDb may satisfy this read from one of its shared read caches. // Do not let a speculative caller (notably leader fee distribution) mutate // that cached parent in place: ordered replay must observe the same parent // value when it reconstructs the locally produced block. - return a.Clone(), nil + cloneStart := time.Now() + acct := a.Clone() + durableStats.CloneNanoseconds += uint64(time.Since(cloneStart).Nanoseconds()) + return acct, durableStats, nil } // GetAccountsBatch returns one entry per requested key, in order, preferring the diff --git a/pkg/replay/publication_concurrency_test.go b/pkg/replay/publication_concurrency_test.go index 7e3fd7e6..b1070ba6 100644 --- a/pkg/replay/publication_concurrency_test.go +++ b/pkg/replay/publication_concurrency_test.go @@ -33,11 +33,12 @@ func TestConcurrentDisjointTransactionPublication(t *testing.T) { parent := accounts.NewMemAccounts() overlay := accounts.NewOverlayAccountsWithLen(parent, totalAccounts) slotCtx := &sealevel.SlotCtx{ - Accounts: overlay, - Features: feats, - AcctMapsMu: &sync.Mutex{}, - ModifiedAccts: make(map[solana.PublicKey]bool, totalAccounts), - WritableAccts: make(map[solana.PublicKey]bool), + Accounts: overlay, + Features: feats, + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool), + WritableAccts: make(map[solana.PublicKey]bool), + ModifiedAccountsFromDelta: true, } start := make(chan struct{}) @@ -78,9 +79,10 @@ func TestConcurrentDisjointTransactionPublication(t *testing.T) { } assert.Empty(t, slotCtx.WritableAccts, "ADH-removed replay must not build the unused writable set") - require.Len(t, slotCtx.ModifiedAccts, totalAccounts) + assert.Empty(t, slotCtx.ModifiedAccts, "overlay delta replaces the contended modified-account map") require.Len(t, overlay.DeltaAccounts(), totalAccounts) - for key := range slotCtx.ModifiedAccts { + for _, modified := range overlay.DeltaAccounts() { + key := modified.Key acct, err := overlay.GetAccount((*[32]byte)(&key)) require.NoError(t, err) assert.Equal(t, binary.LittleEndian.Uint64(key[:8]), acct.Lamports) diff --git a/pkg/sealevel/execution_ctx.go b/pkg/sealevel/execution_ctx.go index d9d6feec..74a48f35 100644 --- a/pkg/sealevel/execution_ctx.go +++ b/pkg/sealevel/execution_ctx.go @@ -79,12 +79,16 @@ type SlotCtx struct { AcctMapsMu *sync.Mutex // AcctMapsMu protects the next 2 maps ModifiedAccts map[solana.PublicKey]bool WritableAccts map[solana.PublicKey]bool - NumSignatures uint64 // signatures processed in this bank (resets for every child bank) - Blockhash [32]byte - LastBlockhash [32]byte - SlotBank SlotBank - Features *features.Features - VoteTimestampMu *sync.Mutex + // ModifiedAccountsFromDelta lets replay use OverlayAccounts' already-unique + // branch delta as the LtHash/store input after AccountsDeltaHash removal. + // Transaction publication then avoids the residual global ModifiedAccts lock. + ModifiedAccountsFromDelta bool + NumSignatures uint64 // signatures processed in this bank (resets for every child bank) + Blockhash [32]byte + LastBlockhash [32]byte + SlotBank SlotBank + Features *features.Features + VoteTimestampMu *sync.Mutex // VoteTimestampsMu protects VoteTimestamps VoteTimestamps map[solana.PublicKey]BlockTimestamp VoteAccts map[solana.PublicKey]uint64 @@ -467,6 +471,9 @@ func (slotCtx *SlotCtx) SetAccount(pubkey solana.PublicKey, acct *accounts.Accou } func (slotCtx *SlotCtx) RecordModifiedAcct(pubkey solana.PublicKey) { + if slotCtx.ModifiedAccountsFromDelta { + return + } slotCtx.AcctMapsMu.Lock() defer slotCtx.AcctMapsMu.Unlock() if slotCtx.Features == nil || !slotCtx.Features.IsActive(features.RemoveAccountsDeltaHash) { @@ -479,6 +486,9 @@ func (slotCtx *SlotCtx) RecordModifiedAccountStates(accountStates []*accounts.Ac if len(accountStates) != len(touched) { panic("account states/touched length mismatch") } + if slotCtx.ModifiedAccountsFromDelta { + return + } slotCtx.AcctMapsMu.Lock() defer slotCtx.AcctMapsMu.Unlock() for idx, acct := range accountStates { From 72b9bc6c03dfd5c0e16c50f8ee7a85a2ad3775ef Mon Sep 17 00:00:00 2001 From: smcio Date: Sat, 25 Jul 2026 19:59:55 +0200 Subject: [PATCH 5/5] fix replay bank hashes across epoch transitions --- pkg/bankhash/bankhash.go | 12 +-- pkg/bankhash/lthash.go | 12 +-- pkg/bankhash/lthash_optimized_test.go | 7 -- pkg/bankhash/lthash_test.go | 12 --- pkg/replay/block.go | 24 +----- pkg/replay/epoch_inflation.go | 12 +++ pkg/replay/epoch_inflation_test.go | 59 +++++++++++++++ pkg/replay/leader_finalize.go | 54 +------------- pkg/replay/lean_writable_fastpath_test.go | 85 +++++++++++++++++----- pkg/replay/publication_concurrency_test.go | 16 ++-- pkg/sealevel/execution_ctx.go | 22 ++---- 11 files changed, 158 insertions(+), 157 deletions(-) diff --git a/pkg/bankhash/bankhash.go b/pkg/bankhash/bankhash.go index c2fc4d59..4d9e02ce 100644 --- a/pkg/bankhash/bankhash.go +++ b/pkg/bankhash/bankhash.go @@ -16,16 +16,6 @@ import ( ) func CalculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Account, modifiedAccts []*accounts.Account, parentBankHash [32]byte, numSigs uint64, blockHash [32]byte) []byte { - return calculateBankHash(slotCtx, writableAccts, modifiedAccts, false, parentBankHash, numSigs, blockHash) -} - -// CalculateBankHashUniqueModified skips LtHash's defensive key dedupe when the -// caller supplies an already-unique OverlayAccounts delta. -func CalculateBankHashUniqueModified(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Account, modifiedAccts []*accounts.Account, parentBankHash [32]byte, numSigs uint64, blockHash [32]byte) []byte { - return calculateBankHash(slotCtx, writableAccts, modifiedAccts, true, parentBankHash, numSigs, blockHash) -} - -func calculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Account, modifiedAccts []*accounts.Account, modifiedAcctsUnique bool, parentBankHash [32]byte, numSigs uint64, blockHash [32]byte) []byte { adhEnabled := !slotCtx.Features.IsActive(features.RemoveAccountsDeltaHash) ltHashEnabled := slotCtx.Features.IsActive(features.AccountsLtHash) @@ -42,7 +32,7 @@ func calculateBankHash(slotCtx *sealevel.SlotCtx, writableAccts []*accounts.Acco } if ltHashEnabled { - updateAcctsLtHash(slotCtx, modifiedAccts, modifiedAcctsUnique) + updateAcctsLtHash(slotCtx, modifiedAccts) } var finalizeStart time.Time diff --git a/pkg/bankhash/lthash.go b/pkg/bankhash/lthash.go index 6bcaa60e..4cbf4656 100644 --- a/pkg/bankhash/lthash.go +++ b/pkg/bankhash/lthash.go @@ -15,8 +15,8 @@ import ( "github.com/Overclock-Validator/mithril/pkg/sealevel" ) -func updateAcctsLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account, inputUnique bool) { - deltaLtHash := calculateDeltaLtHashInternal(slotCtx, modifiedAccts, inputUnique) +func updateAcctsLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account) { + deltaLtHash := calculateDeltaLtHash(slotCtx, modifiedAccts) slotCtx.AcctsLtHash.Add(deltaLtHash) if ltDebug && (ltDebugSlot == 0 || ltDebugSlot == slotCtx.Slot) { dumpPerAcctDeltas(slotCtx, modifiedAccts) @@ -45,10 +45,6 @@ func dumpPerAcctDeltas(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Acco } func calculateDeltaLtHash(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account) *lthash.LtHash { - return calculateDeltaLtHashInternal(slotCtx, modifiedAccts, false) -} - -func calculateDeltaLtHashInternal(slotCtx *sealevel.SlotCtx, modifiedAccts []*accounts.Account, inputUnique bool) *lthash.LtHash { // Dedupe by key (keep the last/newest value): an account that appears more // than once (e.g. both rent-collected and modified, or a sysvar collected via // multiple paths) would otherwise have its delta counted multiple times, @@ -65,9 +61,7 @@ func calculateDeltaLtHashInternal(slotCtx *sealevel.SlotCtx, modifiedAccts []*ac metrics.GlobalBlockReplay.LtHashNewDataBytes = 0 dedupeStart = time.Now() } - if !inputUnique { - modifiedAccts = dedupeModifiedAccts(modifiedAccts) - } + modifiedAccts = dedupeModifiedAccts(modifiedAccts) if recordMetrics { metrics.GlobalBlockReplay.LtHashDedupe.AddTimingSince(dedupeStart) metrics.GlobalBlockReplay.LtHashUniqueAccounts = uint64(len(modifiedAccts)) diff --git a/pkg/bankhash/lthash_optimized_test.go b/pkg/bankhash/lthash_optimized_test.go index f55a9eda..0394f8af 100644 --- a/pkg/bankhash/lthash_optimized_test.go +++ b/pkg/bankhash/lthash_optimized_test.go @@ -364,13 +364,6 @@ func benchmarkCalculateDeltaLtHash(b *testing.B, accountCount int) { benchmarkDeltaLtHashByte = result.Hash()[0] } }) - b.Run("worker_partials_unique_input", func(b *testing.B) { - b.ReportAllocs() - for range b.N { - result := calculateDeltaLtHashInternal(ctx, modified, true) - benchmarkDeltaLtHashByte = result.Hash()[0] - } - }) b.Run("legacy_per_account", func(b *testing.B) { b.ReportAllocs() for range b.N { diff --git a/pkg/bankhash/lthash_test.go b/pkg/bankhash/lthash_test.go index 1135fc49..b42b0dd6 100644 --- a/pkg/bankhash/lthash_test.go +++ b/pkg/bankhash/lthash_test.go @@ -110,18 +110,6 @@ func TestDedupePreventsLtHashDoubleCount(t *testing.T) { } } -func TestUniqueModifiedFastPathMatchesDefensiveDedupe(t *testing.T) { - key := solana.PublicKey{9} - old := &accounts.Account{Key: key, Lamports: 100, Data: []byte{1}, Owner: [32]byte{7}} - changed := &accounts.Account{Key: key, Lamports: 200, Data: []byte{2}, Owner: [32]byte{7}} - - defensive := calculateDeltaLtHashInternal(ctxWithParent(t, key, old), []*accounts.Account{changed}, false) - unique := calculateDeltaLtHashInternal(ctxWithParent(t, key, old), []*accounts.Account{changed}, true) - if !bytes.Equal(defensive.Hash(), unique.Hash()) { - t.Fatal("unique modified-account fast path changed the LtHash delta") - } -} - // PROOF of keep-LAST at the real LtHash level: when a key appears twice with different // values, the delta must reflect the LAST (newest) value, not the first (stale) one. func TestDedupeKeepsLastValueInDelta(t *testing.T) { diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 2f2a4436..cc369abc 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -3204,9 +3204,6 @@ func runIncinerator(slotCtx *sealevel.SlotCtx) { func compileWritableAndModifiedAccts(slotCtx *sealevel.SlotCtx, block *b.Block, rentAccts []*accounts.Account) ([]*accounts.Account, []*accounts.Account) { adhRemoved := accountsDeltaHashRemoved(slotCtx) sysvarAccts := collectAndUpdateSysvarAcctsForAdh(slotCtx) - if modifiedAccts, ok := uniqueOverlayModifiedAccounts(slotCtx, block.EpochUpdatedAccts, rentAccts, sysvarAccts); ok { - return nil, modifiedAccts - } var writableAccts []*accounts.Account var alreadyAdded map[solana.PublicKey]bool if !adhRemoved { @@ -3302,14 +3299,6 @@ func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Ac if block.Features != nil && block.Features.IsActive(features.RemoveAccountsDeltaHash) { writableMapCapacity = 0 } - _, overlayBacked := accts.(*accounts.OverlayAccounts) - modifiedAccountsFromDelta := overlayBacked && - block.Features != nil && - block.Features.IsActive(features.RemoveAccountsDeltaHash) - modifiedMapCapacity := accountMapCapacity - if modifiedAccountsFromDelta { - modifiedMapCapacity = 0 - } slotCtx := &sealevel.SlotCtx{ Accounts: accts, ParentAccts: parentAccts, @@ -3320,10 +3309,9 @@ func newSlotCtx(block *b.Block, accts accounts.Accounts, parentAccts accounts.Ac FeeRateGovernor: block.FeeRateGovernor, NumSignatures: block.NumSignatures, - AcctMapsMu: &sync.Mutex{}, - ModifiedAccts: make(map[solana.PublicKey]bool, modifiedMapCapacity), - WritableAccts: make(map[solana.PublicKey]bool, writableMapCapacity), - ModifiedAccountsFromDelta: modifiedAccountsFromDelta, + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool, accountMapCapacity), + WritableAccts: make(map[solana.PublicKey]bool, writableMapCapacity), Blockhash: block.Blockhash, LastBlockhash: block.LastBlockhash, @@ -3938,11 +3926,7 @@ func ProcessBlock( start = time.Now() setReplayStage("bankhash") - if slotCtx.ModifiedAccountsFromDelta { - slotCtx.FinalBankhash = bankhash.CalculateBankHashUniqueModified(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) - } else { - slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) - } + slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writableAccts, modifiedAccts, block.ParentBankhash, slotCtx.NumSignatures, block.Blockhash) metrics.GlobalBlockReplay.BankHash.AddTimingSince(start) if alpenglowClock { footerVerificationStart := time.Now() diff --git a/pkg/replay/epoch_inflation.go b/pkg/replay/epoch_inflation.go index 8024174e..5201fe98 100644 --- a/pkg/replay/epoch_inflation.go +++ b/pkg/replay/epoch_inflation.go @@ -209,6 +209,18 @@ func loadEpochInflationAccountStateForReplay(slotCtx *sealevel.SlotCtx) (EpochIn if slotCtx == nil { return EpochInflationAccountState{}, fmt.Errorf("missing slot context") } + + // The first executed bank after an epoch boundary stages the new inflation + // account in its current-bank accounts. This is especially important when + // the first eight slots are skipped: that same bank's reward certificate + // already targets the new epoch, while the parent still contains only the + // previous epoch's inflation state. + if slotCtx.Accounts != nil { + if acct, err := slotCtx.GetAccount(VoteRewardAccountAddr()); err == nil && acct != nil { + return decodeEpochInflationAccountFromAcct(acct, slotCtx.Slot) + } + } + acct, err := slotCtx.GetAccountFromAccountsDb(VoteRewardAccountAddr()) if err != nil { return EpochInflationAccountState{}, fmt.Errorf("load vote reward account at parent slot %d: %w", slotCtx.ParentSlot, err) diff --git a/pkg/replay/epoch_inflation_test.go b/pkg/replay/epoch_inflation_test.go index 5ae0ed4b..8a51e532 100644 --- a/pkg/replay/epoch_inflation_test.go +++ b/pkg/replay/epoch_inflation_test.go @@ -122,3 +122,62 @@ func TestStageEpochInflationAccountRollsStateAndCapitalization(t *testing.T) { require.Equal(t, updated.Data, stored.Data) require.Equal(t, updated.Lamports, stored.Lamports) } + +func TestLoadEpochInflationAccountStateForReplayPrefersStagedBoundaryAccount(t *testing.T) { + const ( + parentSlot = uint64(7_019_999) + boundarySlot = uint64(7_020_008) + ) + + parentState := EpochInflationAccountState{ + Current: EpochInflationState{Epoch: 129, SlotsPerEpoch: 54_000}, + } + stagedState := EpochInflationAccountState{ + Current: EpochInflationState{Epoch: 130, SlotsPerEpoch: 54_000}, + Prev: &parentState.Current, + } + key := VoteRewardAccountAddr() + parent := &accounts.Account{ + Key: key, + Data: encodeEpochInflationAccountState(parentState), + } + staged := &accounts.Account{ + Key: key, + Data: encodeEpochInflationAccountState(stagedState), + } + + bankAccounts := accounts.NewMemAccounts() + require.NoError(t, bankAccounts.SetAccountWithoutLock(key, staged)) + slotCtx := &sealevel.SlotCtx{ + Accounts: bankAccounts, + Slot: boundarySlot, + ParentSlot: parentSlot, + UnrootedRead: rewardAccountReader{acct: parent}, + } + + loaded, err := loadEpochInflationAccountStateForReplay(slotCtx) + require.NoError(t, err) + require.Equal(t, stagedState, loaded, + "the first executed bank after eight boundary skips must see its staged epoch inflation state") +} + +func TestLoadEpochInflationAccountStateForReplayFallsBackToSpeculativeParent(t *testing.T) { + parentState := EpochInflationAccountState{ + Current: EpochInflationState{Epoch: 129, SlotsPerEpoch: 54_000}, + } + key := VoteRewardAccountAddr() + parent := &accounts.Account{ + Key: key, + Data: encodeEpochInflationAccountState(parentState), + } + slotCtx := &sealevel.SlotCtx{ + Accounts: accounts.NewMemAccounts(), + Slot: 7_019_999, + ParentSlot: 7_019_998, + UnrootedRead: rewardAccountReader{acct: parent}, + } + + loaded, err := loadEpochInflationAccountStateForReplay(slotCtx) + require.NoError(t, err) + require.Equal(t, parentState, loaded) +} diff --git a/pkg/replay/leader_finalize.go b/pkg/replay/leader_finalize.go index 13b74cf0..c8aca5dd 100644 --- a/pkg/replay/leader_finalize.go +++ b/pkg/replay/leader_finalize.go @@ -148,11 +148,7 @@ func CommitLeaderSlot(in CommitLeaderInput) (*sealevel.SlotCtx, error) { if err := ensureParentAccountsForModified(slotCtx, modified); err != nil { return nil, err } - if slotCtx.ModifiedAccountsFromDelta { - slotCtx.FinalBankhash = bankhash.CalculateBankHashUniqueModified(slotCtx, writable, modified, block.ParentBankhash, block.NumSignatures, block.Blockhash) - } else { - slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writable, modified, block.ParentBankhash, block.NumSignatures, block.Blockhash) - } + slotCtx.FinalBankhash = bankhash.CalculateBankHash(slotCtx, writable, modified, block.ParentBankhash, block.NumSignatures, block.Blockhash) copy(block.ExpectedBankhash[:], slotCtx.FinalBankhash) block.HasExpectedBankhash = true return slotCtx, nil @@ -244,55 +240,7 @@ func ensureParentAccountsForModified(slotCtx *sealevel.SlotCtx, modified []*acco return nil } -func uniqueOverlayModifiedAccounts(slotCtx *sealevel.SlotCtx, extras ...[]*accounts.Account) ([]*accounts.Account, bool) { - if !slotCtx.ModifiedAccountsFromDelta { - return nil, false - } - overlay, ok := slotCtx.Accounts.(*accounts.OverlayAccounts) - if !ok { - return nil, false - } - delta := overlay.DeltaAccounts() - modified := delta[:0] - for _, acct := range delta { - if acct != nil { - modified = append(modified, acct) - } - } - extraCount := 0 - for _, group := range extras { - extraCount += len(group) - } - if extraCount == 0 { - return modified, true - } - - byKey := make(map[solana.PublicKey]int, len(modified)+extraCount) - for idx, acct := range modified { - if acct != nil { - byKey[acct.Key] = idx - } - } - for _, group := range extras { - for _, acct := range group { - if acct == nil { - continue - } - if idx, exists := byKey[acct.Key]; exists { - modified[idx] = acct - continue - } - byKey[acct.Key] = len(modified) - modified = append(modified, acct) - } - } - return modified, true -} - func compileLeaderAccounts(slotCtx *sealevel.SlotCtx, block *b.Block, rentAccts []*accounts.Account) ([]*accounts.Account, []*accounts.Account) { - if modified, ok := uniqueOverlayModifiedAccounts(slotCtx, block.EpochUpdatedAccts, rentAccts); ok { - return nil, modified - } adhRemoved := accountsDeltaHashRemoved(slotCtx) var writable []*accounts.Account var seenWritable map[solana.PublicKey]struct{} diff --git a/pkg/replay/lean_writable_fastpath_test.go b/pkg/replay/lean_writable_fastpath_test.go index 08d72d70..6a582476 100644 --- a/pkg/replay/lean_writable_fastpath_test.go +++ b/pkg/replay/lean_writable_fastpath_test.go @@ -364,32 +364,51 @@ func TestCompileLeaderAccountsGatesWritableListOnADH(t *testing.T) { } } -func TestOverlayModifiedFastPathAvoidsGlobalMapAndKeepsUniqueNewestValues(t *testing.T) { +func TestNewSlotCtxRetainsCanonicalModifiedJournalAfterADHRemoval(t *testing.T) { + featureSet := features.NewFeaturesDefault() + featureSet.EnableFeature(features.RemoveAccountsDeltaHash, 0) + parent := accounts.NewMemAccounts() + overlay := accounts.NewOverlayAccounts(parent) + block := &b.Block{ + Slot: 43, + Features: featureSet, + } + slotCtx := newSlotCtx(block, overlay, parent, nil, nil, 8) + + key := solana.PublicKey{0xa1} + acct := &accounts.Account{Key: key, Lamports: 10} + require.NoError(t, slotCtx.SetAccount(key, acct)) + slotCtx.RecordModifiedAcct(key) + + assert.Contains(t, slotCtx.ModifiedAccts, key, + "the canonical journal remains required even when AccountsDeltaHash is removed") + assert.Empty(t, slotCtx.WritableAccts, + "AccountsDeltaHash removal may omit only the writable-account journal") +} + +func TestCanonicalModifiedJournalDoesNotResurrectBurnedEpochVAT(t *testing.T) { slotCtx, cleanup := newCommitTestSlotCtx() defer cleanup() slotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) overlay := accounts.NewOverlayAccounts(slotCtx.Accounts) slotCtx.Accounts = overlay - slotCtx.ModifiedAccountsFromDelta = true - - firstKey := solana.PublicKey{0xa1} - first := &accounts.Account{Key: firstKey, Lamports: 10} - require.NoError(t, slotCtx.SetAccount(firstKey, first)) - slotCtx.RecordModifiedAcct(firstKey) - slotCtx.RecordModifiedAccountStates([]*accounts.Account{first}, []bool{true}) - assert.Empty(t, slotCtx.ModifiedAccts, "overlay-backed replay must not contend on the legacy modified map") - - replacement := &accounts.Account{Key: firstKey, Lamports: 20} - second := &accounts.Account{Key: solana.PublicKey{0xa2}, Lamports: 30} - modified, ok := uniqueOverlayModifiedAccounts(slotCtx, []*accounts.Account{replacement, second, nil}) - require.True(t, ok) - require.Len(t, modified, 2) - byKey := make(map[solana.PublicKey]*accounts.Account, len(modified)) - for _, acct := range modified { - byKey[acct.Key] = acct + + stagedVAT := &accounts.Account{ + Key: addresses.IncineratorAddr, + Lamports: 76_800_000_000, + Owner: addresses.SystemProgramAddr, + RentEpoch: math.MaxUint64, } - assert.Same(t, replacement, byKey[firstKey]) - assert.Same(t, second, byKey[second.Key]) + require.NoError(t, slotCtx.SetAccount(addresses.IncineratorAddr, stagedVAT)) + + runIncinerator(slotCtx) + require.Equal(t, uint64(76_800_000_000), slotCtx.LamportsBurnt) + + _, modified := compileLeaderAccounts(slotCtx, &b.Block{EpochUpdatedAccts: []*accounts.Account{stagedVAT}}, nil) + require.Len(t, modified, 1) + require.Equal(t, solana.PublicKey(addresses.IncineratorAddr), modified[0].Key) + require.Zero(t, modified[0].Lamports, "bank hash/store input must contain the post-burn account") + require.Equal(t, uint64(math.MaxUint64), modified[0].RentEpoch) } func TestCompileWritableAndModifiedAcctsGatesWritableListOnADH(t *testing.T) { @@ -435,6 +454,8 @@ func TestCompileWritableAndModifiedAcctsGatesWritableListOnADH(t *testing.T) { {Key: sealevel.SysvarSlotHashesAddr, Lamports: 1, Data: slotHashes.MustMarshal()}, {Key: sealevel.SysvarSlotHistoryAddr, Lamports: 1, Data: slotHistory.MustMarshal()}, } + originalRecentData := append([]byte(nil), sysvarAccts[1].Data...) + originalSlotHistoryData := append([]byte(nil), sysvarAccts[3].Data...) for _, acct := range sysvarAccts { require.NoError(t, slotCtx.SetAccount(acct.Key, acct)) } @@ -458,6 +479,30 @@ func TestCompileWritableAndModifiedAcctsGatesWritableListOnADH(t *testing.T) { assert.Empty(t, writable) } assert.ElementsMatch(t, wantKeys, replayTestAccountKeys(modified), "ADH removal must not discard LtHash/store inputs") + + modifiedByKey := make(map[solana.PublicKey]*accounts.Account, len(modified)) + for _, acct := range modified { + modifiedByKey[acct.Key] = acct + } + compiledRecent := modifiedByKey[sealevel.SysvarRecentBlockHashesAddr] + compiledSlotHistory := modifiedByKey[sealevel.SysvarSlotHistoryAddr] + require.NotNil(t, compiledRecent) + require.NotNil(t, compiledSlotHistory) + expectedRecentData := sealevel.SysvarCache.RecentBlockHashes.Sysvar.MustMarshal() + assert.Equal(t, expectedRecentData, compiledRecent.Data[:len(expectedRecentData)], + "bank-hash input must retain the cloned RecentBlockhashes update") + assert.Equal(t, slotHistory.MustMarshal(), compiledSlotHistory.Data, + "bank-hash input must retain the cloned SlotHistory update") + + storedRecent, err := slotCtx.GetAccount(sealevel.SysvarRecentBlockHashesAddr) + require.NoError(t, err) + storedSlotHistory, err := slotCtx.GetAccount(sealevel.SysvarSlotHistoryAddr) + require.NoError(t, err) + assert.Equal(t, originalRecentData, storedRecent.Data, + "the test must exercise a clone-only sysvar update, not an overlay write") + assert.Equal(t, originalSlotHistoryData, storedSlotHistory.Data, + "the test must exercise a clone-only sysvar update, not an overlay write") + assert.Equal(t, slotCtx.Slot+1, slotHistory.NextSlot, "required sysvar updates must still run when ADH is removed") assert.NotZero(t, slotHistory.Bits.Bits.Blocks[0]&(uint64(1)<<(slotCtx.Slot%64))) assert.Equal(t, slotCtx.Blockhash, (*sealevel.SysvarCache.RecentBlockHashes.Sysvar)[0].Blockhash) diff --git a/pkg/replay/publication_concurrency_test.go b/pkg/replay/publication_concurrency_test.go index b1070ba6..7e3fd7e6 100644 --- a/pkg/replay/publication_concurrency_test.go +++ b/pkg/replay/publication_concurrency_test.go @@ -33,12 +33,11 @@ func TestConcurrentDisjointTransactionPublication(t *testing.T) { parent := accounts.NewMemAccounts() overlay := accounts.NewOverlayAccountsWithLen(parent, totalAccounts) slotCtx := &sealevel.SlotCtx{ - Accounts: overlay, - Features: feats, - AcctMapsMu: &sync.Mutex{}, - ModifiedAccts: make(map[solana.PublicKey]bool), - WritableAccts: make(map[solana.PublicKey]bool), - ModifiedAccountsFromDelta: true, + Accounts: overlay, + Features: feats, + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool, totalAccounts), + WritableAccts: make(map[solana.PublicKey]bool), } start := make(chan struct{}) @@ -79,10 +78,9 @@ func TestConcurrentDisjointTransactionPublication(t *testing.T) { } assert.Empty(t, slotCtx.WritableAccts, "ADH-removed replay must not build the unused writable set") - assert.Empty(t, slotCtx.ModifiedAccts, "overlay delta replaces the contended modified-account map") + require.Len(t, slotCtx.ModifiedAccts, totalAccounts) require.Len(t, overlay.DeltaAccounts(), totalAccounts) - for _, modified := range overlay.DeltaAccounts() { - key := modified.Key + for key := range slotCtx.ModifiedAccts { acct, err := overlay.GetAccount((*[32]byte)(&key)) require.NoError(t, err) assert.Equal(t, binary.LittleEndian.Uint64(key[:8]), acct.Lamports) diff --git a/pkg/sealevel/execution_ctx.go b/pkg/sealevel/execution_ctx.go index 74a48f35..d9d6feec 100644 --- a/pkg/sealevel/execution_ctx.go +++ b/pkg/sealevel/execution_ctx.go @@ -79,16 +79,12 @@ type SlotCtx struct { AcctMapsMu *sync.Mutex // AcctMapsMu protects the next 2 maps ModifiedAccts map[solana.PublicKey]bool WritableAccts map[solana.PublicKey]bool - // ModifiedAccountsFromDelta lets replay use OverlayAccounts' already-unique - // branch delta as the LtHash/store input after AccountsDeltaHash removal. - // Transaction publication then avoids the residual global ModifiedAccts lock. - ModifiedAccountsFromDelta bool - NumSignatures uint64 // signatures processed in this bank (resets for every child bank) - Blockhash [32]byte - LastBlockhash [32]byte - SlotBank SlotBank - Features *features.Features - VoteTimestampMu *sync.Mutex + NumSignatures uint64 // signatures processed in this bank (resets for every child bank) + Blockhash [32]byte + LastBlockhash [32]byte + SlotBank SlotBank + Features *features.Features + VoteTimestampMu *sync.Mutex // VoteTimestampsMu protects VoteTimestamps VoteTimestamps map[solana.PublicKey]BlockTimestamp VoteAccts map[solana.PublicKey]uint64 @@ -471,9 +467,6 @@ func (slotCtx *SlotCtx) SetAccount(pubkey solana.PublicKey, acct *accounts.Accou } func (slotCtx *SlotCtx) RecordModifiedAcct(pubkey solana.PublicKey) { - if slotCtx.ModifiedAccountsFromDelta { - return - } slotCtx.AcctMapsMu.Lock() defer slotCtx.AcctMapsMu.Unlock() if slotCtx.Features == nil || !slotCtx.Features.IsActive(features.RemoveAccountsDeltaHash) { @@ -486,9 +479,6 @@ func (slotCtx *SlotCtx) RecordModifiedAccountStates(accountStates []*accounts.Ac if len(accountStates) != len(touched) { panic("account states/touched length mismatch") } - if slotCtx.ModifiedAccountsFromDelta { - return - } slotCtx.AcctMapsMu.Lock() defer slotCtx.AcctMapsMu.Unlock() for idx, acct := range accountStates {