diff --git a/beacon/goclient/attest_test.go b/beacon/goclient/attest_test.go index d230c13d75..2948cdbd2b 100644 --- a/beacon/goclient/attest_test.go +++ b/beacon/goclient/attest_test.go @@ -512,7 +512,7 @@ func createClient( CommonTimeout: defaultHardTimeout, LongTimeout: time.Second, WithWeightedAttestationData: withWeightedAttestationData, - }, 0) + }, 0, BlockFetchPathSafe) if err != nil { return nil, err } diff --git a/beacon/goclient/events_test.go b/beacon/goclient/events_test.go index 05b3aa6b9c..6254e8b5af 100644 --- a/beacon/goclient/events_test.go +++ b/beacon/goclient/events_test.go @@ -251,7 +251,7 @@ func TestNewEventHandler(t *testing.T) { } func eventsTestClient(t *testing.T, serverURL string) *GoClient { - opt, err := NewOptions(Options{BeaconNodeAddr: serverURL}, 0) + opt, err := NewOptions(Options{BeaconNodeAddr: serverURL}, 0, BlockFetchPathSafe) require.NoError(t, err) server, err := New(t.Context(), zap.NewNop(), opt) diff --git a/beacon/goclient/goclient.go b/beacon/goclient/goclient.go index de823542c1..fcc585a9e4 100644 --- a/beacon/goclient/goclient.go +++ b/beacon/goclient/goclient.go @@ -135,12 +135,20 @@ type GoClient struct { weightedAttestationDataSoftTimeout time.Duration weightedAttestationDataHardTimeout time.Duration - // proposalSoftTimeout is the collection period during which we gather proposals - // from multiple beacon nodes to select the best one. After this timeout, we return - // the best proposal seen so far, or wait for the first valid proposal if none - // received yet. The parent context (duty deadline) serves as the hard timeout. + // proposalSoftTimeout is the legacy collection-period timeout used by + // getProposalParallelLegacy. Other paths use proposalSoftDeadline instead. proposalSoftTimeout time.Duration + // proposalSoftDeadline is the slot-relative deadline (ms into slot) for the safe + // and MEV-optimized paths. See docs/MEV_CONSIDERATIONS.md. + proposalSoftDeadline time.Duration + + // blockFetchPath selects the multi-BN block-fetch strategy GetBeaconBlock + // dispatches to: getProposalParallelLegacy for BlockFetchPathLegacy, or + // getProposalParallelByDeadline (with earlyExitOnBlinded set per path) for + // BlockFetchPathSafe and BlockFetchPathMEVOptimized. + blockFetchPath BlockFetchPath + // blockRootToSlotCache is used for attestation data scoring. When multiple Consensus clients are used, // the cache helps reduce the number of Consensus Client calls by `n-1`, where `n` is the number of Consensus clients // that successfully fetched attestation data and proceeded to the scoring phase. Capacity is rather an arbitrary number, @@ -201,6 +209,8 @@ func New(ctx context.Context, logger *zap.Logger, opt Options) (*GoClient, error weightedAttestationDataSoftTimeout: time.Duration(float64(opt.CommonTimeout) / 2.5), weightedAttestationDataHardTimeout: opt.CommonTimeout, proposalSoftTimeout: opt.ProposalSoftTimeout, + proposalSoftDeadline: opt.ProposalSoftDeadline, + blockFetchPath: opt.BlockFetchPath, supportedTopics: []eventTopic{eventTopicHead, eventTopicBlock}, activatedClients: hashmap.New[string, struct{}](), } diff --git a/beacon/goclient/options.go b/beacon/goclient/options.go index fae1208e58..8439bb9ef0 100644 --- a/beacon/goclient/options.go +++ b/beacon/goclient/options.go @@ -1,6 +1,7 @@ package goclient import ( + "fmt" "time" "github.com/ssvlabs/ssv/networkconfig" @@ -14,6 +15,89 @@ const ( defaultLongTimeout = time.Second * 60 ) +// BlockFetchPath identifies which block-header fetch strategy the SSV node is using. +// Determined at startup from operator-provided config; see DetermineBlockFetchPath. +// +// Documented end-to-end in docs/MEV_CONSIDERATIONS.md. +type BlockFetchPath int + +const ( + // BlockFetchPathSafe is the default. Multi-BN parallel fetch with early-exit on + // first blinded response; fallback at slot-relative ProposalSoftDeadline + // (defaults to DefaultProposalSoftDeadline when the operator hasn't set it). + BlockFetchPathSafe BlockFetchPath = iota + // BlockFetchPathLegacy preserves the original ProposerDelay / ProposalSoftTimeout + // behavior bit-for-bit; selected when an operator has set either of those legacy knobs. + BlockFetchPathLegacy + // BlockFetchPathMEVOptimized is opt-in. Multi-BN parallel fetch without early-exit, + // returns the best-scored response collected by ProposalSoftDeadline. Selected when an + // operator sets ProposalSoftDeadline explicitly. + BlockFetchPathMEVOptimized +) + +// String returns a human-readable label for logging. +func (p BlockFetchPath) String() string { + switch p { + case BlockFetchPathSafe: + return "safe" + case BlockFetchPathLegacy: + return "legacy" + case BlockFetchPathMEVOptimized: + return "mev-optimized" + default: + return fmt.Sprintf("unknown(%d)", int(p)) + } +} + +// ProposalSoftDeadline bounds and defaults. Values are slot-relative (measured from slot start). +const ( + // SafeMaxProposalSoftDeadline is the startup-warning threshold for the SSV-side + // ProposalSoftDeadline. Above this value, the worst-case 2-round QBFT scenario + // has no safety margin for latency variance — round 1 effectively has to succeed + // in setups with typical latencies. + // + // Strict math from the typical values in docs/MEV_CONSIDERATIONS.md gives a hard + // upper bound of ProposalSoftDeadline <= 1500ms: + // ProposalSoftDeadline + 2350ms (QBFT worst-case 2-round) + + // 50ms (PostConsensusSigning) + 100ms (BlockSubmission) <= 4000ms (slot deadline) + // => ProposalSoftDeadline <= 1500ms + // + // We set the warning threshold 50ms tighter (1450ms) to preserve a buffer for + // latency variance. Operators following docs/MEV_CONSIDERATIONS.md's recommended + // "PBS cutoff + 50ms BN→SSV transport" formula stay within this threshold when + // their PBS cutoff sits at the recommended ~1400ms ceiling; pushing PBS cutoff + // up to the strict ~1450ms still works in clusters with measurably faster QBFT + + // submission, but consumes the variance buffer (and trips this warning). + SafeMaxProposalSoftDeadline = 1450 * time.Millisecond + + // DefaultProposalSoftDeadline is the default deadline used by the safe path when + // the operator hasn't set ProposalSoftDeadline. Equal to SafeMaxProposalSoftDeadline + // — the largest value that keeps the 50ms latency-variance buffer described above. + DefaultProposalSoftDeadline = SafeMaxProposalSoftDeadline + + // MinProposalSoftDeadline is the lower bound for operator-set ProposalSoftDeadline + // values. Decoupled from DefaultProposalSoftDeadline so operators can opt into the + // MEV-optimized path with a tighter window than the safe-path default if they want + // (e.g., to match an early PBS cutoff). Set at 1000ms — below this, the BN response + // window becomes too tight for meaningful bid collection across BNs. + MinProposalSoftDeadline = 1000 * time.Millisecond + + // MaxProposalSoftDeadline is the hard upper bound for operator-set ProposalSoftDeadline + // values. Intentionally loose — past the SafeMax warning threshold, the operator has + // already opted into "round 1 must succeed". This cap exists to accommodate exceptionally + // performant clusters that can complete the entire post-header pipeline (QBFT round 1 + + // signing + submission) in well under 350ms and want to capture as much of the slot's + // bid growth as possible. Operators in this regime should baseline their own latencies + // (see docs/MEV_CONSIDERATIONS.md "Tuning guidance") before going anywhere near the cap. + MaxProposalSoftDeadline = 3600 * time.Millisecond +) + +// Legacy-path constants — preserved for backward-compat. +const ( + defaultProposalSoftTimeout = 1800 * time.Millisecond + minProposalSoftTimeout = 500 * time.Millisecond +) + // Options defines beacon client options type Options struct { BeaconConfig *networkconfig.Beacon @@ -25,42 +109,140 @@ type Options struct { CommonTimeout time.Duration `yaml:"CommonTimeout" env:"WITH_COMMON_TIMEOUT" env-description:"Specifies the common timeout for network operations"` LongTimeout time.Duration `yaml:"LongTimeout" env:"WITH_LONG_TIMEOUT" env-description:"Specifies the long timeout for network operations"` - ProposalSoftTimeout time.Duration `yaml:"ProposalSoftTimeout" env:"WITH_PROPOSAL_SOFT_TIMEOUT" env-description:"Specifies the beacon proposal collection soft timeout (collection period for comparing proposals from multiple beacon nodes to select the most profitable one). Note: the 1st MEV (blinded) block is accepted immediately, so this timeout mainly affects how long we wait for an MEV block before giving up deciding to use a vanilla block instead (if we got one already). This value cannot be set any lower than 500ms to ensure there is enough time for the Beacon node to serve the block-fetch request"` + // ProposalSoftTimeout is the legacy collection-period timeout in multi-BN parallel + // fetch. Setting this (or ProposerDelay) selects BlockFetchPathLegacy. New operators + // should prefer ProposalSoftDeadline. See docs/MEV_CONSIDERATIONS.md. + ProposalSoftTimeout time.Duration `yaml:"ProposalSoftTimeout" env:"WITH_PROPOSAL_SOFT_TIMEOUT" env-description:"Legacy MEV configuration. Specifies the beacon proposal collection soft timeout (collection period for comparing proposals from multiple beacon nodes to select the most profitable one). Setting this opts the SSV node into the legacy block-fetch path; the recommended approach is to leave this unset and use ProposalSoftDeadline instead. See https://github.com/ssvlabs/ssv/blob/main/docs/MEV_CONSIDERATIONS.md for details."` + + // ProposalSoftDeadline is the slot-relative deadline (in ms-into-slot) for the + // multi-BN proposal-collection window used by the safe and MEV-optimized paths. + // - Unset (zero) -> safe path, defaults to DefaultProposalSoftDeadline. + // - Set explicitly -> MEV-optimized path, value must be in + // [MinProposalSoftDeadline, MaxProposalSoftDeadline]. + // Cannot be combined with ProposerDelay or ProposalSoftTimeout (which select the + // legacy path). + ProposalSoftDeadline time.Duration `yaml:"ProposalSoftDeadline" env:"WITH_PROPOSAL_SOFT_DEADLINE" env-description:"Slot-relative deadline (ms into slot) for the multi-BN proposal-collection window. Leave unset for the default safe path; set explicitly to opt into the MEV-optimized path (value must be in [1000ms, 3600ms]). Cannot be combined with ProposerDelay or ProposalSoftTimeout. See https://github.com/ssvlabs/ssv/blob/main/docs/MEV_CONSIDERATIONS.md for details."` + + // BlockFetchPath is set by NewOptions from the determined path; not directly + // configured by the operator. Consumed by GoClient at runtime to dispatch block + // fetching to the correct strategy. + BlockFetchPath BlockFetchPath `yaml:"-"` } -func NewOptions(base Options, proposerDelay time.Duration) (Options, error) { +// DetermineBlockFetchPath returns the block-fetch path selected by the operator's config. +// +// Must be called with raw operator-provided values (before NewOptions applies any defaults) +// so that the "operator explicitly set" vs "defaulted" distinction is preserved. +// +// Returns an error when: +// - any of the MEV-related duration knobs is negative; or +// - the config combines legacy knobs (ProposerDelay / ProposalSoftTimeout) with +// the MEV-optimized ProposalSoftDeadline — operators must pick one. +func DetermineBlockFetchPath(base Options, proposerDelay time.Duration) (BlockFetchPath, error) { + // Negative values are nonsensical for any of these and would silently be + // treated as "unset" by the `> 0` checks below — reject them upfront so the + // operator gets a clear startup error instead of a confusing late-firing + // soft-deadline or skipped legacy-path selection. + if proposerDelay < 0 { + return 0, fmt.Errorf("ProposerDelay must be non-negative, got %v", proposerDelay) + } + if base.ProposalSoftTimeout < 0 { + return 0, fmt.Errorf("ProposalSoftTimeout must be non-negative, got %v", base.ProposalSoftTimeout) + } + if base.ProposalSoftDeadline < 0 { + return 0, fmt.Errorf("ProposalSoftDeadline must be non-negative, got %v", base.ProposalSoftDeadline) + } + + legacySet := proposerDelay > 0 || base.ProposalSoftTimeout > 0 + deadlineSet := base.ProposalSoftDeadline > 0 + + if legacySet && deadlineSet { + return 0, fmt.Errorf("ProposalSoftDeadline conflicts with legacy ProposerDelay/ProposalSoftTimeout config — remove one. See docs/MEV_CONSIDERATIONS.md for path selection guidance") + } + + switch { + case legacySet: + return BlockFetchPathLegacy, nil + case deadlineSet: + return BlockFetchPathMEVOptimized, nil + default: + return BlockFetchPathSafe, nil + } +} + +// ValidateProposalSoftDeadline ensures the value is within the acceptable range for +// the MEV-optimized fetch path. The caller is responsible for emitting an additional +// log-warning when the value exceeds SafeMaxProposalSoftDeadline. +func ValidateProposalSoftDeadline(d time.Duration) error { + if d < MinProposalSoftDeadline || d > MaxProposalSoftDeadline { + return fmt.Errorf("ProposalSoftDeadline value %dms is out of range [%dms, %dms]", + d.Milliseconds(), + MinProposalSoftDeadline.Milliseconds(), + MaxProposalSoftDeadline.Milliseconds()) + } + return nil +} + +// NewOptions applies path-specific defaults to base options and returns the result. +// +// path is the value returned by DetermineBlockFetchPath. proposerDelay is only consumed +// when path == BlockFetchPathLegacy. The selected path is stashed in the returned +// Options.BlockFetchPath for consumption by GoClient at runtime. +func NewOptions(base Options, proposerDelay time.Duration, path BlockFetchPath) (Options, error) { options := base + options.BlockFetchPath = path if options.CommonTimeout == 0 { options.CommonTimeout = defaultCommonTimeout } - if options.LongTimeout == 0 { options.LongTimeout = defaultLongTimeout } - // If user explicitly set ProposalSoftTimeout, use it as-is (power user mode). - // Otherwise, use the default value and reduce it by proposer delay if needed. - if options.ProposalSoftTimeout == 0 { - // The default value shouldn't be too high because an operator might not be able to participate - // in QBFT round 2 (or finish it in time) if it is roughly > 2000 ms. - const defaultProposalSoftTimeout = time.Millisecond * 1800 - options.ProposalSoftTimeout = defaultProposalSoftTimeout - // Reduce soft timeout by proposer delay to maintain consistent duty-execution timelines - // for different operators in the cluster, ensuring QBFT consensus starts at roughly - // the same time (timing out round 1 at roughly the same time) regardless of proposer - // delay configuration a particular operator is using - operators with higher proposer - // delay start fetching blocks later, so they must have a shorter collection period. - if proposerDelay > 0 { - options.ProposalSoftTimeout -= proposerDelay + switch path { + case BlockFetchPathLegacy: + // Legacy path: preserve the original ProposalSoftTimeout defaulting (1800ms, + // reduced by ProposerDelay, floored at 500ms). Behavior bit-for-bit unchanged + // from before the path split was introduced. + if options.ProposalSoftTimeout == 0 { + options.ProposalSoftTimeout = defaultProposalSoftTimeout + // Reduce by proposer delay to maintain consistent duty-execution timelines + // for different operators in the cluster, ensuring QBFT consensus starts at + // roughly the same time regardless of proposer-delay configuration. + if proposerDelay > 0 { + options.ProposalSoftTimeout -= proposerDelay + } } - } + if options.ProposalSoftTimeout < minProposalSoftTimeout { + options.ProposalSoftTimeout = minProposalSoftTimeout + } + + case BlockFetchPathSafe: + // Safe path: slot-relative deadline, default DefaultProposalSoftDeadline. + // + // The == 0 check is defensive: in production, DetermineBlockFetchPath only + // routes ProposalSoftDeadline == 0 to the safe path (a non-zero value selects + // MEV-optimized), so this branch is always taken when path == safe. The check + // guards tests that construct Options directly and bypass DetermineBlockFetchPath. + if options.ProposalSoftDeadline == 0 { + options.ProposalSoftDeadline = DefaultProposalSoftDeadline + } + + case BlockFetchPathMEVOptimized: + // MEV-optimized path: ProposalSoftDeadline is set by the operator. No defaults + // to apply. + // + // Note: range validation via ValidateProposalSoftDeadline is the *caller's* + // responsibility — cli/operator/node.go runs it for production startup, but + // NewOptions does not enforce it. Tests that bypass the CLI should call + // ValidateProposalSoftDeadline themselves if they want the bounds check. - // minProposalSoftTimeout is the minimum soft timeout value allowed. - // It ensures we always have enough time to fetch and compare proposals. - const minProposalSoftTimeout = time.Millisecond * 500 - if options.ProposalSoftTimeout < minProposalSoftTimeout { - options.ProposalSoftTimeout = minProposalSoftTimeout + default: + // Defense-in-depth: DetermineBlockFetchPath returns only the three values + // above, but callers that construct Options directly (notably tests) can + // reach here. Reject at startup rather than at per-slot dispatch in + // proposer.go. + return Options{}, fmt.Errorf("unknown block-fetch path %d", path) } // Note: There is no hard timeout for proposals. The parent context from the diff --git a/beacon/goclient/options_test.go b/beacon/goclient/options_test.go new file mode 100644 index 0000000000..b0b9308e3e --- /dev/null +++ b/beacon/goclient/options_test.go @@ -0,0 +1,227 @@ +package goclient + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetermineBlockFetchPath(t *testing.T) { + tests := []struct { + name string + options Options + proposerDelay time.Duration + wantPath BlockFetchPath + wantErr string // substring match; empty = no error expected + }{ + { + name: "nothing set -> safe (default)", + options: Options{}, + proposerDelay: 0, + wantPath: BlockFetchPathSafe, + }, + { + name: "only ProposerDelay set -> legacy", + options: Options{}, + proposerDelay: 300 * time.Millisecond, + wantPath: BlockFetchPathLegacy, + }, + { + name: "only ProposalSoftTimeout set -> legacy", + options: Options{ProposalSoftTimeout: 1500 * time.Millisecond}, + proposerDelay: 0, + wantPath: BlockFetchPathLegacy, + }, + { + name: "both ProposerDelay and ProposalSoftTimeout set -> legacy", + options: Options{ProposalSoftTimeout: 1500 * time.Millisecond}, + proposerDelay: 300 * time.Millisecond, + wantPath: BlockFetchPathLegacy, + }, + { + name: "only ProposalSoftDeadline set -> MEV-optimized", + options: Options{ProposalSoftDeadline: 1100 * time.Millisecond}, + proposerDelay: 0, + wantPath: BlockFetchPathMEVOptimized, + }, + { + name: "ProposerDelay + ProposalSoftDeadline -> error", + options: Options{ProposalSoftDeadline: 1100 * time.Millisecond}, + proposerDelay: 300 * time.Millisecond, + wantErr: "ProposalSoftDeadline conflicts with legacy", + }, + { + name: "ProposalSoftTimeout + ProposalSoftDeadline -> error", + options: Options{ProposalSoftTimeout: 1500 * time.Millisecond, ProposalSoftDeadline: 1100 * time.Millisecond}, + proposerDelay: 0, + wantErr: "ProposalSoftDeadline conflicts with legacy", + }, + { + name: "all three set -> error (legacy + deadline still conflicts)", + options: Options{ProposalSoftTimeout: 1500 * time.Millisecond, ProposalSoftDeadline: 1100 * time.Millisecond}, + proposerDelay: 300 * time.Millisecond, + wantErr: "ProposalSoftDeadline conflicts with legacy", + }, + { + name: "negative ProposerDelay -> error", + options: Options{}, + proposerDelay: -100 * time.Millisecond, + wantErr: "ProposerDelay must be non-negative", + }, + { + name: "negative ProposalSoftTimeout -> error", + options: Options{ProposalSoftTimeout: -100 * time.Millisecond}, + proposerDelay: 0, + wantErr: "ProposalSoftTimeout must be non-negative", + }, + { + name: "negative ProposalSoftDeadline -> error (would otherwise silently fall to safe path)", + options: Options{ProposalSoftDeadline: -100 * time.Millisecond}, + proposerDelay: 0, + wantErr: "ProposalSoftDeadline must be non-negative", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := DetermineBlockFetchPath(tt.options, tt.proposerDelay) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantPath, path) + }) + } +} + +func TestValidateProposalSoftDeadline(t *testing.T) { + tests := []struct { + name string + value time.Duration + wantErr bool + }{ + {name: "at minimum (1000ms) -> ok", value: 1000 * time.Millisecond, wantErr: false}, + {name: "below minimum (999ms) -> error", value: 999 * time.Millisecond, wantErr: true}, + {name: "below safe max (1100ms) -> ok", value: 1100 * time.Millisecond, wantErr: false}, + {name: "at safe max (1450ms) -> ok (warn handled externally)", value: 1450 * time.Millisecond, wantErr: false}, + {name: "above safe max but below hard max (2500ms) -> ok", value: 2500 * time.Millisecond, wantErr: false}, + {name: "at hard max (3600ms) -> ok", value: 3600 * time.Millisecond, wantErr: false}, + {name: "above hard max (3601ms) -> error", value: 3601 * time.Millisecond, wantErr: true}, + {name: "zero -> error", value: 0, wantErr: true}, + {name: "negative -> error", value: -100 * time.Millisecond, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateProposalSoftDeadline(tt.value) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "out of range") + return + } + require.NoError(t, err) + }) + } +} + +func TestNewOptions_PathDefaulting(t *testing.T) { + t.Run("safe path defaults ProposalSoftDeadline to the largest safest value", func(t *testing.T) { + base := Options{BeaconNodeAddr: "http://localhost:5052"} + opts, err := NewOptions(base, 0, BlockFetchPathSafe) + require.NoError(t, err) + assert.Equal(t, DefaultProposalSoftDeadline, opts.ProposalSoftDeadline) + assert.Equal(t, BlockFetchPathSafe, opts.BlockFetchPath) + // ProposalSoftTimeout should not be touched by the safe path. + assert.Equal(t, time.Duration(0), opts.ProposalSoftTimeout) + }) + + t.Run("safe path keeps operator-set ProposalSoftDeadline", func(t *testing.T) { + base := Options{ + BeaconNodeAddr: "http://localhost:5052", + ProposalSoftDeadline: 1500 * time.Millisecond, + } + opts, err := NewOptions(base, 0, BlockFetchPathSafe) + require.NoError(t, err) + assert.Equal(t, 1500*time.Millisecond, opts.ProposalSoftDeadline) + }) + + t.Run("legacy path defaults ProposalSoftTimeout to 1800ms", func(t *testing.T) { + base := Options{BeaconNodeAddr: "http://localhost:5052"} + opts, err := NewOptions(base, 0, BlockFetchPathLegacy) + require.NoError(t, err) + assert.Equal(t, defaultProposalSoftTimeout, opts.ProposalSoftTimeout) + assert.Equal(t, BlockFetchPathLegacy, opts.BlockFetchPath) + }) + + t.Run("legacy path subtracts proposer delay from ProposalSoftTimeout", func(t *testing.T) { + base := Options{BeaconNodeAddr: "http://localhost:5052"} + opts, err := NewOptions(base, 300*time.Millisecond, BlockFetchPathLegacy) + require.NoError(t, err) + assert.Equal(t, defaultProposalSoftTimeout-300*time.Millisecond, opts.ProposalSoftTimeout) + }) + + t.Run("legacy path floors ProposalSoftTimeout at 500ms", func(t *testing.T) { + // With ProposerDelay = 1500ms, the natural ProposalSoftTimeout would be + // 1800ms - 1500ms = 300ms, which is below the 500ms floor. + base := Options{BeaconNodeAddr: "http://localhost:5052"} + opts, err := NewOptions(base, 1500*time.Millisecond, BlockFetchPathLegacy) + require.NoError(t, err) + assert.Equal(t, minProposalSoftTimeout, opts.ProposalSoftTimeout) + }) + + t.Run("legacy path keeps operator-set ProposalSoftTimeout (no reduction)", func(t *testing.T) { + // When the operator explicitly sets ProposalSoftTimeout, the legacy path + // uses it as-is without subtracting ProposerDelay (power-user mode). + base := Options{ + BeaconNodeAddr: "http://localhost:5052", + ProposalSoftTimeout: 1200 * time.Millisecond, + } + opts, err := NewOptions(base, 300*time.Millisecond, BlockFetchPathLegacy) + require.NoError(t, err) + assert.Equal(t, 1200*time.Millisecond, opts.ProposalSoftTimeout) + }) + + t.Run("MEV-optimized path keeps operator-set ProposalSoftDeadline", func(t *testing.T) { + base := Options{ + BeaconNodeAddr: "http://localhost:5052", + ProposalSoftDeadline: 1850 * time.Millisecond, + } + opts, err := NewOptions(base, 0, BlockFetchPathMEVOptimized) + require.NoError(t, err) + assert.Equal(t, 1850*time.Millisecond, opts.ProposalSoftDeadline) + assert.Equal(t, BlockFetchPathMEVOptimized, opts.BlockFetchPath) + // ProposalSoftTimeout should not be touched. + assert.Equal(t, time.Duration(0), opts.ProposalSoftTimeout) + }) + + t.Run("unknown path -> error at startup", func(t *testing.T) { + // Defense-in-depth: callers that bypass DetermineBlockFetchPath and pass an + // invalid BlockFetchPath value should fail at NewOptions rather than at the + // per-slot dispatch in proposer.go. + base := Options{BeaconNodeAddr: "http://localhost:5052"} + _, err := NewOptions(base, 0, BlockFetchPath(99)) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown block-fetch path") + }) +} + +func TestBlockFetchPath_String(t *testing.T) { + tests := []struct { + path BlockFetchPath + want string + }{ + {BlockFetchPathSafe, "safe"}, + {BlockFetchPathLegacy, "legacy"}, + {BlockFetchPathMEVOptimized, "mev-optimized"}, + {BlockFetchPath(99), "unknown(99)"}, + } + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + assert.Equal(t, tt.want, tt.path.String()) + }) + } +} diff --git a/beacon/goclient/proposer.go b/beacon/goclient/proposer.go index a072c476f0..b1a3d048ac 100644 --- a/beacon/goclient/proposer.go +++ b/beacon/goclient/proposer.go @@ -106,8 +106,19 @@ func (gc *GoClient) GetBeaconBlock( return nil, nil, err } } else { - // For multiple clients, race them in parallel for the fastest response - beaconBlock, err = gc.getProposalParallel(ctx, logger, slot, sig, graffiti) + // For multiple clients, dispatch to the selected block-fetch path. + // Safe and MEV-optimized share an implementation differing only in whether + // to early-exit on the first blinded response. See docs/MEV_CONSIDERATIONS.md. + switch gc.blockFetchPath { + case BlockFetchPathSafe: + beaconBlock, err = gc.getProposalParallelByDeadline(ctx, logger, slot, sig, graffiti, true /* earlyExitOnBlinded */) + case BlockFetchPathLegacy: + beaconBlock, err = gc.getProposalParallelLegacy(ctx, logger, slot, sig, graffiti) + case BlockFetchPathMEVOptimized: + beaconBlock, err = gc.getProposalParallelByDeadline(ctx, logger, slot, sig, graffiti, false /* earlyExitOnBlinded */) + default: + return nil, nil, fmt.Errorf("unknown block-fetch path %d", gc.blockFetchPath) + } if err != nil { return nil, nil, err } @@ -155,25 +166,18 @@ func (gc *GoClient) GetBeaconBlock( } } -// getProposalParallel races all beacon nodes and collects proposals for a short time -// and returns the best one according to our score function. -// If no valid proposals are collected in this time it returns the first valid one -// it sees. -// -// This minimizes latency for time-critical block proposals, while still affording -// some time for selecting maximally profitable proposals. Remaining requests are -// canceled immediately to reduce load. +// getProposalParallelLegacy implements the legacy block-fetch path — preserved +// bit-for-bit from the pre-path-split code for backward-compat with operators using +// ProposerDelay / ProposalSoftTimeout. // -// Note: We used to prioritize speed over fee recipient validation - returning -// the first response rather than waiting to compare fee recipients, as missing -// a proposal slot is worse than a nil fee recipient. -// However, it has been observed that the first proposal is usually not the most -// profitable, so we added a little slack time to collect proposals. +// Races all beacon nodes, collects proposals for a short relative-duration timeout +// (gc.proposalSoftTimeout), and returns the best one according to our score function. +// Early-exits on the first blinded response (assumes blinded == MEV). If no valid +// proposals are collected by the soft timeout, returns the first valid one received. // // The parent context (from duty runner, bounded by slot timing) serves as the hard -// deadline. We never give up early on getting a block proposal - missing a proposal -// is catastrophic, so we wait as long as the slot allows. -func (gc *GoClient) getProposalParallel( +// deadline. +func (gc *GoClient) getProposalParallelLegacy( ctx context.Context, logger *zap.Logger, slot phase0.Slot, @@ -315,6 +319,181 @@ collect: return nil, fmt.Errorf("all %d clients failed to get proposal for slot %d, encountered errors: %w", len(gc.clients), slot, errs) } +// proposalFetchResult bundles the outcome of a single per-BN fetch goroutine. +type proposalFetchResult struct { + proposal *api.VersionedProposal + err error + client string +} + +// spawnProposalFetchers starts a goroutine per beacon-node client; each goroutine +// fetches a proposal and writes its result to the returned channel. Used by the +// safe and MEV-optimized block-fetch implementations. +// +// The channel is buffered to `len(gc.clients)` so each goroutine can write without +// blocking even if the consumer has already returned. +func (gc *GoClient) spawnProposalFetchers( + ctx context.Context, + slot phase0.Slot, + sig phase0.BLSSignature, + graffiti [32]byte, +) <-chan proposalFetchResult { + resultCh := make(chan proposalFetchResult, len(gc.clients)) + for _, client := range gc.clients { + go func(c Client) { + proposal, err := gc.fetchProposal(ctx, c, slot, sig, graffiti) + select { + case resultCh <- proposalFetchResult{proposal: proposal, err: err, client: c.Address()}: + case <-ctx.Done(): + // Context canceled, exit without blocking. + } + }(client) + } + return resultCh +} + +// waitForFirstValidProposal returns the first valid proposal received from the +// remaining in-flight fetchers. Used by the safe and MEV-optimized paths as the +// fallback after the soft-deadline collection window has elapsed without producing +// a usable best proposal. Bounded by the parent context's slot deadline. +func (gc *GoClient) waitForFirstValidProposal( + ctx context.Context, + logger *zap.Logger, + slot phase0.Slot, + startCollect time.Time, + resultCh <-chan proposalFetchResult, + pendingClients int, + errs error, +) (*api.VersionedProposal, error) { + for pendingClients > 0 { + select { + case res := <-resultCh: + pendingClients-- + if res.err != nil { + errs = errors.Join(errs, res.err) + continue + } + proposalScore := gc.scoreProposal(res.proposal) + logger.Debug("received proposal; selected first proposal", + zap.String("client", res.client), + zap.Float64("score", proposalScore), + zap.Duration("latency", time.Since(startCollect)), + zap.Int("pending", pendingClients), + zap.Bool("blinded", res.proposal.Blinded), + fields.Slot(slot), + ) + return res.proposal, nil + case <-ctx.Done(): + // Preserve any accumulated BN failure context alongside the parent + // deadline error — operators need both to diagnose missed slots. + return nil, errors.Join(ctx.Err(), errs) + } + } + return nil, fmt.Errorf("all %d clients failed to get proposal for slot %d, encountered errors: %w", len(gc.clients), slot, errs) +} + +// getProposalParallelByDeadline implements the slot-relative-deadline parallel +// block-fetch shared by the safe and MEV-optimized paths. +// +// Spawns a per-BN fetch in parallel; collects responses until the slot-relative +// ProposalSoftDeadline (slot_start + gc.proposalSoftDeadline) fires. The +// earlyExitOnBlinded flag distinguishes the two paths: +// - true (safe path): returns immediately on the first blinded response +// (treats blinded == MEV). +// - false (MEV-optimized path): keeps collecting after blinded so the +// highest-value bid across BNs can be selected. +// +// After the deadline (or early-exit), returns the best-scored proposal collected +// so far, or falls through to waitForFirstValidProposal if nothing usable arrived. +// The parent ctx serves as the hard deadline. +// +// Note: in-flight BN fetches are spawned with the parent ctx (not softCtx), so a +// slow BN's HTTP call may keep running after we return — until the duty's slot +// deadline cancels ctx. Matches legacy behavior; the fetchProposal call's own +// HTTP timeouts bound the worst case. +func (gc *GoClient) getProposalParallelByDeadline( + ctx context.Context, + logger *zap.Logger, + slot phase0.Slot, + sig phase0.BLSSignature, + graffiti [32]byte, + earlyExitOnBlinded bool, +) (*api.VersionedProposal, error) { + // Slot-relative deadline: fires at slot_start + ProposalSoftDeadline regardless + // of when this function is invoked. + slotStart := gc.getBeaconConfig().SlotStartTime(slot) + softCtx, cancelSoft := context.WithDeadline(ctx, slotStart.Add(gc.proposalSoftDeadline)) + defer cancelSoft() + + resultCh := gc.spawnProposalFetchers(ctx, slot, sig, graffiti) + + var errs error + var bestProposal *api.VersionedProposal + var bestScore float64 + var bestClient string + + startCollect := time.Now() + pendingClients := len(gc.clients) +collect: + for pendingClients > 0 { + select { + case res := <-resultCh: + pendingClients-- + + if res.err != nil { + errs = errors.Join(errs, res.err) + continue + } + + proposalScore := gc.scoreProposal(res.proposal) + logger.Debug("received proposal", + zap.String("client", res.client), + zap.Float64("score", proposalScore), + zap.Duration("latency", time.Since(startCollect)), + zap.Int("pending", pendingClients), + zap.Bool("blinded", res.proposal.Blinded), + fields.Slot(slot), + ) + + if bestProposal == nil || + proposalScore > bestScore || + // prefer the blinded proposal even if same score as the best so far + (res.proposal.Blinded && proposalScore == bestScore) { + bestProposal = res.proposal + bestScore = proposalScore + bestClient = res.client + } + + if earlyExitOnBlinded && res.proposal.Blinded { + // Safe path: treat blinded == MEV and return immediately. + // MEV-optimized path keeps collecting to compare bids across BNs. + break collect + } + + case <-softCtx.Done(): + break collect + } + } + + if bestProposal != nil { + logger.Debug("selected best proposal", + zap.String("client", bestClient), + zap.Float64("score", bestScore), + zap.Bool("blinded", bestProposal.Blinded), + fields.Slot(slot), + ) + return bestProposal, nil + } + + logger.Debug("did not receive any valid proposals during the collection period", + zap.Int("clients", len(gc.clients)), + zap.Int("pending", pendingClients), + fields.Slot(slot), + ) + + return gc.waitForFirstValidProposal(ctx, logger, slot, startCollect, resultCh, pendingClients, errs) +} + // scoreProposal computes a score for a beacon proposal. // see https://github.com/attestantio/vouch/blob/master/strategies/beaconblockproposal/best/score.go as well func (gc *GoClient) scoreProposal( diff --git a/beacon/goclient/proposer_path_dispatch_test.go b/beacon/goclient/proposer_path_dispatch_test.go new file mode 100644 index 0000000000..7bc158a738 --- /dev/null +++ b/beacon/goclient/proposer_path_dispatch_test.go @@ -0,0 +1,319 @@ +package goclient + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/attestantio/go-eth2-client/api" + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ssvlabs/ssv/observability/log" +) + +// Tests for the block-fetch path dispatch (BlockFetchPathSafe / Legacy / MEVOptimized). +// See docs/MEV_CONSIDERATIONS.md for path semantics. +// +// These tests gate BN responses on Release channels rather than time.Sleep delays, +// so the assertions are identity/ordering-based rather than wall-clock-based. +// A 2s safety timeout on the result channel catches the "GetBeaconBlock never returns" +// failure mode without depending on tight CI-sensitive elapsed-time bounds. + +// TestNew_StoresBlockFetchPath verifies that the selected path and its associated +// timing field (proposalSoftDeadline / proposalSoftTimeout) get propagated from +// Options into the resulting GoClient. +func TestNew_StoresBlockFetchPath(t *testing.T) { + for _, path := range []BlockFetchPath{BlockFetchPathSafe, BlockFetchPathLegacy, BlockFetchPathMEVOptimized} { + t.Run(path.String(), func(t *testing.T) { + server, _ := createProposalBeaconServer(t, beaconProposalServerOptions{}) + defer server.Close() + + base := Options{ + BeaconNodeAddr: server.URL, + CommonTimeout: time.Second * 2, + LongTimeout: time.Second * 5, + } + if path == BlockFetchPathMEVOptimized { + base.ProposalSoftDeadline = 1100 * time.Millisecond + } + opt, err := NewOptions(base, 0, path) + require.NoError(t, err) + + client, err := New(t.Context(), log.TestLogger(t), opt) + require.NoError(t, err) + + assert.Equal(t, path, client.blockFetchPath, "GoClient.blockFetchPath should reflect opt.BlockFetchPath") + + switch path { + case BlockFetchPathSafe: + assert.Equal(t, DefaultProposalSoftDeadline, client.proposalSoftDeadline, + "safe path should default proposalSoftDeadline to %v", DefaultProposalSoftDeadline) + case BlockFetchPathMEVOptimized: + assert.Equal(t, 1100*time.Millisecond, client.proposalSoftDeadline, + "MEV-optimized path should propagate operator-set ProposalSoftDeadline") + case BlockFetchPathLegacy: + assert.NotZero(t, client.proposalSoftTimeout, + "legacy path should have non-zero proposalSoftTimeout") + } + }) + } +} + +// TestGetBeaconBlock_MultiBN_SafePath_EarlyExitOnBlinded verifies the safe path's +// early-exit-on-first-blinded behavior. With BN1 released and BN2 left blocked, +// the safe path must return on BN1's blinded response without waiting for BN2. +func TestGetBeaconBlock_MultiBN_SafePath_EarlyExitOnBlinded(t *testing.T) { + release1 := make(chan struct{}) + release2 := make(chan struct{}) // intentionally never closed + + bn1, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release1, + BlindedProposal: true, + FeeRecipient: feeRecipientAllOnes(), + }) + defer bn1.Close() + bn2, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release2, + BlindedProposal: true, + FeeRecipient: feeRecipientAllTwos(), + }) + defer bn2.Close() + + client := setupMultiBNClient(t, bn1.URL, bn2.URL, BlockFetchPathSafe, 1500*time.Millisecond) + + // Future slot so the slot-relative deadline lands after both BN responses — + // we want to observe early-exit, not the deadline firing. + slot := client.getBeaconConfig().EstimatedCurrentSlot() + 2 + + resultCh, cancel := launchGetBeaconBlock(t, client, slot) + defer cancel() // unblocks BN2's still-pending request when test returns + + close(release1) + + proposal := requireBeaconBlockResult(t, resultCh, "safe path should early-exit on first blinded") + assertProposalFeeRecipient(t, proposal, feeRecipientAllOnes(), + "safe path should return BN1's blinded (early-exit), not BN2's") +} + +// TestGetBeaconBlock_MultiBN_MEVOptimizedPath_NoEarlyExit verifies that the +// MEV-optimized path does NOT early-exit on the first blinded response. After +// releasing BN1, GetBeaconBlock must still be waiting; only after releasing BN2 +// does it return. +func TestGetBeaconBlock_MultiBN_MEVOptimizedPath_NoEarlyExit(t *testing.T) { + release1 := make(chan struct{}) + release2 := make(chan struct{}) + + bn1, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release1, + BlindedProposal: true, + FeeRecipient: feeRecipientAllOnes(), + }) + defer bn1.Close() + bn2, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release2, + BlindedProposal: true, + FeeRecipient: feeRecipientAllTwos(), + }) + defer bn2.Close() + + client := setupMultiBNClient(t, bn1.URL, bn2.URL, BlockFetchPathMEVOptimized, 1500*time.Millisecond) + + slot := client.getBeaconConfig().EstimatedCurrentSlot() + 2 + + resultCh, cancel := launchGetBeaconBlock(t, client, slot) + defer cancel() + + close(release1) + + // MEV-optimized path keeps collecting after blinded. A short non-return check + // (100ms) is enough to detect a regression to early-exit behavior; we're + // asserting "no event for a brief window" rather than the much weaker + // "wall-clock bound around 500ms". + assertNoReturnWithin(t, resultCh, 100*time.Millisecond, + "MEV-optimized path returned after BN1 alone — should not early-exit on blinded") + + close(release2) + + requireBeaconBlockResult(t, resultCh, "MEV-optimized path should return after both BNs respond") +} + +// TestGetBeaconBlock_MultiBN_MEVOptimizedPath_HighestScoringBlindedWins verifies +// that when multiple BNs return blinded proposals within the collection window, +// the MEV-optimized path selects the one with the highest scoreProposal value +// (sum of ConsensusValue and ExecutionValue) rather than the first-arriving one. +func TestGetBeaconBlock_MultiBN_MEVOptimizedPath_HighestScoringBlindedWins(t *testing.T) { + release1 := make(chan struct{}) + release2 := make(chan struct{}) + + bn1, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release1, + BlindedProposal: true, + FeeRecipient: feeRecipientAllOnes(), + ExecutionValue: big.NewInt(1_000_000), // low bid + }) + defer bn1.Close() + bn2, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release2, + BlindedProposal: true, + FeeRecipient: feeRecipientAllTwos(), + ExecutionValue: big.NewInt(5_000_000), // high bid (must win) + }) + defer bn2.Close() + + client := setupMultiBNClient(t, bn1.URL, bn2.URL, BlockFetchPathMEVOptimized, 1500*time.Millisecond) + + slot := client.getBeaconConfig().EstimatedCurrentSlot() + 2 + + resultCh, cancel := launchGetBeaconBlock(t, client, slot) + defer cancel() + + // Release BN1 first so it arrives first (lower bid). MEV-optimized must keep + // collecting until BN2 (higher bid) arrives, then prefer BN2 by score. + close(release1) + close(release2) + + proposal := requireBeaconBlockResult(t, resultCh, "MEV-optimized path should select highest-scoring proposal") + assertProposalFeeRecipient(t, proposal, feeRecipientAllTwos(), + "MEV-optimized path should select the higher-value blinded (BN2's), not the first-arriving (BN1's)") +} + +// TestGetBeaconBlock_MultiBN_SoftDeadlineFires_FallsBackToFirstValid verifies that +// when the slot-relative soft deadline has already fired before any BN responds, +// the parallel-fetch path falls through to waitForFirstValidProposal and returns +// the first valid BN response. Uses a slot in the past so the deadline is past. +func TestGetBeaconBlock_MultiBN_SoftDeadlineFires_FallsBackToFirstValid(t *testing.T) { + release1 := make(chan struct{}) + release2 := make(chan struct{}) // intentionally never closed + + bn1, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release1, + BlindedProposal: true, + FeeRecipient: feeRecipientAllOnes(), + }) + defer bn1.Close() + bn2, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ + Release: release2, + BlindedProposal: true, + FeeRecipient: feeRecipientAllTwos(), + }) + defer bn2.Close() + + client := setupMultiBNClient(t, bn1.URL, bn2.URL, BlockFetchPathSafe, 1000*time.Millisecond) + + // Slot 1 is in the past (mainnet genesis is in 2020). The slot-relative + // deadline = slotStart + 1000ms is also in the past, so softCtx is already + // done when the collection loop starts. + pastSlot := phase0.Slot(1) + + resultCh, cancel := launchGetBeaconBlock(t, client, pastSlot) + defer cancel() + + close(release1) + + proposal := requireBeaconBlockResult(t, resultCh, "waitForFirstValidProposal should return the first BN response") + assertProposalFeeRecipient(t, proposal, feeRecipientAllOnes(), + "waitForFirstValidProposal should return BN1's response (first released), not BN2's") +} + +// beaconBlockResult bundles the values returned by client.GetBeaconBlock for +// transmission over a channel from the background goroutine spawned by +// launchGetBeaconBlock. +type beaconBlockResult struct { + proposal *api.VersionedProposal + err error +} + +// launchGetBeaconBlock runs client.GetBeaconBlock in a background goroutine and +// returns a channel that receives the result when it returns, plus a cancel +// function that aborts any in-flight BN requests (used to unblock test-server +// handlers whose Release channels weren't closed). +func launchGetBeaconBlock(t *testing.T, client *GoClient, slot phase0.Slot) (<-chan beaconBlockResult, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + resultCh := make(chan beaconBlockResult, 1) + go func() { + p, _, e := client.GetBeaconBlock(ctx, slot, []byte("test"), getTestRANDAO()) + resultCh <- beaconBlockResult{proposal: p, err: e} + }() + return resultCh, cancel +} + +// requireBeaconBlockResult waits for the result channel with a 2s safety timeout, +// failing the test if no result arrives. Returns the proposal on success. +func requireBeaconBlockResult(t *testing.T, resultCh <-chan beaconBlockResult, msg string) *api.VersionedProposal { + t.Helper() + select { + case r := <-resultCh: + require.NoError(t, r.err, msg) + require.NotNil(t, r.proposal, msg) + return r.proposal + case <-time.After(2 * time.Second): + t.Fatalf("GetBeaconBlock did not return within 2s: %s", msg) + return nil + } +} + +// assertNoReturnWithin fails the test if a result arrives on the channel within +// the given window. Used to verify that the MEV-optimized path does NOT +// early-exit after one BN responds. +func assertNoReturnWithin(t *testing.T, resultCh <-chan beaconBlockResult, window time.Duration, msg string) { + t.Helper() + select { + case <-resultCh: + t.Fatalf("unexpected early return within %v: %s", window, msg) + case <-time.After(window): + // Expected: still waiting. + } +} + +// assertProposalFeeRecipient extracts the fee recipient from a VersionedProposal +// and asserts equality with the expected value. +func assertProposalFeeRecipient(t *testing.T, proposal *api.VersionedProposal, expected bellatrix.ExecutionAddress, msg string) { + t.Helper() + actual, err := proposal.FeeRecipient() + require.NoError(t, err) + assert.Equal(t, expected, actual, msg) +} + +// setupMultiBNClient builds a GoClient connected to two test BN servers via +// semicolon-separated URLs, with the given block-fetch path and deadline. Used by +// the per-path behavior tests. +func setupMultiBNClient(t *testing.T, bn1URL, bn2URL string, path BlockFetchPath, deadline time.Duration) *GoClient { + t.Helper() + + base := Options{ + BeaconNodeAddr: bn1URL + ";" + bn2URL, + CommonTimeout: time.Second * 2, + LongTimeout: time.Second * 5, + ProposalSoftDeadline: deadline, + } + opt, err := NewOptions(base, 0, path) + require.NoError(t, err) + + client, err := New(t.Context(), log.TestLogger(t), opt) + require.NoError(t, err) + return client +} + +// feeRecipientAllOnes returns a fee-recipient address filled with 0x01 bytes, +// distinguishable from feeRecipientAllTwos in test assertions. +func feeRecipientAllOnes() bellatrix.ExecutionAddress { + var addr bellatrix.ExecutionAddress + for i := range addr { + addr[i] = 1 + } + return addr +} + +// feeRecipientAllTwos returns a fee-recipient address filled with 0x02 bytes. +func feeRecipientAllTwos() bellatrix.ExecutionAddress { + var addr bellatrix.ExecutionAddress + for i := range addr { + addr[i] = 2 + } + return addr +} diff --git a/beacon/goclient/proposer_test.go b/beacon/goclient/proposer_test.go index adf02efb16..5c16a4212f 100644 --- a/beacon/goclient/proposer_test.go +++ b/beacon/goclient/proposer_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math/big" "net/http" "net/http/httptest" "strconv" @@ -53,6 +54,20 @@ type beaconProposalServerOptions struct { FeeRecipient bellatrix.ExecutionAddress // Use blinded proposal BlindedProposal bool + // Optional Eth-Execution-Payload-Value header value; left nil ⇒ header not set + // (go-eth2-client defaults ExecutionValue to 0). Used by tests that need to + // influence scoreProposal across multiple BN responses. + ExecutionValue *big.Int + // Release, if non-nil, gates the proposal-endpoint handler: the response is + // withheld until the channel receives a value or is closed (whichever first). + // Use this in tests that need deterministic response ordering instead of + // wall-clock delays via ProposalResponseDuration. The handler also unblocks + // on request context cancellation (e.g., when the parent test ends), so a + // never-released channel does not stall server shutdown. + // + // Mutually exclusive with ProposalResponseDuration: when Release is non-nil, + // the duration is ignored. + Release <-chan struct{} } // Creates a mock beacon server for proposal testing @@ -85,16 +100,23 @@ func createProposalBeaconServer(t *testing.T, options beaconProposalServerOption require.NoError(t, err) require.NotZero(t, slot) - // Add delay if specified - time.Sleep(options.ProposalResponseDuration) + // Gate the response: either wait for the Release signal (deterministic + // ordering) or sleep for the configured duration. The request context + // also unblocks the handler so an unreleased channel doesn't stall + // server shutdown. + if options.Release != nil { + select { + case <-options.Release: + case <-r.Context().Done(): + return + } + } else if options.ProposalResponseDuration > 0 { + time.Sleep(options.ProposalResponseDuration) + } // Return custom response if provided if len(options.ProposalResponse) > 0 { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Eth-Consensus-Version", "electra") - if options.BlindedProposal { - w.Header().Set("Eth-Execution-Payload-Blinded", "true") - } + writeProposalHeaders(w, options) if _, err := w.Write(options.ProposalResponse); err != nil { w.WriteHeader(http.StatusInternalServerError) } @@ -103,11 +125,7 @@ func createProposalBeaconServer(t *testing.T, options beaconProposalServerOption // Generate response dynamically (this should not cause races since each server has its own goroutine) proposalResp := createProposalResponseSafe(phase0.Slot(slot), options.FeeRecipient, options.BlindedProposal) - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Eth-Consensus-Version", "electra") - if options.BlindedProposal { - w.Header().Set("Eth-Execution-Payload-Blinded", "true") - } + writeProposalHeaders(w, options) if _, err := w.Write(proposalResp); err != nil { w.WriteHeader(http.StatusInternalServerError) } @@ -128,8 +146,34 @@ func createProposalBeaconServer(t *testing.T, options beaconProposalServerOption return server, serverGotRequests } -// Create a safe proposal response using ssv-spec utilities (called once during server setup) +// writeProposalHeaders sets the response headers that go-eth2-client parses to +// reconstruct VersionedProposal — Eth-Consensus-Version, the blinded flag, and +// optionally Eth-Execution-Payload-Value when the test wants to influence +// scoreProposal across multiple BN responses. +func writeProposalHeaders(w http.ResponseWriter, options beaconProposalServerOptions) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Eth-Consensus-Version", "electra") + if options.BlindedProposal { + w.Header().Set("Eth-Execution-Payload-Blinded", "true") + } + if options.ExecutionValue != nil { + w.Header().Set("Eth-Execution-Payload-Value", options.ExecutionValue.String()) + } +} + +// spectestingMu serializes mutations on the shared ssv-spec testing fixtures. +// TestingBlindedBeaconBlockV / TestingBeaconBlockV return wrappers that point at +// cached block structs; without this lock, concurrent test-server goroutines +// (e.g., the multi-BN tests) racially mutate the same struct on `block.Slot` and +// `block.Body.*.FeeRecipient`. The JSON marshaling happens inside the lock so +// each call captures its own intended state before the next mutation begins. +var spectestingMu sync.Mutex + +// Create a safe proposal response using ssv-spec utilities. func createProposalResponseSafe(slot phase0.Slot, feeRecipient bellatrix.ExecutionAddress, blinded bool) []byte { + spectestingMu.Lock() + defer spectestingMu.Unlock() + if blinded { // Get a blinded block from ssv-spec testing utilities versionedBlinded := spectestingutils.TestingBlindedBeaconBlockV(spec.DataVersionElectra) @@ -388,26 +432,25 @@ func TestGetProposalParallel_MultiClient(t *testing.T) { feeRecipient2 := bellatrix.ExecutionAddress{0x22} feeRecipient3 := bellatrix.ExecutionAddress{0x33} - // Pre-generate block responses to avoid race conditions - blockResponse1 := createProposalResponseSafe(testSlot, feeRecipient1, false) - blockResponse2 := createProposalResponseSafe(testSlot, feeRecipient2, false) - blockResponse3 := createProposalResponseSafe(testSlot, feeRecipient3, false) - + // Responses are generated per-request from the URL slot (rather than + // pre-generated) so the safe path's slot-relative deadline can be set against + // a future slot below — pre-baking a fixed slot would trip go-eth2-client's + // "expected slot N" response check. server1, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ ProposalResponseDuration: 500 * time.Millisecond, - ProposalResponse: blockResponse1, + FeeRecipient: feeRecipient1, }) defer server1.Close() server2, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ ProposalResponseDuration: 50 * time.Millisecond, // Fastest - ProposalResponse: blockResponse2, + FeeRecipient: feeRecipient2, }) defer server2.Close() server3, _ := createProposalBeaconServer(t, beaconProposalServerOptions{ ProposalResponseDuration: 1000 * time.Millisecond, - ProposalResponse: blockResponse3, + FeeRecipient: feeRecipient3, }) defer server3.Close() @@ -418,8 +461,13 @@ func TestGetProposalParallel_MultiClient(t *testing.T) { graffiti := []byte(testGraffiti) randao := getTestRANDAO() + // Use a future slot so the safe path's slot-relative ProposalSoftDeadline doesn't + // fire before the collection loop starts — otherwise this test would exercise + // waitForFirstValidProposal instead of the multi-BN scoring/racing logic. + slot := client.getBeaconConfig().EstimatedCurrentSlot() + 2 + startTime := time.Now() - versionedProposal, marshaledBlk, err := client.GetBeaconBlock(t.Context(), testSlot, graffiti, randao) + versionedProposal, marshaledBlk, err := client.GetBeaconBlock(t.Context(), slot, graffiti, randao) elapsed := time.Since(startTime) require.NoError(t, err) @@ -684,7 +732,7 @@ func createClientForProposerTest(t *testing.T, serverURL string) (*GoClient, err BeaconNodeAddr: serverURL, CommonTimeout: time.Second * 2, LongTimeout: time.Second * 5, - }, 0) + }, 0, BlockFetchPathSafe) if err != nil { return nil, err } diff --git a/cli/operator/node.go b/cli/operator/node.go index c3df393f1f..9075859126 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -110,8 +110,8 @@ type config struct { KeyStore KeyStore `yaml:"KeyStore"` SSVSigner SSVSignerConfig `yaml:"SSVSigner" env-prefix:"SSV_SIGNER_"` Graffiti string `yaml:"Graffiti" env:"GRAFFITI" env-description:"Custom graffiti for block proposals" env-default:"ssv.network"` - ProposerDelay time.Duration `yaml:"ProposerDelay" env:"PROPOSER_DELAY" env-description:"Duration to wait out before requesting Ethereum block to propose if this Operator is proposer-duty Leader (eg. 300ms). See https://github.com/ssvlabs/ssv/blob/main/docs/MEV_CONSIDERATIONS.md#getting-started-with-mev-configuration for detailed instructions on how to use it."` - AllowDangerousProposerDelay bool `yaml:"AllowDangerousProposerDelay" env:"ALLOW_DANGEROUS_PROPOSER_DELAY" env-description:"Allow ProposerDelay values higher than 1s (dangerous, may cause missed block proposals)"` + ProposerDelay time.Duration `yaml:"ProposerDelay" env:"PROPOSER_DELAY" env-description:"Legacy MEV configuration. The recommended approach is PBS-side timing games (mev-boost v1.11+ launched with -config, or commit-boost); leave ProposerDelay at 0 when those are configured. See https://github.com/ssvlabs/ssv/blob/main/docs/MEV_CONSIDERATIONS.md for details."` + AllowDangerousProposerDelay bool `yaml:"AllowDangerousProposerDelay" env:"ALLOW_DANGEROUS_PROPOSER_DELAY" env-description:"Allow ProposerDelay values higher than 1000ms (dangerous, may cause missed block proposals)"` OperatorPrivateKey string `yaml:"OperatorPrivateKey" env:"OPERATOR_KEY" env-description:"Operator private key for contract event decryption"` MetricsAPIPort int `yaml:"MetricsAPIPort" env:"METRICS_API_PORT" env-description:"Port for metrics API server"` EnableTraces bool `yaml:"EnableTraces" env:"ENABLE_TRACES" env-description:"Enable Open Telemetry traces"` @@ -191,13 +191,52 @@ var StartNodeCmd = &cobra.Command{ logger.Fatal("could not setup network", zap.Error(err)) } + // Determine the block-fetch path from operator-provided config before NewOptions + // applies any defaults. See docs/MEV_CONSIDERATIONS.md. + blockFetchPath, err := goclient.DetermineBlockFetchPath(cfg.ConsensusClient, cfg.ProposerDelay) + if err != nil { + logger.Fatal("invalid block-fetch path configuration", zap.Error(err)) + } + + switch blockFetchPath { + case goclient.BlockFetchPathLegacy: + // validateProposerDelayConfig is scoped to the legacy path because the + // dangerous-delay check is only meaningful when ProposerDelay > 0, and a + // non-zero ProposerDelay is precisely what selects this path. + if err := validateProposerDelayConfig(logger); err != nil { + logger.Fatal("invalid ProposerDelay configuration", zap.Error(err)) + } + logger.Warn("Using legacy MEV configuration path — there is a better way to opt into MEV, see docs/MEV_CONSIDERATIONS.md") + case goclient.BlockFetchPathMEVOptimized: + if err := goclient.ValidateProposalSoftDeadline(cfg.ConsensusClient.ProposalSoftDeadline); err != nil { + logger.Fatal("invalid ProposalSoftDeadline configuration", zap.Error(err)) + } + // Strict `>` (not `>=`) is intentional: DefaultProposalSoftDeadline equals + // SafeMaxProposalSoftDeadline, so the safe-path default sits exactly at the + // threshold and should not trip the warning. + if cfg.ConsensusClient.ProposalSoftDeadline > goclient.SafeMaxProposalSoftDeadline { + logger.Warn( + "ProposalSoftDeadline exceeds the safe-max threshold: "+ + "round-2 QBFT fallback may not fit within the slot deadline "+ + "for clusters with typical latencies, "+ + "so the slot may be missed when round 1 fails "+ + "(this is an explicit 'round 1 must succeed' configuration).", + zap.Int64("proposal_soft_deadline_ms", cfg.ConsensusClient.ProposalSoftDeadline.Milliseconds()), + zap.Int64("safe_max_ms", goclient.SafeMaxProposalSoftDeadline.Milliseconds())) + } + case goclient.BlockFetchPathSafe: + // No path-specific validation needed. + } + + logger.Info("block-fetch path selected", zap.String("path", blockFetchPath.String())) + logger.Info("connecting CL(s)", fields.Address(cfg.ConsensusClient.BeaconNodeAddr), zap.Bool("with_weighted_attestation_data", cfg.ConsensusClient.WithWeightedAttestationData), zap.Bool("with_parallel_submissions", cfg.ConsensusClient.WithParallelSubmissions), ) - cliopt, err := goclient.NewOptions(cfg.ConsensusClient, cfg.ProposerDelay) + cliopt, err := goclient.NewOptions(cfg.ConsensusClient, cfg.ProposerDelay, blockFetchPath) if err != nil { logger.Fatal("failed to create beacon client options", zap.Error(err), @@ -224,10 +263,6 @@ var StartNodeCmd = &cobra.Command{ usingSSVSigner, usingKeystore, usingPrivKey = assertSigningConfig(logger) } - if err := validateProposerDelayConfig(logger); err != nil { - logger.Fatal("invalid ProposerDelay configuration", zap.Error(err)) - } - var operatorPrivKey keys.OperatorPrivateKey var operatorPrivKeyPEM string var ssvSignerClient *ssvsigner.Client @@ -843,14 +878,14 @@ func validateProposerDelayConfig(logger *zap.Logger) error { if cfg.ProposerDelay > maxSafeProposerDelay { if !cfg.AllowDangerousProposerDelay { - return fmt.Errorf("ProposerDelay value %v exceeds maximum safe delay of %v. "+ + return fmt.Errorf("ProposerDelay value %dms exceeds maximum safe delay of %dms. "+ "This may cause missed block proposals. "+ "If you understand the risks and want to proceed, set AllowDangerousProposerDelay to true or use the ALLOW_DANGEROUS_PROPOSER_DELAY environment variable", - cfg.ProposerDelay, maxSafeProposerDelay) + cfg.ProposerDelay.Milliseconds(), maxSafeProposerDelay.Milliseconds()) } logger.Warn("Using dangerous ProposerDelay value that may cause missed block proposals", - zap.Duration("proposer_delay", cfg.ProposerDelay), - zap.Duration("max_safe_proposer_delay", maxSafeProposerDelay)) + zap.Int64("proposer_delay_ms", cfg.ProposerDelay.Milliseconds()), + zap.Int64("max_safe_proposer_delay_ms", maxSafeProposerDelay.Milliseconds())) } return nil diff --git a/cli/operator/node_test.go b/cli/operator/node_test.go index 67ce2eadac..e9ec2c5c3b 100644 --- a/cli/operator/node_test.go +++ b/cli/operator/node_test.go @@ -459,10 +459,10 @@ func Test_validateProposerDelayConfig(t *testing.T) { // Check log fields fields := logs[0].ContextMap() - require.Contains(t, fields, "proposer_delay") - require.Contains(t, fields, "max_safe_proposer_delay") - require.Equal(t, delay, fields["proposer_delay"]) - require.Equal(t, 1000*time.Millisecond, fields["max_safe_proposer_delay"]) + require.Contains(t, fields, "proposer_delay_ms") + require.Contains(t, fields, "max_safe_proposer_delay_ms") + require.Equal(t, delay.Milliseconds(), fields["proposer_delay_ms"]) + require.Equal(t, int64(1000), fields["max_safe_proposer_delay_ms"]) }) } }) diff --git a/config/config.example.yaml b/config/config.example.yaml index adc5f36b3c..ca9588c44b 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -19,6 +19,28 @@ eth2: # HTTP URL of the Beacon node to connect to. BeaconNodeAddr: http://example.url:5052 + # Block-fetch tuning. The SSV node selects between the safe (default), MEV-optimized, + # and legacy paths at startup based on the settings below; see + # docs/MEV_CONSIDERATIONS.md for the full model. + # + # ProposalSoftDeadline opts the SSV node into the MEV-optimized path: SSV waits for all + # multi-BN responses until this slot-relative deadline (ms into slot) and returns the + # highest-value bid received, without early-exiting on the first MEV block. Match this + # to your PBS late_in_slot_time_ms + ~50ms BN→SSV transport. Valid range: [1000ms, + # 3600ms]; values above 1450ms emit a startup warning (round-2 QBFT fallback may not + # fit within the slot for typical clusters). Cannot be combined with ProposerDelay or + # ProposalSoftTimeout. + # Leave unset to use the default safe path (early-exit on first MEV block, deadline 1450ms). + # Note: setting ProposalSoftDeadline = 1450ms is *not* a no-op — it opts into the + # MEV-optimized path at the same numeric deadline the safe path uses by default + # (the difference is behavioral: no early-exit on first blinded, best-bid wins). + # ProposalSoftDeadline: 1100ms + + # ProposalSoftTimeout (legacy): collection-period timeout for multi-BN proposal scoring + # in the legacy block-fetch path. Setting this (or ProposerDelay below) opts into the + # legacy path. New operators should leave this unset and use ProposalSoftDeadline above. + # ProposalSoftTimeout: 1800ms + ValidatorOptions: eth1: @@ -37,13 +59,17 @@ p2p: OperatorPrivateKey: # MEV Configuration (Optional) -# Duration to wait before requesting block proposal if this operator is proposer-duty Leader. -# This allows extracting higher MEV by waiting for better bids. Default is 0 (no delay). -# Recommended starting value: 300ms. See docs/MEV_CONSIDERATIONS.md for details. +# Recommended approach: configure timing games on the PBS layer (mev-boost v1.11+ launched +# with -config, or commit-boost). With PBS-side timing games configured, leave ProposerDelay +# at its default value of 0. See docs/MEV_CONSIDERATIONS.md for details. +# +# ProposerDelay is the legacy SSV-side lever — a delay before requesting the block-proposal, +# extracting higher MEV by waiting for better bids. Use this only when your PBS does not +# support timing games. Default is 0 (no delay). # ProposerDelay: 300ms -# Safety flag to allow ProposerDelay values higher than 1s. -# WARNING: Values above 1s significantly increase the risk of missing block proposals! +# Safety flag to allow ProposerDelay values higher than 1000ms. +# WARNING: Values above 1000ms significantly increase the risk of missing block proposals! # Only set to true if you understand the risks and have carefully read the MEV documentation. # AllowDangerousProposerDelay: false diff --git a/docs/MEV_CONSIDERATIONS.md b/docs/MEV_CONSIDERATIONS.md index 0db7877d38..902ff6fc0d 100644 --- a/docs/MEV_CONSIDERATIONS.md +++ b/docs/MEV_CONSIDERATIONS.md @@ -1,112 +1,280 @@ -## Getting started with `MEV` configuration +# MEV considerations + +## TL;DR + +To get the most out of MEV opportunities, configure **timing games on the PBS layer** — either mev-boost v1.11+ launched with `-config ` (and optionally `-watch-config` for hot reload), or commit-boost. With PBS-side timing games configured, SSV's `ProposerDelay` should stay at its default value of `0` — operators relying on `ProposerDelay` can't also use the multi-BN bid scoring described in [Multi-BN setup](#multi-bn-setup). + +If your PBS does not support timing games (mev-boost < v1.11, mev-boost without `-config `, or any other PBS lacking the feature), SSV's `ProposerDelay` is still available — see [Appendix A](#appendix-a--legacy-proposerdelay-approach). PBS-side timing games are preferred because the PBS polls each relay multiple times within a precise slot-relative auction window — yielding higher-value bids than a single `getHeader` call after an SSV-side `ProposerDelay` sleep. + +If you run **multiple Beacon nodes**, set `ProposalSoftDeadline` = your PBS `late_in_slot_time_ms` + ~50ms BN→SSV transport. See [Multi-BN setup](#multi-bn-setup) for details. Single-BN operators can skip that section entirely. + +`ProposalSoftDeadline` and the legacy `ProposerDelay` / `ProposalSoftTimeout` select mutually-exclusive SSV-side block-fetch paths — combining them is rejected at startup. + +## Definitions and typical values + +The variables below name the stages of the SSV proposer-duty timeline. The values are typical/average for a healthy mainnet SSV cluster — real-world variance is significant, and operators should baseline their own latencies (see [Tuning guidance](#tuning-guidance--measurement-methodology)) before treating them as hard numbers. + +| Variable | Typical | Description | +|---|---|---| +| `RANDAO` | ~50ms | Pre-consensus phase: SSV operators build the RANDAO signature used in the block-fetch request. | +| `(auction window)` | varies | PBS-side relay polling. Configurable; see [PBS-side timing games](#pbs-side-timing-games-recommended). | +| `MEVBoostRelayTimeout` | ~200ms | *Legacy path only:* mev-boost's single getHeader call when SSV asks for a block. Replaced by `(auction window)` in PBS-timing-games setups. | +| `QBFT` | ~2350ms worst case | QBFT consensus over the blinded block. Worst-case decomposes into `QBFTRound1Time` (~2000ms round-1 timer, fires if round 1 fails) + `QBFTRoundChange` (~100ms ROUND-CHANGE handshake) + `QBFTRound2Time` (~250ms successful round 2). | +| `PostConsensusSigning` | ~50ms | Operators reconstruct the validator BLS signature from partial signatures. | +| `BlockSubmission` | ~100ms | Leader submits the signed blinded block to the BN; relay reveals the payload; block propagates. | + +The Ethereum slot-propagation deadline is **4000ms** after slot start. + +## SSV proposer-duty flow background + +The proposer-duty flow runs the stages above in sequence. For an SSV cluster to function reliably, the following must hold even in the worst case where round 1 fails and round 2 runs as fallback: -To get the most out of MEV opportunities Operator can configure ProposerDelay configuration setting using a configuration -file (or `PROPOSER_DELAY` environment variable): ``` -ProposerDelay: 300ms +RANDAO + (auction window) + QBFT + PostConsensusSigning + BlockSubmission < 4000ms ``` -As per our own estimates the max reasonable value of `ProposerDelay` for Ethereum mainnet is around ~1.2s, -although we recommend starting with something like 300ms gradually increasing it up - the higher -`ProposerDelay` value is the higher the chance of missing Ethereum block proposal will be. +You must budget for the worst case: in the common case round 1 succeeds quickly and round 2 never runs, but the budget reserved by the equation cannot be reclaimed. If the equation doesn't hold, the validator risks missing its proposal slot whenever round 1 fails. + +Where `(auction window)` sits in the slot is what determines MEV capture — bid value grows as the slot ages, so later auction windows yield higher-value bids on average, subject to staying within this deadline. + +## PBS-side timing games (recommended) + +The PBS layer implements "timing games" — proactively polling relays at intervals defined in its own config, decoupling *when* the auction happens from *when* SSV asks for the block. SSV asks once and receives whatever bid the PBS has selected by its slot-relative cutoff. + +Preferred over `ProposerDelay` because: +- The PBS polls each relay multiple times within the auction window (`target_first_request_ms` + `frequency_get_header_ms`), capturing higher-value bids than a single `getHeader` per relay. `ProposerDelay` + single `getHeader` gets one bid per relay at one point in time; PBS-side timing games sample several and keep the best. +- The auction cutoff is slot-relative (`late_in_slot_time_ms`), so QBFT round 1 starts at a predictable point in the slot. With `ProposerDelay`, QBFT starts at `ProposerDelay + variable relay-response time`, which makes the post-auction budget harder to size. +- The PBS handles auction-timing risk internally. If all polls time out or no relay responds, the PBS falls back to a local block (vanilla), not a missed slot. +- Configuration is concentrated in the PBS rather than split across SSV-side and PBS-side knobs. -### Important Safety Limitation +### Configuration knobs + +Both mev-boost and commit-boost expose the same five knobs: + +- `timeout_get_header_ms` — per-relay-request timeout for a single `getHeader` call. +- `late_in_slot_time_ms` — slot-relative hard cutoff. The PBS returns to the caller no later than this point in the slot. +- `enable_timing_games` (per-relay) — opt in to the multi-poll behavior for this relay. Defaults to `false`; must be set per-relay. +- `target_first_request_ms` — when the first poll for this relay fires, measured from slot start. +- `frequency_get_header_ms` — interval between subsequent polls. + +The effective per-request deadline is: +``` +max_timeout_ms = min(timeout_get_header_ms, late_in_slot_time_ms - ms_into_slot) +slot-relative cutoff = ms_into_slot + max_timeout_ms +``` +When the PBS receives the request early in the slot, `timeout_get_header_ms` tends to bind; when asked later, `late_in_slot_time_ms - ms_into_slot` binds. -**The SSV node will refuse to start if ProposerDelay is set higher than 1s without explicit confirmation.** +### PBS-specific notes -If you attempt to use a ProposerDelay value higher than 1s, the node will exit with an error message. -If you understand the risks and want to proceed anyway, you must also set the `AllowDangerousProposerDelay` flag: +- **commit-boost** validates `timeout_get_header_ms < late_in_slot_time_ms` at config load — refuses to start otherwise. Set `timeout_get_header_ms` just below `late_in_slot_time_ms`. +- **mev-boost (v1.11+)** has the same knobs and budget math but does not enforce that inequality — values may be equal. Requires `-config ` to enable the YAML-based timing-games config; `-watch-config` enables hot reload. +- Default `late_in_slot_time_ms`: mev-boost `2000ms`, commit-boost `3000ms`. Both are aggressive. +- mev-boost selects the most-recently-received bid per relay, then compares across relays for the highest value. +Upstream references: +- mev-boost: [github.com/flashbots/mev-boost/blob/main/docs/timing-games.md](https://github.com/flashbots/mev-boost/blob/main/docs/timing-games.md) +- commit-boost: [commit-boost.github.io/commit-boost-client](https://commit-boost.github.io/commit-boost-client/) + +## Configuration examples + +Two scenarios shown for both PBSes. The numbers are starting points for a healthy mainnet cluster — operators should validate against their own measured latencies (see [Tuning guidance](#tuning-guidance--measurement-methodology)). + +### Example A — bid-sample equivalent of legacy `ProposerDelay ≈ 1000ms` (recommended starting point) + +Lands the last relay poll at ~1000ms, matching when legacy `ProposerDelay = 1000ms` would have queried the relays. Useful as a migration baseline — same bid quality, but the header arrives at SSV at ~1100ms (1050ms PBS cutoff + ~50ms BN→SSV) instead of legacy's ~1300–2000ms (depending on relay response speed), leaving more slot budget for QBFT and submission. + +The polling pattern (`target_first_request_ms = 700`, `frequency_get_header_ms = 150`) fires polls at 700ms, 850ms, and 1000ms. + +**commit-boost** (TOML): +```toml +[pbs] +late_in_slot_time_ms = 1050 +timeout_get_header_ms = 1030 # must be < late_in_slot_time_ms in commit-boost +timeout_get_payload_ms = 4000 + +[[relays]] +url = "https://@relay-1.example" +enable_timing_games = true +target_first_request_ms = 700 # polls at 700ms, 850ms, 1000ms +frequency_get_header_ms = 150 + +[[relays]] +url = "https://@relay-2.example" +enable_timing_games = true +target_first_request_ms = 700 +frequency_get_header_ms = 150 +``` + +**mev-boost** (YAML): ```yaml -ProposerDelay: 2000ms -AllowDangerousProposerDelay: true +timeout_get_header_ms: 1050 +late_in_slot_time_ms: 1050 # mev-boost permits equality +relays: + - url: https://@relay-1.example + enable_timing_games: true + target_first_request_ms: 700 + frequency_get_header_ms: 150 + - url: https://@relay-2.example + enable_timing_games: true + target_first_request_ms: 700 + frequency_get_header_ms: 150 ``` -Or using environment variables: -```bash -PROPOSER_DELAY=2000ms ALLOW_DANGEROUS_PROPOSER_DELAY=true ./bin/ssvnode start-node +**SSV-side** (multi-BN setups only — see [Multi-BN setup](#multi-bn-setup); single-BN operators skip this): +```yaml +eth2: + ProposalSoftDeadline: 1100ms # = PBS late_in_slot_time_ms (1050ms) + ~50ms BN→SSV transport ``` -**Warning:** Using ProposerDelay values higher than 1s significantly increases the risk of missing block proposals, -which can result in penalties and lost rewards. +### Example B — aggressive: PBS-side cutoff at 1800ms (round 1 must succeed) -As per the notes in other sections of this document `ProposerDelay` depends on a number of things, to find -the best value Operator might want to start with lower values like 300ms gradually increasing it up. +Pushes the PBS-side cutoff to `1800ms` — past the ~1450ms threshold where round-2 QBFT fallback may no longer fit within the slot for typical clusters. This accepts "round 1 must succeed" in exchange for capturing more intra-slot bid growth (clusters with measurably faster QBFT + submission may still leave room for round 2). Last relay poll at ~1600ms; header at SSV by ~1850ms. -## MEV considerations & SSV proposer-duty flow background +The polling pattern (`target_first_request_ms = 1000`, `frequency_get_header_ms = 200`) fires polls at 1000ms, 1200ms, 1400ms, 1600ms — four chances with ~200ms RTT margin. -To understand how MEV fits with the SSV cluster, here is some background on the SSV proposer-duty flow: -- SSV node participates in the pre-consensus phase to build RANDAO signature that will be used when - requesting block from Beacon node (let's say it takes `RANDAOTime`) -- SSV node (all nodes in the cluster really to handle round-changes, but current round Leader - specifically) requests blinded block header from Beacon node which in turn "proxies" this request - to MEV-boost that runs with some pre-configured timeout (call it `MEVBoostRelayTimeout`) -- MEV-boost sends multiple requests to Relays it knows about and waits until that - `MEVBoostRelayTimeout` time to choose the best block (based on the corresponding bid) -- SSV node receives the response with the chosen block header and goes through QBFT consensus phase - to sign it as Validator (let's say it takes `QBFTTime` at most - we can estimate it - statistically with some probability/confidence) -- QBFT consensus phase might require several rounds to complete in case there is a fault with the - chosen round leader, each round can take up to `RoundTimeout` (currently set to 2s on SSV-protocol - level, which also means there will be 2 rounds at most because Ethereum block must be proposed - within 4s from slot start) meaning if round 1 doesn't complete in under `RoundTimeout` another - leader will be chosen to try and complete QBFT in round 2, etc. -- once QBFT completes successfully, Operator needs to submit the signed block to Beacon node to - propagate it throughout Ethereum network (call it `BlockSubmissionTime`) -- there is some time spent on executing various code to "glue" this whole thing together - that's small but still matters (call it `MiscellaneousTime`) +Trade-off vs Example A: bid-sample time shifts ~600ms later, capturing more intra-slot bid growth, but the remaining slot budget for QBFT and submission shrinks from ~2900ms to ~2150ms — below the ~2500ms typically needed for the worst-case 2-round QBFT scenario. Example B accepts that round 1 must succeed; if round 1 fails, the slot may be missed (whether it's actually missed depends on your cluster's QBFT + submission latencies). Use only after baselining your stack's round-1 success rate. -and so this means for the best SSV cluster operations we want the following condition to always hold true: +**commit-boost** (TOML): +```toml +[pbs] +late_in_slot_time_ms = 1800 +timeout_get_header_ms = 1780 # must be < late_in_slot_time_ms in commit-boost +timeout_get_payload_ms = 4000 + +[[relays]] +url = "https://@relay-1.example" +enable_timing_games = true +target_first_request_ms = 1000 # polls at 1000ms, 1200ms, 1400ms, 1600ms +frequency_get_header_ms = 200 + +[[relays]] +url = "https://@relay-2.example" +enable_timing_games = true +target_first_request_ms = 1000 +frequency_get_header_ms = 200 ``` -RANDAOTime + MEVBoostRelayTimeout + QBFTTime + BlockSubmissionTime + MiscellaneousTime < 4s + +**mev-boost** (YAML): +```yaml +timeout_get_header_ms: 1800 +late_in_slot_time_ms: 1800 # mev-boost permits equality +relays: + - url: https://@relay-1.example + enable_timing_games: true + target_first_request_ms: 1000 + frequency_get_header_ms: 200 + - url: https://@relay-2.example + enable_timing_games: true + target_first_request_ms: 1000 + frequency_get_header_ms: 200 ``` -if this equation doesn't hold, Validator will miss his opportunity to propose the block (slot will be -missed if the corresponding Ethereum block isn't published within 4s after slot start time) -and so the most straightforward approach to extract highest MEV is (and it probably works out-of-the box -with SSV nodes already, but with caveats mentioned below): +**SSV-side** (multi-BN setups only — 1850ms triggers the safe-max startup warning since it exceeds the ~1450ms threshold; see [Multi-BN setup](#multi-bn-setup); single-BN operators skip this): +```yaml +eth2: + ProposalSoftDeadline: 1850ms # = PBS late_in_slot_time_ms (1800ms) + ~50ms BN→SSV transport +``` + +## Tuning guidance & measurement methodology + +The example configs are starting points. Production tuning requires measuring your own stack — relay RTTs, QBFT consensus times, and submission latencies vary enough between operators that a single recommended value isn't optimal for everyone. + +### Where the auction window should land + +Bid value grows through the slot, so the auction cutoff should be as late as possible, subject to: + +- **Round-2 fallback should fit:** `QBFT + PostConsensusSigning + BlockSubmission < 4000ms − late_in_slot_time_ms − ~50ms` (the ~50ms covers BN→SSV transport between the PBS cutoff and SSV receiving the header). Using the typical values from [Definitions](#definitions-and-typical-values), the post-cutoff budget needed is ~2500ms, giving a strict bound of `late_in_slot_time_ms ≲ ~1450ms`. **Recommended:** stay at `late_in_slot_time_ms ≲ ~1400ms` to keep a 50ms buffer for latency variance — this also matches SSV's startup-warning threshold (`SafeMaxProposalSoftDeadline = 1450ms` SSV-side, which equals `~1400ms` PBS-side plus the `~50ms` BN→SSV transport). +- **Cutoffs above ~1400ms** consume the variance buffer; SSV emits a startup warning. **Cutoffs above ~1450ms** are past the strict bound and accept that round 1 must succeed — if round 1 fails, the slot may be missed (depending on your cluster's QBFT + submission latencies). Example B (1800ms) sits in this regime. +- **Round-1-only variance buffer:** even in the round-1-must-succeed regime, cutoffs much beyond ~3000ms tighten the slot enough that occasional latency spikes risk missing the deadline even when round 1 succeeds. + +### What to measure + +Useful signals to baseline before tuning, by data source: + +**On the SSV side** — metrics on Grafana (if export is enabled) and structured logs: +- **RANDAO completion time** — pre-consensus duration. +- **QBFT round-1 completion distribution** — consensus duration. +- `"got beacon block proposal"` log with `took` duration. +- `"received proposal"` debug log with `score`, `latency`, `blinded`, `pending` fields — emitted per BN response in multi-BN setups. +- `"successfully finished duty processing"` log with pre-consensus, consensus, and post-consensus splits. + +**On the PBS side** — PBS logs: +- BN → PBS RTT — typically same machine, well under 10ms. +- Per-relay RTT distribution (p50/p95/p99) — logged per `getHeader` call. + +**End-to-end** — submission round-trip from the signed block leaving SSV through the relay payload-reveal step (visible from PBS and relay logs). + +## Multi-BN setup + +> Single-BN operators can skip this section — SSV bypasses parallel fetch entirely and calls the single BN directly regardless of which knobs are set below. + +With multiple Beacon nodes, SSV races them in parallel for the block proposal. The recommended action is to set `ProposalSoftDeadline`: + +```yaml +eth2: + ProposalSoftDeadline: +``` + +This makes SSV wait for all BN responses up to that slot-relative deadline and return the highest-scored bid. Valid range `[1000ms, 3600ms]`; values above ~1450ms emit a startup warning — for typical clusters, the worst-case 2-round QBFT scenario may no longer fit within the slot, so round 1 effectively has to succeed. + +### Default behavior (if you don't set `ProposalSoftDeadline`) + +SSV returns as soon as one BN delivers a blinded (MEV) block — treating the first blinded response as the chosen MEV bid. If no blinded response arrives by the default slot-relative deadline (1450ms — the largest safest deadline for typical clusters; see [Tuning guidance](#tuning-guidance--measurement-methodology)), SSV returns the best non-blinded response collected so far, waiting for the first valid response if nothing usable arrived. -### approach 1 +This default is faster but doesn't compare bid *values* across BNs — the first BN to return blinded wins regardless of bid quality. Fine for multi-BN setups run primarily for redundancy. -To set `MEVBoostRelayTimeout` as high as possible, and have plenty of Relays -(MEV-boost is configured with) so that Relays themselves decide "how much time they want to wait -since Slot start before replying with a block/bid" - some Relays would be fast to reply but not -as profitable as those that withhold for longer +### Legacy approach -^ the problem of this approach is that SSV node Operator might not be able to find/choose Relays -that fit his MEV desires (maybe he can only use those that respond right away without any additional -delay for the sake of higher MEV) +Setting `ProposerDelay` or `ProposalSoftTimeout` selects legacy block-fetch behavior (preserved bit-for-bit) — see [Appendix A](#appendix-a--legacy-proposerdelay-approach). SSV logs a startup warning suggesting migration. -thus an alternative approach would be: +### Interaction -### approach 2 +The approaches are mutually exclusive. Selection at startup: -To introduce an additional configurable delay `ProposerDelay` SSV Operator can set so -that it will work nicely even with Relays that "reply as fast as possible", the equation from above -becomes: ``` -RANDAOTime + ProposerDelay + MEVBoostRelayTimeout + QBFTTime + BlockSubmissionTime + MiscellaneousTime < 4s +if ProposerDelay > 0 || ProposalSoftTimeout is set: + -> legacy approach (see Appendix A) +elif ProposalSoftDeadline is set: + -> new approach (waits for all BN responses, picks highest-scored) +else: + -> new approach default (returns first blinded response) ``` -plugging in some realistic numbers into that ^ formula we get a rough estimate of ~2.2s for `ProposerDelay`: -```go -const randaoTime = 100 * time.Millisecond -const mevBoostRelayTimeout = 200 * time.Millisecond -const qbftTime = 350 * time.Millisecond -const miscellaneousTime = 150 * time.Millisecond -const blockSubmissionTime = 1000 * time.Millisecond -const proposerDelay = 4*time.Second - randaoTime - mevBoostRelayTimeout - qbftTime - blockSubmissionTime - miscellaneousTime + +Setting `ProposalSoftDeadline` together with either legacy knob (`ProposerDelay` or `ProposalSoftTimeout`) is rejected at startup with a clear error — pick one approach. + +## Appendix A — Legacy `ProposerDelay` approach + +`ProposerDelay` remains supported. It is the right tool when: +- Your PBS does not support timing games (mev-boost < v1.11, or any PBS without the feature). +- You are running mev-boost without `-config ` and don't want to introduce a YAML config file. +- Operator constraints prevent PBS-side configuration changes. + +To configure, set in the SSV config file (or via the `PROPOSER_DELAY` environment variable): +```yaml +ProposerDelay: 300ms +``` + +With `ProposerDelay` active, the slot-budget equation becomes: ``` -but on top of that, another consideration Operator needs to take into account is QBFT round timeout, specifically -round 1 timeout. For proposer duty round 1 times out at ~2s after slot start time (so that proposer duty can execute 2 -QBFT rounds, if necessary, and still complete before that desirable 4s after slot start deadline). To avoid round 1 timing out -we'd want the following equation to hold: -```go -RANDAOTime + ProposerDelay + MEVBoostRelayTimeout + QBFTTime + MiscellaneousTime < 2s +RANDAO + ProposerDelay + MEVBoostRelayTimeout + QBFT + PostConsensusSigning + BlockSubmission < 4000ms +``` + +Using the typical values from [Definitions](#definitions-and-typical-values), `ProposerDelay ≤ 4000ms − (50 + 200 + 2350 + 50 + 100) = 1250ms` is the theoretical maximum. In practice, latency variance can easily add several hundred ms — we consider **~700ms** the maximum reasonable value for `ProposerDelay` on Ethereum mainnet, leaving ~550ms of headroom for variance. + +We recommend starting with a small value such as 300ms and increasing gradually while monitoring miss rate. + +**The SSV node refuses to start if `ProposerDelay` is set higher than 1000ms without explicit confirmation.** + +If you attempt to use a `ProposerDelay` value higher than 1000ms, the node exits with an error message. If you understand the risks and want to proceed anyway, set the `AllowDangerousProposerDelay` flag: + +```yaml +ProposerDelay: 2000ms +AllowDangerousProposerDelay: true ``` -and with the values listed above this gives us `ProposerDelay` value of ~1.2s. -Therefore, we consider ~1.2s to be the maximum reasonable value for `ProposerDelay`, going beyond that value might -result in missed block proposal. +Or via environment variables: +```bash +PROPOSER_DELAY=2000ms ALLOW_DANGEROUS_PROPOSER_DELAY=true ./bin/ssvnode start-node +``` -**To enforce proposer safety limits, the SSV node will automatically prevent startup if ProposerDelay exceeds 1s -unless the Operator explicitly acknowledges the risk by setting `AllowDangerousProposerDelay: true`.** +**Warning:** `ProposerDelay` values higher than 1000ms significantly increase the risk of missed block proposals, which can result in penalties and lost rewards. diff --git a/protocol/v2/ssv/runner/proposer.go b/protocol/v2/ssv/runner/proposer.go index 06316ea8a6..2c16565423 100644 --- a/protocol/v2/ssv/runner/proposer.go +++ b/protocol/v2/ssv/runner/proposer.go @@ -45,9 +45,7 @@ type ProposerRunner struct { // ValCheck is used to validate the qbft-value(s) proposed by other Operators. ValCheck ssv.ValueChecker - // proposerDelay allows Operator to configure a delay to wait out before requesting Ethereum - // block to propose if this Operator is proposer-duty Leader. This allows Operator to extract - // higher MEV. + // proposerDelay; see ProposerRunnerOptions.ProposerDelay. proposerDelay time.Duration // cachedFullBlock holds the initially fetched full (non-blinded) block @@ -69,9 +67,10 @@ type ProposerRunnerOptions struct { ValCheck ssv.ValueChecker HighestDecidedSlot phase0.Slot Graffiti []byte - // ProposerDelay allows Operator to configure a delay to wait out before requesting Ethereum - // block to propose if this Operator is proposer-duty Leader. This allows Operator to extract - // higher MEV. + // ProposerDelay is the legacy SSV-side MEV-extraction lever — a delay before requesting + // the Ethereum block to capture later (higher-value) bids. The recommended approach is + // PBS-side timing games (mev-boost v1.11+ with -config, or commit-boost), in which case + // this stays at 0. See docs/MEV_CONSIDERATIONS.md. ProposerDelay time.Duration }