From 09756e7256a2772e44566f459793c577bdd82760 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 20 Aug 2026 20:18:14 +0200 Subject: [PATCH 1/3] fix(indexer): read dependentState once per epoch stats load loadEpochStats read epochStats.dependentState eight times across a loadState call that can run for seconds, while runCachePruning writes nil into that same field from another goroutine. getEpochStatsByEpoch releases cacheMutex before the write, so nothing serialises the two: a prune landing mid-load turns the next read into a nil dereference, the loader goroutine panics, and the epoch's stats are never completed. Read the field once into a local and work from that, at every check-then-read site in the file. The window is normally too small to hit, since a state comes off a local node in milliseconds. It reproduces reliably against a replayed chain, where the state arrives over the network and the epoch being loaded can fall out of the in-memory window before the load returns, leaving epochs with a row but zeroed validator counts and no participation. --- indexer/beacon/epochcache.go | 41 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index add1d0bb..30a1fade 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -144,8 +144,9 @@ func (cache *epochCache) getPendingEpochStats() []*EpochStats { pendingStats := make([]*EpochStats, 0) for _, stats := range cache.statsMap { - if stats.dependentState != nil && stats.dependentState.loadingStatus == 0 { - if loadingRoots[stats.dependentState.slotRoot] { + dependentState := stats.dependentState + if dependentState != nil && dependentState.loadingStatus == 0 { + if loadingRoots[dependentState.slotRoot] { continue // another epochState with same root is already loading } pendingStats = append(pendingStats, stats) @@ -238,9 +239,9 @@ func (cache *epochCache) removeEpochStats(epochStats *EpochStats) { delete(cache.statsMap, statsKey) - if epochStats.dependentState != nil { + if dependentState := epochStats.dependentState; dependentState != nil { stateKey := getEpochStatsKey(epochStats.epoch, epochStats.dependentRoot) - epochStats.dependentState.dispose() + dependentState.dispose() delete(cache.stateMap, stateKey) } } @@ -393,6 +394,14 @@ func (cache *epochCache) runLoaderLoop() { func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { defer utils.HandleSubroutinePanic("indexer.beacon.epochCache.loadEpochStats", nil) + // pruning clears dependentState on epochs that leave the in-memory window, and it + // can do so while this load is in flight. Work from one read of the field so a + // prune halfway through cannot turn a later use into a nil dereference. + dependentState := epochStats.dependentState + if dependentState == nil { + return false + } + clients := []*Client{} preferArchive := epochStats.epoch < cache.indexer.lastFinalizedEpoch for _, client := range cache.indexer.GetReadyClientsByBlockRoot(epochStats.dependentRoot, preferArchive) { @@ -427,7 +436,7 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { if len(clients) == 0 { cache.indexer.logger.Debugf("no clients available to load epoch %v stats (dep: %v)", epochStats.epoch, epochStats.dependentRoot.String()) - epochStats.dependentState.retryCount++ + dependentState.retryCount++ return false } @@ -456,25 +465,25 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { return bytes.Compare(hashA[:], hashB[:]) < 0 }) - client := clients[int(epochStats.dependentState.retryCount)%len(clients)] + client := clients[int(dependentState.retryCount)%len(clients)] log := cache.indexer.logger.WithField("client", client.client.GetName()) - if epochStats.dependentState.retryCount > 0 { - log = log.WithField("retry", epochStats.dependentState.retryCount) + if dependentState.retryCount > 0 { + log = log.WithField("retry", dependentState.retryCount) } log.Infof("loading epoch %v stats (dep: %v, req: %v)", epochStats.epoch, epochStats.dependentRoot.String(), len(epochStats.requestedBy)) t1 := time.Now() - state, err := epochStats.dependentState.loadState(client.getContext(), client, cache) - if err != nil && epochStats.dependentState.loadingStatus == 0 { + state, err := dependentState.loadState(client.getContext(), client, cache) + if err != nil && dependentState.loadingStatus == 0 { client.logger.Warnf("failed loading epoch %v stats (dep: %v): %v", epochStats.epoch, epochStats.dependentRoot.String(), err) } loadDuration := time.Since(t1) - if epochStats.dependentState.loadingStatus != 2 { + if dependentState.loadingStatus != 2 { // epoch state could not be loaded - epochStats.dependentState.retryCount++ + dependentState.retryCount++ return false } @@ -501,14 +510,16 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { if stats == epochStats { continue } - if stats.dependentState == nil || stats.dependentState.loadingStatus != 0 { + // same read-once rule as above: pruning may clear this concurrently + pendingState := stats.dependentState + if pendingState == nil || pendingState.loadingStatus != 0 { continue } - if stats.dependentState.slotRoot != epochStats.dependentState.slotRoot { + if pendingState.slotRoot != dependentState.slotRoot { continue } pendingOthers = append(pendingOthers, pendingEntry{ - epochState: stats.dependentState, + epochState: pendingState, stats: stats, }) } From 51ed2db17faf033d03724d071f11baf0a2a3f64c Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 20 Aug 2026 20:18:28 +0200 Subject: [PATCH 2/3] feat(replay): step a past slot range through dora as if it were live dora-replay serves a fake beacon/execution node pair backed by a real upstream and drives a virtual clock the explorer follows, so a past slot range unfolds slot by slot with pause, step, seek and play while the normal UI stays live and inspectable. No indexer, service or handler code is aware of it, which is the point: a run has to exercise the real code paths. The explorer's whole share is the clock. ChainState gains an optional replay clock that polls the control server and interpolates between polls; CurrentSlot follows it, and a small ticker replaces ethwallclock, which reads time.Now() internally and cannot be paused. Genesis stays real, so SlotToTime keeps returning true historical times and only "now" moves. Page timestamps use the same clock. The control panel is side-loaded from the replay process, so the layout carries a script tag and nothing else. Notes on the fake nodes: - States are read from tracoor first and blocks are not. Beacon nodes prune all but the most recent states (a node tagged archive: true 404s a state two days old), while tracoor keeps every one it sampled. Blocks are the other way round: the node answers in one round trip where a tracoor read costs a lookup plus a download, and routing blocks through it tripled the explorer's per-block indexing time. Tracoor artifacts arrive zstd-compressed and without Eth-Consensus-Version, both of which have to be handled or the explorer cannot decode them. - states/head/finality_checkpoints is answered locally. Reading it upstream forces the node to load the head state, which for a replayed head is expensive at best and pruned away at worst. - Chain events are never dropped: the hub waits for its subscriber. That is what makes "advance as fast as upstream allows" self-throttle to what the explorer can actually index. Control status events are the opposite, so a stalled browser tab can never hold the replay up. - Halfway through each slot the driver waits for any beacon state still on its way to the explorer, freezing the clock while it waits. The wait costs real time and no virtual time, so the explorer does not end up several slots behind the slot the replay claims to be at. Docs in replay/README.md, worked config in replay/example-config.yaml. --- clients/consensus/chainstate.go | 124 ++++- clients/consensus/replayclock.go | 166 ++++++ cmd/dora-explorer/main.go | 8 + cmd/dora-replay/main.go | 106 ++++ go.mod | 2 +- handlers/index.go | 2 +- handlers/pageData.go | 16 +- replay/README.md | 257 ++++++++++ replay/assets/replay-ui.js | 407 +++++++++++++++ replay/cache.go | 108 ++++ replay/cache_test.go | 67 +++ replay/chain.go | 190 +++++++ replay/clock.go | 133 +++++ replay/clock_test.go | 72 +++ replay/clproxy.go | 299 +++++++++++ replay/clproxy_test.go | 123 +++++ replay/config.go | 95 ++++ replay/console.go | 219 ++++++++ replay/console_test.go | 136 +++++ replay/control.go | 147 ++++++ replay/control_test.go | 257 ++++++++++ replay/elproxy.go | 703 ++++++++++++++++++++++++++ replay/elproxy_test.go | 277 ++++++++++ replay/events.go | 181 +++++++ replay/events_test.go | 78 +++ replay/example-config.yaml | 96 ++++ replay/replay.go | 843 +++++++++++++++++++++++++++++++ replay/replay_test.go | 285 +++++++++++ replay/stateloads.go | 71 +++ replay/stateloads_test.go | 217 ++++++++ replay/tracoor.go | 169 +++++++ replay/upstream.go | 499 ++++++++++++++++++ static/js/page-index.js | 19 +- templates/_layout/layout.html | 4 + types/config.go | 10 + types/models.go | 6 + 36 files changed, 6380 insertions(+), 12 deletions(-) create mode 100644 clients/consensus/replayclock.go create mode 100644 cmd/dora-replay/main.go create mode 100644 replay/README.md create mode 100644 replay/assets/replay-ui.js create mode 100644 replay/cache.go create mode 100644 replay/cache_test.go create mode 100644 replay/chain.go create mode 100644 replay/clock.go create mode 100644 replay/clock_test.go create mode 100644 replay/clproxy.go create mode 100644 replay/clproxy_test.go create mode 100644 replay/config.go create mode 100644 replay/console.go create mode 100644 replay/console_test.go create mode 100644 replay/control.go create mode 100644 replay/control_test.go create mode 100644 replay/elproxy.go create mode 100644 replay/elproxy_test.go create mode 100644 replay/events.go create mode 100644 replay/events_test.go create mode 100644 replay/example-config.yaml create mode 100644 replay/replay.go create mode 100644 replay/replay_test.go create mode 100644 replay/stateloads.go create mode 100644 replay/stateloads_test.go create mode 100644 replay/tracoor.go create mode 100644 replay/upstream.go diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index 67681286..5032be55 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -2,16 +2,19 @@ package consensus import ( "bytes" + "context" "crypto/sha256" "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/ethpandaops/dora/utils" "github.com/ethpandaops/ethwallclock" v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/ethpandaops/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" ) type ChainState struct { @@ -22,8 +25,13 @@ type ChainState struct { genesisMutex sync.Mutex genesis *v1.Genesis - wallclockMutex sync.Mutex - wallclock *ethwallclock.EthereumBeaconChain + wallclockMutex sync.Mutex + wallclockStarted bool + wallclock *ethwallclock.EthereumBeaconChain + + // replayClock replaces the wall clock with the virtual clock of a dora-replay + // control server, so a past slot range can be stepped through as if it were live. + replayClock atomic.Pointer[replayClock] finalityMutex sync.RWMutex finality *v1.Finality @@ -198,7 +206,7 @@ func (cs *ChainState) initWallclock() { cs.wallclockMutex.Lock() defer cs.wallclockMutex.Unlock() - if cs.wallclock != nil { + if cs.wallclockStarted { return } @@ -206,6 +214,15 @@ func (cs *ChainState) initWallclock() { return } + cs.wallclockStarted = true + + // ethwallclock reads time.Now() internally and cannot be paused, so a replay run + // drives the slot/epoch dispatchers off the virtual clock instead. + if clock := cs.replayClock.Load(); clock != nil { + go cs.runReplayWallclock(clock) + return + } + cs.wallclock = ethwallclock.NewEthereumBeaconChain(cs.genesis.GenesisTime, time.Duration(cs.specs.SlotDurationMs)*time.Millisecond, cs.specs.SlotsPerEpoch) cs.wallclock.OnEpochChanged(func(current ethwallclock.Epoch) { cs.wallclockEpochDispatcher.Fire(¤t) @@ -215,6 +232,105 @@ func (cs *ChainState) initWallclock() { }) } +// EnableReplayClock points the chain state at a dora-replay control server, so every +// "now" in the explorer (current slot, wallclock ticks, page timestamps) follows the +// replayed slot range instead of the real wall clock. It blocks until the control +// server has answered once. +func (cs *ChainState) EnableReplayClock(ctx context.Context, logger logrus.FieldLogger, controlURL string, pollInterval time.Duration) error { + clock, err := newReplayClock(ctx, logger, controlURL, pollInterval) + if err != nil { + return err + } + + cs.replayClock.Store(clock) + + logger.WithField("now", clock.now().UTC().Format(time.RFC3339)).Info("replay clock enabled") + + return nil +} + +// IsReplaying reports whether the chain state is driven by a replay clock. +func (cs *ChainState) IsReplaying() bool { + return cs.replayClock.Load() != nil +} + +// Now returns the current time as the chain sees it: the real wall clock normally, +// or the virtual replay time while a replay is running. +func (cs *ChainState) Now() time.Time { + if clock := cs.replayClock.Load(); clock != nil { + return clock.now() + } + + return time.Now() +} + +// maxReplayWallclockCatchup bounds how many slot ticks are replayed in one go after +// the virtual clock jumped, so a large seek does not flood the dispatchers. +const maxReplayWallclockCatchup = 16 + +// runReplayWallclock fires the slot and epoch dispatchers from the virtual clock, +// standing in for ethwallclock while a replay is running. +func (cs *ChainState) runReplayWallclock(clock *replayClock) { + defer utils.HandleSubroutinePanic("clients.consensus.ChainState.runReplayWallclock", func() { + cs.runReplayWallclock(clock) + }) + + genesis := cs.genesis.GenesisTime + slotDuration := time.Duration(cs.specs.SlotDurationMs) * time.Millisecond + slotsPerEpoch := cs.specs.SlotsPerEpoch + + interval := slotDuration / 20 + if interval < 50*time.Millisecond { + interval = 50 * time.Millisecond + } else if interval > 250*time.Millisecond { + interval = 250 * time.Millisecond + } + + lastSlot := uint64(cs.CurrentSlot()) + lastEpoch := lastSlot / slotsPerEpoch + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-clock.ctx.Done(): + return + case <-ticker.C: + } + + slot := uint64(cs.CurrentSlot()) + if slot <= lastSlot { + // the replay was rewound; resync without firing anything + lastSlot = slot + lastEpoch = slot / slotsPerEpoch + + continue + } + + firstSlot := lastSlot + 1 + if slot-lastSlot > maxReplayWallclockCatchup { + firstSlot = slot + } + + for tickSlot := firstSlot; tickSlot <= slot; tickSlot++ { + slotStart := genesis.Add(time.Duration(tickSlot) * slotDuration) + + if epoch := tickSlot / slotsPerEpoch; epoch != lastEpoch { + epochStart := genesis.Add(time.Duration(epoch*slotsPerEpoch) * slotDuration) + wallclockEpoch := ethwallclock.NewEpoch(epoch, epochStart, epochStart.Add(slotDuration*time.Duration(slotsPerEpoch))) + cs.wallclockEpochDispatcher.Fire(&wallclockEpoch) + lastEpoch = epoch + } + + wallclockSlot := ethwallclock.NewSlot(tickSlot, slotStart, slotStart.Add(slotDuration)) + cs.wallclockSlotDispatcher.Fire(&wallclockSlot) + } + + lastSlot = slot + } +} + func (cs *ChainState) setFinalizedCheckpoint(finality *v1.Finality) { cs.finalityMutex.Lock() if cs.finality != nil && finality.Justified.Epoch <= cs.finality.Justified.Epoch && finality.Finalized.Epoch <= cs.finality.Finalized.Epoch { @@ -300,7 +416,7 @@ func (cs *ChainState) GetFinalizedSlot() phase0.Slot { } func (cs *ChainState) CurrentSlot() phase0.Slot { - return cs.TimeToSlot(time.Now()) + return cs.TimeToSlot(cs.Now()) } func (cs *ChainState) CurrentEpoch() phase0.Epoch { diff --git a/clients/consensus/replayclock.go b/clients/consensus/replayclock.go new file mode 100644 index 00000000..04c98bbb --- /dev/null +++ b/clients/consensus/replayclock.go @@ -0,0 +1,166 @@ +package consensus + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +const ( + // defaultReplayPollInterval is how often the virtual clock is refreshed from the + // control server. Between polls the clock is interpolated locally from the last + // reported rate, so a coarse interval does not make the clock jumpy. + defaultReplayPollInterval = 100 * time.Millisecond + + // replayConnectTimeout bounds how long EnableReplayClock waits for the control + // server to become reachable before giving up. + replayConnectTimeout = 2 * time.Minute +) + +// replayClockState is the payload served by a dora-replay control server at +// GET /replay/clock. Rate is the number of virtual milliseconds that pass per real +// millisecond; it is 0 while the replay is paused. +type replayClockState struct { + TimeMs int64 `json:"time_ms"` + Rate float64 `json:"rate"` +} + +// replayClock mirrors the virtual clock of a dora-replay control server, so the whole +// explorer sees a simulated "now" while a past slot range is stepped through. +type replayClock struct { + ctx context.Context + logger logrus.FieldLogger + url string + client *http.Client + + mutex sync.RWMutex + anchorReal time.Time + anchorVirt time.Time + rate float64 +} + +// newReplayClock connects to the control server and blocks until the first clock +// state has been read, so callers never observe an uninitialized virtual time. +func newReplayClock(ctx context.Context, logger logrus.FieldLogger, controlURL string, pollInterval time.Duration) (*replayClock, error) { + if controlURL == "" { + return nil, fmt.Errorf("no replay control url configured") + } + + if pollInterval <= 0 { + pollInterval = defaultReplayPollInterval + } + + clock := &replayClock{ + ctx: ctx, + logger: logger, + url: strings.TrimSuffix(controlURL, "/") + "/replay/clock", + client: &http.Client{Timeout: 10 * time.Second}, + } + + if err := clock.awaitFirstPoll(); err != nil { + return nil, err + } + + go clock.runPollLoop(pollInterval) + + return clock, nil +} + +// awaitFirstPoll retries the control server until it answers or the connect timeout +// expires. The explorer cannot compute any slot before this succeeds. +func (c *replayClock) awaitFirstPoll() error { + deadline := time.Now().Add(replayConnectTimeout) + lastLog := time.Time{} + + for { + err := c.poll() + if err == nil { + return nil + } + + if time.Now().After(deadline) { + return fmt.Errorf("could not reach replay control server at %v: %w", c.url, err) + } + + if time.Since(lastLog) > 5*time.Second { + c.logger.WithError(err).Warnf("waiting for replay control server at %v", c.url) + lastLog = time.Now() + } + + select { + case <-c.ctx.Done(): + return c.ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + +func (c *replayClock) runPollLoop(pollInterval time.Duration) { + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + if err := c.poll(); err != nil { + c.logger.WithError(err).Debugf("failed polling replay clock") + } + } + } +} + +func (c *replayClock) poll() error { + ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url, http.NoBody) + if err != nil { + return err + } + + rsp, err := c.client.Do(req) + if err != nil { + return err + } + defer func() { _ = rsp.Body.Close() }() + + if rsp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status %v", rsp.StatusCode) + } + + state := replayClockState{} + if err := json.NewDecoder(rsp.Body).Decode(&state); err != nil { + return fmt.Errorf("error parsing replay clock response: %w", err) + } + + c.mutex.Lock() + c.anchorReal = time.Now() + c.anchorVirt = time.UnixMilli(state.TimeMs) + c.rate = state.Rate + c.mutex.Unlock() + + return nil +} + +// now returns the current virtual time, interpolated from the last polled state so +// the clock keeps moving smoothly between polls while the replay is playing. +func (c *replayClock) now() time.Time { + c.mutex.RLock() + defer c.mutex.RUnlock() + + if c.rate == 0 { + return c.anchorVirt + } + + elapsed := time.Since(c.anchorReal) + + return c.anchorVirt.Add(time.Duration(float64(elapsed) * c.rate)) +} diff --git a/cmd/dora-explorer/main.go b/cmd/dora-explorer/main.go index c36dabc3..b9fad57e 100644 --- a/cmd/dora-explorer/main.go +++ b/cmd/dora-explorer/main.go @@ -63,6 +63,14 @@ func main() { services.InitChainService(ctx, logger) + if cfg.Replay.Enabled { + chainState := services.GlobalBeaconService.GetChainState() + err = chainState.EnableReplayClock(ctx, logger.WithField("service", "replay-clock"), cfg.Replay.ControlUrl, cfg.Replay.PollInterval) + if err != nil { + logger.Fatalf("error connecting to replay control server: %v", err) + } + } + var webserver *http.Server if cfg.Frontend.Enabled || cfg.Api.Enabled { websrv, err := startWebserver(logger) diff --git a/cmd/dora-replay/main.go b/cmd/dora-replay/main.go new file mode 100644 index 00000000..60cd0240 --- /dev/null +++ b/cmd/dora-replay/main.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/ethpandaops/dora/replay" +) + +func main() { + if err := newRootCmd().Execute(); err != nil { + os.Exit(1) + } +} + +func newRootCmd() *cobra.Command { + cfg := replay.DefaultConfig() + + var ( + startEpoch uint64 + logLevel string + noBids bool + ) + + cmd := &cobra.Command{ + Use: "dora-replay", + Short: "Replay a past slot range through Dora as if it were happening live", + Long: "dora-replay serves a fake beacon/execution node pair backed by a real upstream and\n" + + "drives a virtual clock, so an explorer pointed at it steps through a past slot range\n" + + "with pause, step, seek and play. Point dora at the two listeners and set\n" + + "`replay.enabled: true` with `replay.controlUrl` pointing at the control listener.", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + logger := logrus.New() + logger.SetOutput(os.Stderr) + + level, err := logrus.ParseLevel(logLevel) + if err != nil { + return fmt.Errorf("invalid log level %q: %w", logLevel, err) + } + + logger.SetLevel(level) + + if startEpoch > 0 && cfg.StartSlot == 0 { + slotsPerEpoch, err := replay.SlotsPerEpoch(cmd.Context(), cfg.UpstreamURL) + if err != nil { + return err + } + + cfg.StartSlot = startEpoch * slotsPerEpoch + } + + cfg.EmitBids = !noBids + + return run(cmd.Context(), logger, cfg) + }, + } + + flags := cmd.Flags() + flags.StringVar(&cfg.UpstreamURL, "upstream", "", "beacon node HTTP API to read the chain from (required, must serve historical states)") + flags.StringVar(&cfg.ExecutionURL, "el-upstream", "", "execution node JSON-RPC to read from; omit to run without an execution proxy") + flags.StringVar(&cfg.TracoorURL, "tracoor", "", "tracoor instance used as a fallback for artifacts the beacon node has pruned") + flags.StringVar(&cfg.TracoorNetwork, "tracoor-network", "", "network name to query tracoor with") + flags.StringVar(&cfg.CLListen, "cl-listen", cfg.CLListen, "listen address of the fake beacon node") + flags.StringVar(&cfg.ELListen, "el-listen", cfg.ELListen, "listen address of the fake execution node") + flags.StringVar(&cfg.ControlListen, "control-listen", cfg.ControlListen, "listen address of the control endpoint dora polls the clock from") + flags.Uint64Var(&cfg.StartSlot, "start-slot", 0, "first slot to serve") + flags.Uint64Var(&startEpoch, "start-epoch", 0, "first epoch to serve (alternative to --start-slot)") + flags.Float64Var(&cfg.Speed, "speed", 0, "start playing at this multiple of real time; 0 starts paused") + flags.StringVar(&cfg.CacheDir, "cache-dir", "", "record fetched artifacts here so the range replays offline afterwards") + flags.DurationVar(&cfg.StepSettle, "step-settle", cfg.StepSettle, "real time to pause at each phase of a stepped slot") + flags.DurationVar(&cfg.StateHoldTimeout, "state-hold-timeout", cfg.StateHoldTimeout, "how long to freeze the clock waiting for the explorer to load a beacon state") + flags.BoolVar(&noBids, "no-bids", false, "do not replay execution payload bids (saves one block fetch per slot)") + flags.StringVar(&logLevel, "log-level", "info", "log level (trace, debug, info, warn, error)") + + return cmd +} + +func run(ctx context.Context, logger logrus.FieldLogger, cfg replay.Config) error { + ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer cancel() + + instance, err := replay.New(ctx, logger, cfg) + if err != nil { + return err + } + + if err := instance.Start(ctx); err != nil { + return err + } + + in, out := replay.Stdio() + instance.RunConsole(ctx, in, out) + + if err := instance.Stop(); err != nil { + logger.WithError(err).Warn("error shutting down replay") + } + + return nil +} diff --git a/go.mod b/go.mod index a3a68e6a..df995ebf 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/jackc/pgx/v4 v4.18.3 github.com/jmoiron/sqlx v1.4.0 github.com/kelseyhightower/envconfig v1.4.0 + github.com/klauspost/compress v1.19.1 github.com/lib/pq v1.12.3 github.com/libp2p/go-libp2p v0.43.0 github.com/mashingan/smapping v0.1.19 @@ -85,7 +86,6 @@ require ( github.com/ipfs/go-log/v2 v2.8.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.19.1 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/koron/go-ssdp v0.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/handlers/index.go b/handlers/index.go index 885260b0..92166617 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -66,7 +66,7 @@ func IndexData(w http.ResponseWriter, r *http.Request) { return } // stamp live server time outside of cached page data so clients can correct local clock drift - w.Header().Set("X-Server-Time", strconv.FormatInt(time.Now().UnixMilli(), 10)) + w.Header().Set("X-Server-Time", strconv.FormatInt(services.GlobalBeaconService.GetChainState().Now().UnixMilli(), 10)) w.Header().Set("Content-Type", "application/json") err := json.NewEncoder(w).Encode(pageData) if err != nil { diff --git a/handlers/pageData.go b/handlers/pageData.go index 33b27d34..dad15349 100644 --- a/handlers/pageData.go +++ b/handlers/pageData.go @@ -38,6 +38,8 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st siteDomain = r.Host } + chainState := services.GlobalBeaconService.GetChainState() + data := &types.PageData{ Meta: &types.Meta{ Title: fullTitle, @@ -50,7 +52,7 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st Data: &types.Empty{}, Version: utils.GetExplorerVersion(), BuildTime: fmt.Sprintf("%v", buildTime.Unix()), - ServerTime: time.Now().UnixMilli(), + ServerTime: chainState.Now().UnixMilli(), Year: time.Now().UTC().Year(), ExplorerTitle: utils.Config.Frontend.SiteName, ExplorerSubtitle: utils.Config.Frontend.SiteSubtitle, @@ -61,9 +63,9 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st ApiEnabled: utils.Config.Api.Enabled && !utils.Config.Api.RequireAuth, ExecutionIndexerEnabled: utils.Config.ExecutionIndexer.Enabled, EnsSearchEnabled: ensSearchEnabled(), + ReplayControlUrl: replayControlURL(), } - chainState := services.GlobalBeaconService.GetChainState() if specs := chainState.GetSpecs(); specs != nil { data.IsReady = true data.ChainSlotsPerEpoch = specs.SlotsPerEpoch @@ -94,6 +96,16 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st return data } +// replayControlURL returns the dora-replay control server, or "" when the explorer is +// not running a replay. +func replayControlURL() string { + if !utils.Config.Replay.Enabled { + return "" + } + + return strings.TrimSuffix(utils.Config.Replay.ControlUrl, "/") +} + func createMenuItems(active string) []types.MainMenuItem { chainState := services.GlobalBeaconService.GetChainState() specs := chainState.GetSpecs() diff --git a/replay/README.md b/replay/README.md new file mode 100644 index 00000000..8a5b45c9 --- /dev/null +++ b/replay/README.md @@ -0,0 +1,257 @@ +# dora-replay + +Step a past slot range through Dora as if it were happening live. + +`dora-replay` serves a fake beacon node and a fake execution node backed by a real +upstream, and drives a virtual clock that the explorer follows. A past slot range then +unfolds slot by slot with pause, step, seek and play, while the normal Dora UI stays +live and inspectable — the same indexer, service and handler code paths that run against +a real chain. + +``` + ┌───────────────┐ ┌──────────────────────┐ + dora-explorer ──►│ dora-replay │──────► │ beacon node (archive)│ + (replay: on) │ CL :15052 │ │ execution node │ + ▲ │ EL :15545 │──────► │ tracoor (states) │ + └───clock───│ ctl :15000 │ └──────────────────────┘ + └───────────────┘ + replay> step 32 +``` + +## Running one + +```sh +go build -o bin/dora-replay ./cmd/dora-replay + +./bin/dora-replay \ + --upstream https://user:pass@bn-archive-1.example \ + --el-upstream https://user:pass@rpc-1.example \ + --tracoor https://tracoor.example \ + --tracoor-network my-devnet \ + --start-slot 48000 \ + --cache-dir ./temp/replay-cache +``` + +Then start Dora with a config whose `beaconapi`/`executionapi` endpoints point at the +two listeners and whose `replay` block points at the control listener: + +```yaml +replay: + enabled: true + controlUrl: "http://127.0.0.1:15000" + +beaconapi: + endpoints: + - url: "http://127.0.0.1:15052" + name: "replay-cl" + +executionapi: + endpoints: + - url: "http://127.0.0.1:15545" + name: "replay-el" +``` + +`replay/example-config.yaml` is a complete working example. + +## Controlling it from the explorer + +With `replay.controlUrl` set, the explorer side-loads a control panel from the replay +process and hangs it off the header: a collapsed pill showing the replayed epoch and +what the replay is doing, expanding into live state and controls. + +``` + ● REPLAY epoch 1442 · playing 6x ▼ + ┌──────────────────────────────────────────────┐ + │ Slot 46154 → 48960 │ + │ Epoch 1442 of 1624 upstream │ + │ Time 2026-08-19 15:50:48Z │ + │ Head 46154 0x28a442f2… │ + │ Checkpoints justified 1440 · finalized 1439 │ + │ EL block 45376 │ + │ ▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░│░░░░░░░░░░░░░░░░░░░░░░░ │ + │ from 44800 chain head 51968 │ + │ [ Stop ] [+1] [+1 epoch] [ 6x ▾ ] │ + │ [ 48960 ] [ Forward to ] │ + └──────────────────────────────────────────────┘ +``` + +The explorer's whole share of this is one template block that sets +`window.doraReplayApi` and loads `/replay/ui.js`. The UI itself — markup, +styling, API calls — is embedded in the replay binary and talks to its control API +directly, so it can be changed without rebuilding or restarting the explorer. + +The panel also retunes how often the explorer's polling pages refresh, by setting +`window.doraIndexRefreshInterval` from the replay's current pace. At 16x a slot goes by +in well under a second, and the stock 15-second refresh would leave the index page +several epochs behind what the replay is actually showing; while paused it falls back to +the stock interval. + +## Control API + +Served on `--control-listen`, with CORS open so the browser can reach it from the +explorer's origin. + +| Endpoint | Purpose | +|---|---| +| `GET /replay/clock` | `{time_ms, rate}` — the virtual clock the explorer follows | +| `GET /replay/status` | full state snapshot (see below) | +| `GET /replay/events` | SSE stream of `status` events, pushed on every change | +| `POST /replay/command` | drive the replay; returns the new status | +| `GET /replay/ui.js` | the side-loaded control panel | + +Commands mirror the console: + +```jsonc +{"action": "play", "speed": 6} // speed 0 = as fast as upstream allows +{"action": "speed", "speed": 4} // change the rate without starting or stopping +{"action": "step", "slots": 32} +{"action": "forward", "slot": 48960, "speed": 6} +{"action": "stop"} +{"action": "start"} // resume, keeping the speed and any target +``` + +The status carries the replayed position (`virtual_slot`, `virtual_epoch`, +`virtual_time`, `rate`), the chain state at that point (`head_slot`, `head_root`, +`justified_epoch`, `finalized_epoch`, `execution_block`), the drive state (`running`, +`speed`, `start_slot`, `target_slot`) and the chain context needed to render it +(`upstream_slot`/`upstream_epoch` — the head of the *real* chain, re-read every 30s — +plus `genesis_time`, `slot_duration_ms`, `slots_per_epoch`). + +Status events are lossy on purpose: a snapshot that a stalled browser tab did not read +is dropped rather than queued, so the UI can never hold the replay up. Chain events on +the fake beacon node are the opposite — never dropped, see below. + +## The console + +``` +replay> status + slot 48160 epoch 1505 head 48160 [0x513df450…] + justified 1503 finalized 1502 el block 47370 + playing 6x time 2026-08-20T04:32:08Z streams 2 + upstream https://eth:xxxxx@bn-grandine-besu-1.example (+tracoor) + +replay> step # advance one slot, then pause +replay> step 32 # advance an epoch +replay> forward 48960 # run to a slot as fast as upstream allows +replay> forward 48960 6x # run to a slot at 6x real time +replay> play 4x # run on without a target +replay> stop # pause; the virtual clock freezes where it is +replay> start # resume, keeping the speed and any target +replay> quit +``` + +Pick the speed from what the explorer has to keep up with. Stepping runs as fast as the +upstream answers, which is far quicker than the explorer can index; a `play`/`forward` +speed gives it a fixed budget per slot instead. On a 12-second-slot devnet, 6x leaves +two real seconds per slot, which is comfortable for block indexing and epoch stats. + +## Waiting for state loads + +Once per epoch the explorer pulls a full beacon state — tens of megabytes to fetch, +decompress and decode, during which it cannot process anything else. Nothing is lost +when that happens (every link from the replay to the indexer applies backpressure rather +than dropping), but the virtual clock would otherwise keep running, so by the time the +state arrived the explorer would be several slots behind the slot the replay claims to +be at. + +So halfway through every slot the replay checks whether a state is still on its way to +the explorer, and if so **freezes the clock** until it arrives. The wait costs real time +and no virtual time: the explorer's own clock mirrors the frozen rate, so it stays +exactly where the replay left it instead of drifting. The console and the control panel +both report this as `holding for N state load(s)`. + +A read that never finishes cannot wedge a run — the gate gives up after +`--state-hold-timeout` (default 2m) and logs a warning. + +## Upstreams + +**Beacon node** (`--upstream`, required) answers everything: genesis, specs, headers, +blocks, blob sidecars, payload envelopes, finality. Point it at **one** node that keeps +historical data, not at a load balancer — a fan-out over mixed-retention backends fails +intermittently mid-replay. Nodes differ a lot here: many prune all but the most recent +states even when they are labelled "archive". + +**Tracoor** (`--tracoor`, optional but recommended) keeps every beacon state and block it +sampled, addressed by root. **States** are read from it first: it has the depth a replay +needs, where beacon nodes prune all but the most recent, and it spares the devnet from +serving a ~17 MB state per epoch. **Blocks are not** — the node still has every block of +the replayed range and answers in one round trip, where a tracoor read costs a lookup +plus a download; tracoor only backs blocks up when the node 404s one. It stores SSZ +only, so a JSON-only client falls back to the node. A tracoor-only run is rejected — +tracoor cannot resolve "the block at slot N". + +**Execution node** (`--el-upstream`, optional) backs the JSON-RPC proxy. Without it the +execution proxy is not started and the explorer runs consensus-only. + +**Cache** (`--cache-dir`, optional) records every immutable artifact fetched, so a range +is downloaded once and replays offline and reproducibly afterwards. + +## What the fake nodes do + +The consensus proxy passes everything through except where the virtual head matters: + +* `/eth/v1/node/syncing` is synthesized: head at the replayed slot, never syncing +* `head`, `finalized` and `justified` identifiers resolve to the roots the chain had at + the virtual head, not the ones the upstream has today +* any slot beyond the virtual head answers 404, the same as a node that has not seen it +* `/eth/v1/beacon/states/head/finality_checkpoints` is answered from the finality the + replay tracks — reading it upstream would force a historical state load +* `/eth/v1/events` is generated locally: `execution_payload_bid` → `block` → `head` → + `execution_payload_available`, plus `finalized_checkpoint` when it moves + +The execution proxy tracks a virtual head derived from block timestamps (a payload's +timestamp is its slot's time, so the execution head follows the consensus head exactly): + +* `latest`/`safe`/`finalized` are pinned to the virtual head, and a numeric block beyond + it answers as if the node did not have it yet +* `eth_getLogs` is clamped to the virtual head +* `eth_newBlockFilter`/`eth_getFilterChanges`/`eth_uninstallFilter` are served locally, + handing out the blocks that appeared since the last poll +* everything else is forwarded, with batches forwarded as batches + +## Dora's side + +Everything the explorer knows about "now" runs through `ChainState`. In replay mode it +polls `GET /replay/clock` and interpolates between polls, `CurrentSlot()` follows that +virtual time, and a small ticker replaces `ethwallclock` (which reads `time.Now()` +internally and cannot be paused) to fire the slot and epoch dispatchers. Genesis stays +the real one, so `SlotToTime` keeps returning true historical timestamps — only "now" +moves. Page timestamps use the same clock, so "x ago" is computed against the simulated +present. + +The only other thing Dora contributes is one block in the page layout, which sets +`window.doraReplayApi` and side-loads `/replay/ui.js`. Everything the +control panel is lives in the replay binary. + +That is the whole change on Dora's side. No indexer, service or handler code is aware of +the replay, which is the point: the run has to exercise the real code paths. + +## Starting mid-chain + +A replay starts in the middle of the chain, so use a fresh database, `blockdb`, +`statecache` and pubkey-cache path per run. Dora takes its finalized epoch from the +chain, so it will not try to finalize from epoch 0 — but the synchronizer starts from +whatever `indexer.syncstate` says. Two options: + +* `indexer.disableSynchronizer: true` — index only from the start slot forward. Fast to + get going; everything before the start slot is simply absent. +* leave the synchronizer on — it backfills epochs 0..start in the background while the + replay steps forward, ending with a complete database. Much slower, but the result can + be copied and reused as the starting point for later runs. + +## Fidelity limits + +* **Reorgs are not replayed.** The proxy serves the canonical chain as it exists today, + so blocks that were briefly head during the original run never appear and fork handling + is not exercised. +* **Intra-slot timing is approximated.** Events are emitted at fixed points in the slot + (block at ⅓, payload at ⅔) rather than when they actually arrived, so anything + timing-sensitive within a slot will not reproduce. +* **Real timers keep running.** Dora's retry and backoff timers run on real time, so a + long pause still logs client-health retries. Client readiness is not wallclock-gated, + so pausing is otherwise safe. +* **Execution head is inferred** from block timestamps rather than observed, so a slot + whose payload was never revealed shows up as "no new execution block" — which matches + reality, but by derivation. +* **Historical execution state** (`eth_getBalance` at an old block) needs an archive + execution node; a full node only keeps a shallow window. diff --git a/replay/assets/replay-ui.js b/replay/assets/replay-ui.js new file mode 100644 index 00000000..9f6ff4c4 --- /dev/null +++ b/replay/assets/replay-ui.js @@ -0,0 +1,407 @@ +// Replay control UI. +// +// This file is served by the dora-replay process and side-loaded into the explorer, so +// the explorer itself carries nothing but a script tag. Everything here talks to the +// replay's own control API on the origin this script came from. +(function () { + "use strict"; + + // the explorer hands us the control server's address; deriving it from this + // script's own URL would depend on document.currentScript, which is not something + // to rely on across browsers and script loading modes + var base = (window.doraReplayApi || "").replace(/\/+$/, ""); + if (!base) { + return; + } + + var STORAGE_KEY = "dora-replay-open"; + + // how often pages that poll for updates should refresh, relative to how fast the + // replay is producing slots + var REFRESH_MIN_MS = 1000; + var REFRESH_MAX_MS = 15000; + var REFRESH_STEPPING_MS = 2000; + var SPEEDS = [ + { value: 0, label: "max" }, + { value: 0.5, label: "0.5x" }, + { value: 1, label: "1x" }, + { value: 2, label: "2x" }, + { value: 4, label: "4x" }, + { value: 8, label: "8x" }, + { value: 16, label: "16x" }, + { value: 32, label: "32x" }, + ]; + + var CSS = [ + ".replay-callout{position:fixed;top:70px;right:16px;z-index:1010;max-width:calc(100vw - 32px);font-size:.8125rem}", + ".replay-toggle{display:flex;align-items:center;gap:.5rem;margin-left:auto;padding:.25rem .75rem;border:1px solid var(--bs-border-color,#dee2e6);border-top:none;border-radius:0 0 .5rem .5rem;background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);box-shadow:0 .25rem .75rem rgba(0,0,0,.15);cursor:pointer;white-space:nowrap}", + ".replay-toggle:hover{background:var(--bs-tertiary-bg,#f8f9fa)}", + ".replay-tag{font-weight:600;letter-spacing:.04em;font-size:.6875rem;text-transform:uppercase;color:var(--bs-warning-text-emphasis,#997404)}", + ".replay-dot{width:.5rem;height:.5rem;border-radius:50%;background:var(--bs-secondary,#6c757d);flex:none}", + ".replay-dot.is-playing{background:var(--bs-success,#198754);animation:replay-pulse 1.6s ease-in-out infinite}", + ".replay-dot.is-stepping{background:var(--bs-info,#0dcaf0)}", + ".replay-dot.is-waiting{background:var(--bs-warning,#ffc107);animation:replay-pulse 1s ease-in-out infinite}", + ".replay-dot.is-offline{background:var(--bs-danger,#dc3545)}", + "@keyframes replay-pulse{0%,100%{opacity:1}50%{opacity:.35}}", + ".replay-chevron{transition:transform .15s ease;font-size:.625rem}", + ".replay-callout.is-open .replay-chevron{transform:rotate(180deg)}", + ".replay-panel{display:none;width:23rem;max-width:calc(100vw - 32px);margin-top:.375rem;padding:.75rem;border:1px solid var(--bs-border-color,#dee2e6);border-radius:.5rem;background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);box-shadow:0 .5rem 1.5rem rgba(0,0,0,.2)}", + ".replay-callout.is-open .replay-panel{display:block}", + ".replay-grid{display:grid;grid-template-columns:auto 1fr;gap:.125rem .75rem;margin:0 0 .625rem}", + ".replay-grid dt{color:var(--bs-secondary-color,#6c757d);font-weight:400}", + ".replay-grid dd{margin:0;font-variant-numeric:tabular-nums;overflow:hidden;text-overflow:ellipsis}", + ".replay-track{height:.375rem;border-radius:.25rem;background:var(--bs-tertiary-bg,#e9ecef);overflow:hidden;position:relative}", + ".replay-track-fill{height:100%;background:var(--bs-primary,#0d6efd);width:0;transition:width .2s linear}", + ".replay-track-target{position:absolute;top:0;bottom:0;width:2px;background:var(--bs-warning,#ffc107);display:none}", + ".replay-range{display:flex;justify-content:space-between;color:var(--bs-secondary-color,#6c757d);font-size:.6875rem;margin:.25rem 0 .625rem}", + ".replay-row{display:flex;flex-wrap:wrap;gap:.375rem;align-items:center;margin-bottom:.5rem}", + ".replay-row .replay-btn,.replay-row select,.replay-row input{font-size:.8125rem;padding:.1875rem .5rem;border-radius:.25rem;border:1px solid var(--bs-border-color,#dee2e6);background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529)}", + ".replay-row .replay-btn{cursor:pointer}", + ".replay-row .replay-btn:hover:not(:disabled){background:var(--bs-tertiary-bg,#f8f9fa)}", + ".replay-row .replay-btn:disabled{opacity:.5;cursor:default}", + ".replay-btn-main{background:var(--bs-primary,#0d6efd)!important;border-color:var(--bs-primary,#0d6efd)!important;color:#fff!important;min-width:4.5rem}", + ".replay-seek-slot{flex:1;min-width:5rem;font-variant-numeric:tabular-nums}", + ".replay-source{color:var(--bs-secondary-color,#6c757d);font-size:.6875rem;word-break:break-all}", + ".replay-error{display:none;margin-top:.5rem;padding:.25rem .5rem;border-radius:.25rem;background:var(--bs-danger-bg-subtle,#f8d7da);color:var(--bs-danger-text-emphasis,#842029);font-size:.6875rem}", + ".replay-error.is-shown{display:block}", + "@media (max-width:575.98px){.replay-callout{right:8px;left:8px}.replay-panel{width:100%}}", + ].join(""); + + var state = null; + var stateReadAt = 0; + var connected = false; + var el = {}; + + function h(tag, className, text) { + var node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; + } + + function field(grid, label) { + grid.appendChild(h("dt", null, label)); + var value = h("dd", null, "-"); + grid.appendChild(value); + return value; + } + + function build() { + var style = h("style"); + style.textContent = CSS; + document.head.appendChild(style); + + var root = h("div", "replay-callout"); + + var toggle = h("button", "replay-toggle"); + toggle.type = "button"; + el.dot = h("span", "replay-dot"); + el.summary = h("span", "replay-summary", "connecting…"); + toggle.appendChild(el.dot); + toggle.appendChild(h("span", "replay-tag", "replay")); + toggle.appendChild(el.summary); + toggle.appendChild(h("span", "replay-chevron", "▼")); + toggle.addEventListener("click", function () { + var open = !root.classList.contains("is-open"); + root.classList.toggle("is-open", open); + try { + window.localStorage.setItem(STORAGE_KEY, open ? "1" : "0"); + } catch (err) { + /* private mode */ + } + }); + root.appendChild(toggle); + + var panel = h("div", "replay-panel"); + + var grid = h("dl", "replay-grid"); + el.slot = field(grid, "Slot"); + el.epoch = field(grid, "Epoch"); + el.time = field(grid, "Time"); + el.head = field(grid, "Head"); + el.checkpoints = field(grid, "Checkpoints"); + el.execution = field(grid, "EL block"); + panel.appendChild(grid); + + var track = h("div", "replay-track"); + el.fill = h("div", "replay-track-fill"); + el.marker = h("div", "replay-track-target"); + track.appendChild(el.fill); + track.appendChild(el.marker); + panel.appendChild(track); + + var range = h("div", "replay-range"); + el.rangeFrom = h("span", null, ""); + el.rangeTo = h("span", null, ""); + range.appendChild(el.rangeFrom); + range.appendChild(el.rangeTo); + panel.appendChild(range); + + var controls = h("div", "replay-row"); + el.toggleRun = button("Start", "replay-btn replay-btn-main", onToggleRun); + el.step1 = button("+1", "replay-btn", function () { + send({ action: "step", slots: 1 }); + }); + el.stepEpoch = button("+1 epoch", "replay-btn", function () { + send({ action: "step", slots: (state && state.slots_per_epoch) || 32 }); + }); + el.speed = h("select"); + SPEEDS.forEach(function (speed) { + var option = h("option", null, speed.label); + option.value = String(speed.value); + el.speed.appendChild(option); + }); + el.speed.addEventListener("change", function () { + send({ action: "speed", speed: parseFloat(el.speed.value) }); + }); + controls.appendChild(el.toggleRun); + controls.appendChild(el.step1); + controls.appendChild(el.stepEpoch); + controls.appendChild(el.speed); + panel.appendChild(controls); + + var seek = h("div", "replay-row"); + el.seekSlot = h("input", "replay-seek-slot"); + el.seekSlot.type = "number"; + el.seekSlot.placeholder = "slot"; + el.seekSlot.addEventListener("keydown", function (event) { + if (event.key === "Enter") onSeek(); + }); + seek.appendChild(el.seekSlot); + seek.appendChild(button("Forward to", "replay-btn", onSeek)); + panel.appendChild(seek); + + el.source = h("div", "replay-source", ""); + panel.appendChild(el.source); + + el.error = h("div", "replay-error"); + panel.appendChild(el.error); + + root.appendChild(panel); + document.body.appendChild(root); + el.root = root; + + try { + if (window.localStorage.getItem(STORAGE_KEY) === "1") { + root.classList.add("is-open"); + } + } catch (err) { + /* private mode */ + } + } + + function button(label, className, handler) { + var node = h("button", className, label); + node.type = "button"; + node.addEventListener("click", handler); + return node; + } + + function onToggleRun() { + if (!state) return; + send(state.running ? { action: "stop" } : { action: "play", speed: parseFloat(el.speed.value) }); + } + + function onSeek() { + var slot = parseInt(el.seekSlot.value, 10); + if (!isFinite(slot)) { + showError("enter a slot to run to"); + return; + } + send({ action: "forward", slot: slot, speed: parseFloat(el.speed.value) }); + } + + function send(command) { + showError(""); + + fetch(base + "/replay/command", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(command), + }) + .then(function (response) { + return response.json().then(function (body) { + if (!response.ok) throw new Error(body.message || "command failed"); + return body; + }); + }) + .then(apply) + .catch(function (err) { + showError(err.message || String(err)); + }); + } + + function showError(message) { + el.error.textContent = message; + el.error.classList.toggle("is-shown", !!message); + } + + // tuneRefresh retunes the explorer's polling pages to the replay's pace. At 16x a + // slot goes by in well under a second, so the stock 15s refresh would leave the + // index page many epochs behind what the replay is actually showing. + function tuneRefresh(status) { + var interval = REFRESH_MAX_MS; + + if (status.running) { + interval = status.rate > 0 + ? (status.slot_duration_ms || 12000) / status.rate + : REFRESH_STEPPING_MS; + } + + window.doraIndexRefreshInterval = Math.max(REFRESH_MIN_MS, Math.min(REFRESH_MAX_MS, interval)); + } + + function apply(status) { + state = status; + stateReadAt = Date.now(); + + tuneRefresh(status); + + // only follow the server's speed when the user is not mid-selection + if (document.activeElement !== el.speed) { + var speed = String(status.speed); + if (!SPEEDS.some(function (entry) { return String(entry.value) === speed; })) { + var extra = h("option", null, speed + "x"); + extra.value = speed; + el.speed.appendChild(extra); + } + el.speed.value = speed; + } + + render(); + } + + // virtualNow interpolates the replay clock between status updates, so a playing + // replay shows a moving clock rather than stepping once per event. + function virtualNow() { + if (!state) return null; + var anchor = Date.parse(state.virtual_time); + if (isNaN(anchor)) return null; + if (!state.rate) return anchor; + return anchor + (Date.now() - stateReadAt) * state.rate; + } + + function slotAt(timeMs) { + if (!state || !state.slot_duration_ms) return state ? state.virtual_slot : 0; + var genesis = Date.parse(state.genesis_time); + if (isNaN(genesis) || timeMs < genesis) return state.virtual_slot; + return Math.floor((timeMs - genesis) / state.slot_duration_ms); + } + + function mode() { + if (!connected) return { label: "disconnected", dot: "is-offline" }; + if (!state) return { label: "connecting…", dot: "" }; + if (!state.running) return { label: "paused", dot: "" }; + if (state.holding) { + return { label: "loading state" + (state.state_loads > 1 ? " (" + state.state_loads + ")" : ""), dot: "is-waiting" }; + } + if (state.speed > 0) return { label: "playing " + trim(state.speed) + "x", dot: "is-playing" }; + if (state.target_slot) return { label: "stepping", dot: "is-stepping" }; + return { label: "max speed", dot: "is-stepping" }; + } + + function trim(value) { + return String(Math.round(value * 100) / 100); + } + + function num(value) { + return (value === undefined || value === null) ? "-" : value.toLocaleString("en-US"); + } + + function render() { + var status = mode(); + el.dot.className = "replay-dot " + status.dot; + + if (!state) { + el.summary.textContent = status.label; + return; + } + + var now = virtualNow(); + var slot = state.running && state.rate && !state.holding ? slotAt(now) : state.virtual_slot; + + el.summary.textContent = "epoch " + num(state.virtual_epoch) + " · " + status.label; + + el.slot.textContent = num(slot) + + (state.target_slot ? " → " + num(state.target_slot) : ""); + el.epoch.textContent = num(state.virtual_epoch) + + (state.upstream_epoch ? " of " + num(state.upstream_epoch) + " upstream" : ""); + el.time.textContent = now ? new Date(now).toISOString().replace("T", " ").replace(/\.\d+Z$/, "Z") : "-"; + el.head.textContent = num(state.head_slot) + + (state.head_root ? " " + state.head_root.slice(0, 10) + "…" : ""); + el.checkpoints.textContent = "justified " + num(state.justified_epoch) + + " · finalized " + num(state.finalized_epoch); + el.execution.textContent = num(state.execution_block); + + var from = state.start_slot || 0; + var to = state.upstream_slot || state.target_slot || slot; + var span = to - from; + el.fill.style.width = span > 0 ? Math.max(0, Math.min(100, ((slot - from) / span) * 100)) + "%" : "0"; + + if (state.target_slot && span > 0) { + el.marker.style.display = "block"; + el.marker.style.left = Math.max(0, Math.min(100, ((state.target_slot - from) / span) * 100)) + "%"; + } else { + el.marker.style.display = "none"; + } + + el.rangeFrom.textContent = "from " + num(from); + el.rangeTo.textContent = state.upstream_slot ? "chain head " + num(state.upstream_slot) : ""; + + el.toggleRun.textContent = state.running ? "Stop" : "Start"; + el.step1.disabled = false; + el.stepEpoch.disabled = false; + + el.source.textContent = state.upstream + (state.tracoor ? " (+tracoor)" : ""); + } + + function connect() { + var stream = new EventSource(base + "/replay/events?topics=status"); + + stream.addEventListener("open", function () { + connected = true; + render(); + }); + + stream.addEventListener("status", function (event) { + connected = true; + try { + apply(JSON.parse(event.data)); + } catch (err) { + /* ignore a malformed frame and wait for the next one */ + } + }); + + stream.addEventListener("error", function () { + connected = false; + render(); + // EventSource reconnects on its own + }); + } + + function start() { + build(); + + fetch(base + "/replay/status") + .then(function (response) { return response.json(); }) + .then(function (status) { + connected = true; + apply(status); + }) + .catch(function () { + connected = false; + render(); + }); + + connect(); + + // keep the interpolated clock and progress bar moving between status events + window.setInterval(render, 500); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", start); + } else { + start(); + } +})(); diff --git a/replay/cache.go b/replay/cache.go new file mode 100644 index 00000000..4c8fa994 --- /dev/null +++ b/replay/cache.go @@ -0,0 +1,108 @@ +package replay + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +// artifactCache records the immutable artifacts a replay fetched, so a slot range is +// downloaded from the devnet once and replays offline and reproducibly afterwards. +type artifactCache struct { + logger logrus.FieldLogger + dir string +} + +func newArtifactCache(logger logrus.FieldLogger, dir string) (*artifactCache, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("could not create cache dir %v: %w", dir, err) + } + + return &artifactCache{logger: logger, dir: dir}, nil +} + +// isImmutablePath reports whether a response for this path can never change. Only +// root-addressed artifacts qualify: slot-addressed ones would go stale across a reorg, +// and everything under /node or /config reflects the upstream's live state. +func isImmutablePath(path string) bool { + segments := strings.Split(strings.Trim(path, "/"), "/") + + for _, segment := range segments { + if strings.HasPrefix(segment, "0x") && len(segment) == 66 { + return true + } + } + + return false +} + +func (c *artifactCache) key(path, accept string) string { + sum := sha256.Sum256([]byte(path + "\x00" + accept)) + + return hex.EncodeToString(sum[:]) +} + +type cacheMeta struct { + Header http.Header `json:"header"` +} + +func (c *artifactCache) paths(key string) (string, string) { + dir := filepath.Join(c.dir, key[:2]) + + return filepath.Join(dir, key+".bin"), filepath.Join(dir, key+".json") +} + +// load returns a cached artifact, or nil when it is not recorded. +func (c *artifactCache) load(key string) *artifact { + bodyPath, metaPath := c.paths(key) + + body, err := os.ReadFile(bodyPath) + if err != nil { + return nil + } + + metaData, err := os.ReadFile(metaPath) + if err != nil { + return nil + } + + meta := cacheMeta{} + if err := json.Unmarshal(metaData, &meta); err != nil { + return nil + } + + return &artifact{body: body, header: meta.Header} +} + +func (c *artifactCache) store(key string, art *artifact) { + bodyPath, metaPath := c.paths(key) + + if err := os.MkdirAll(filepath.Dir(bodyPath), 0o755); err != nil { + c.logger.WithError(err).Debugf("could not create cache subdir for %v", key) + return + } + + metaData, err := json.Marshal(cacheMeta{Header: art.header}) + if err != nil { + c.logger.WithError(err).Debugf("could not encode cache meta for %v", key) + return + } + + // write the body first: a body without meta is simply treated as a cache miss, + // while meta without a body would be too + if err := os.WriteFile(bodyPath, art.body, 0o644); err != nil { + c.logger.WithError(err).Debugf("could not write cache body for %v", key) + return + } + + if err := os.WriteFile(metaPath, metaData, 0o644); err != nil { + c.logger.WithError(err).Debugf("could not write cache meta for %v", key) + } +} diff --git a/replay/cache_test.go b/replay/cache_test.go new file mode 100644 index 00000000..54c3b079 --- /dev/null +++ b/replay/cache_test.go @@ -0,0 +1,67 @@ +package replay + +import ( + "net/http" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestIsImmutablePath(t *testing.T) { + root := "0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + + tests := []struct { + path string + want bool + }{ + {path: "/eth/v2/debug/beacon/states/" + root, want: true}, + {path: "/eth/v2/beacon/blocks/" + root, want: true}, + {path: "/eth/v1/beacon/states/" + root + "/finality_checkpoints", want: true}, + {path: "/eth/v1/beacon/headers/49152", want: false}, + {path: "/eth/v1/beacon/headers/head", want: false}, + {path: "/eth/v1/config/spec", want: false}, + {path: "/eth/v1/node/syncing", want: false}, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + require.Equal(t, test.want, isImmutablePath(test.path)) + }) + } +} + +func TestArtifactCacheRoundTrip(t *testing.T) { + cache, err := newArtifactCache(logrus.New(), t.TempDir()) + require.NoError(t, err) + + key := cache.key("/eth/v2/beacon/blocks/0xabc", "application/octet-stream") + + require.Nil(t, cache.load(key), "an unrecorded artifact must be a cache miss") + + stored := &artifact{ + body: []byte{0x01, 0x02, 0x03}, + header: http.Header{ + "Content-Type": []string{"application/octet-stream"}, + "Eth-Consensus-Version": []string{"gloas"}, + }, + } + + cache.store(key, stored) + + loaded := cache.load(key) + require.NotNil(t, loaded) + require.Equal(t, stored.body, loaded.body) + require.Equal(t, "gloas", loaded.header.Get("Eth-Consensus-Version"), + "the fork header must survive the cache, or SSZ responses cannot be decoded") +} + +func TestArtifactCacheKeyIncludesEncoding(t *testing.T) { + cache, err := newArtifactCache(logrus.New(), t.TempDir()) + require.NoError(t, err) + + json := cache.key("/eth/v2/beacon/blocks/0xabc", "application/json") + ssz := cache.key("/eth/v2/beacon/blocks/0xabc", "application/octet-stream") + + require.NotEqual(t, json, ssz, "the same artifact in two encodings must not share a cache entry") +} diff --git a/replay/chain.go b/replay/chain.go new file mode 100644 index 00000000..a3bc70cb --- /dev/null +++ b/replay/chain.go @@ -0,0 +1,190 @@ +package replay + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/sirupsen/logrus" +) + +// forkOrder lists the forks oldest first, paired with the spec key that activates them. +// The names are the ones the beacon API reports in Eth-Consensus-Version. +var forkOrder = []struct { + name string + specKey string +}{ + {name: "altair", specKey: "ALTAIR_FORK_EPOCH"}, + {name: "bellatrix", specKey: "BELLATRIX_FORK_EPOCH"}, + {name: "capella", specKey: "CAPELLA_FORK_EPOCH"}, + {name: "deneb", specKey: "DENEB_FORK_EPOCH"}, + {name: "electra", specKey: "ELECTRA_FORK_EPOCH"}, + {name: "fulu", specKey: "FULU_FORK_EPOCH"}, + {name: "gloas", specKey: "GLOAS_FORK_EPOCH"}, + {name: "heze", specKey: "HEZE_FORK_EPOCH"}, +} + +type forkActivation struct { + name string + epoch uint64 +} + +// chainInfo holds the genesis and timing parameters the replay needs to translate +// between slots and (virtual) wall clock time. Genesis stays the real one, so every +// timestamp the explorer renders is the true historical time of the replayed slot. +type chainInfo struct { + genesisTime time.Time + slotDuration time.Duration + slotsPerEpoch uint64 + forks []forkActivation +} + +// bidsActiveAt reports whether a block at this slot can carry an execution payload bid, +// which only exists from the Gloas fork (EIP-7732) onwards. Before that, fetching the +// block to look for one would be a wasted round trip on every slot. +func (c *chainInfo) bidsActiveAt(slot uint64) bool { + epoch := c.epochOf(slot) + + for _, fork := range c.forks { + if fork.name == "gloas" { + return epoch >= fork.epoch + } + } + + return false +} + +// forkAt returns the fork a slot belongs to, as reported in Eth-Consensus-Version. +// Artifacts served from tracoor carry no such header of their own, and without it a +// client cannot tell which SSZ container it received. +func (c *chainInfo) forkAt(slot uint64) string { + epoch := c.epochOf(slot) + name := "phase0" + + for _, fork := range c.forks { + if epoch >= fork.epoch { + name = fork.name + } + } + + return name +} + +func (c *chainInfo) slotTime(slot uint64) time.Time { + return c.genesisTime.Add(time.Duration(slot) * c.slotDuration) +} + +func (c *chainInfo) epochOf(slot uint64) uint64 { + return slot / c.slotsPerEpoch +} + +// loadChainInfo reads genesis and the timing specs from the beacon upstream. +func loadChainInfo(ctx context.Context, up *upstream) (*chainInfo, error) { + genesisRsp := struct { + Data struct { + GenesisTime string `json:"genesis_time"` + } `json:"data"` + }{} + + if err := up.getJSON(ctx, "/eth/v1/beacon/genesis", &genesisRsp); err != nil { + return nil, fmt.Errorf("error fetching genesis: %w", err) + } + + genesisUnix, err := strconv.ParseInt(genesisRsp.Data.GenesisTime, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid genesis_time %q: %w", genesisRsp.Data.GenesisTime, err) + } + + specRsp := struct { + Data map[string]any `json:"data"` + }{} + + if err := up.getJSON(ctx, "/eth/v1/config/spec", &specRsp); err != nil { + return nil, fmt.Errorf("error fetching config spec: %w", err) + } + + slotDurationMs, err := specUint(specRsp.Data, "SLOT_DURATION_MS") + if err != nil { + secondsPerSlot, secErr := specUint(specRsp.Data, "SECONDS_PER_SLOT") + if secErr != nil { + return nil, fmt.Errorf("could not determine slot duration: %w", err) + } + + slotDurationMs = secondsPerSlot * 1000 + } + + slotsPerEpoch, err := specUint(specRsp.Data, "SLOTS_PER_EPOCH") + if err != nil { + return nil, err + } + + if slotDurationMs == 0 || slotsPerEpoch == 0 { + return nil, fmt.Errorf("upstream reported a zero slot duration or epoch length") + } + + return &chainInfo{ + genesisTime: time.Unix(genesisUnix, 0).UTC(), + slotDuration: time.Duration(slotDurationMs) * time.Millisecond, + slotsPerEpoch: slotsPerEpoch, + forks: parseForkSchedule(specRsp.Data), + }, nil +} + +// parseForkSchedule reads the activation epoch of every fork the upstream knows about, +// in activation order. Forks that are not scheduled are left out. +func parseForkSchedule(specs map[string]any) []forkActivation { + forks := make([]forkActivation, 0, len(forkOrder)) + + for _, fork := range forkOrder { + epoch, err := specUint(specs, fork.specKey) + if err != nil { + continue + } + + forks = append(forks, forkActivation{name: fork.name, epoch: epoch}) + } + + return forks +} + +// specUint reads a numeric spec value. The config endpoint reports numbers as decimal +// strings, but not every entry is a number (the blob schedule is a list), so anything +// that is not a plain numeric string is reported as an error rather than coerced. +func specUint(specs map[string]any, key string) (uint64, error) { + raw, ok := specs[key] + if !ok { + return 0, fmt.Errorf("spec value %v missing", key) + } + + text, ok := raw.(string) + if !ok { + return 0, fmt.Errorf("spec value %v is not a number (got %T)", key, raw) + } + + value, err := strconv.ParseUint(text, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid spec value %v=%q: %w", key, text, err) + } + + return value, nil +} + +// SlotsPerEpoch reads just the epoch length from a beacon node, so a start epoch given +// on the command line can be converted to a slot before the replay is constructed. +func SlotsPerEpoch(ctx context.Context, upstreamURL string) (uint64, error) { + up, err := newUpstream(logrus.New(), &Config{UpstreamURL: upstreamURL}) + if err != nil { + return 0, err + } + + rsp := struct { + Data map[string]any `json:"data"` + }{} + + if err := up.getJSON(ctx, "/eth/v1/config/spec", &rsp); err != nil { + return 0, fmt.Errorf("error fetching config spec: %w", err) + } + + return specUint(rsp.Data, "SLOTS_PER_EPOCH") +} diff --git a/replay/clock.go b/replay/clock.go new file mode 100644 index 00000000..97b522c9 --- /dev/null +++ b/replay/clock.go @@ -0,0 +1,133 @@ +package replay + +import ( + "sync" + "time" +) + +// clock is the virtual time the replay serves to the explorer. It is anchored to a +// point in virtual time and advances at `rate` virtual seconds per real second, so a +// paused replay (rate 0) simply holds its anchor while a playing one interpolates. +type clock struct { + mutex sync.RWMutex + anchorReal time.Time + anchorVirt time.Time + rate float64 + + // holds freezes the clock without forgetting the rate it was running at, so the + // replay can wait for the explorer without any virtual time passing. Holds nest. + holds int +} + +func newClock(virt time.Time) *clock { + return &clock{ + anchorReal: time.Now(), + anchorVirt: virt, + } +} + +// now returns the current virtual time. +func (c *clock) now() time.Time { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.nowLocked() +} + +func (c *clock) nowLocked() time.Time { + rate := c.effectiveRateLocked() + if rate == 0 { + return c.anchorVirt + } + + return c.anchorVirt.Add(time.Duration(float64(time.Since(c.anchorReal)) * rate)) +} + +// effectiveRateLocked is the rate the clock is actually running at: zero while held, +// whatever was configured otherwise. +func (c *clock) effectiveRateLocked() float64 { + if c.holds > 0 { + return 0 + } + + return c.rate +} + +// set jumps the virtual clock to an absolute point in time, keeping the current rate. +func (c *clock) set(virt time.Time) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.anchorReal = time.Now() + c.anchorVirt = virt +} + +// setRate changes the playback rate, re-anchoring so no virtual time is lost. +func (c *clock) setRate(rate float64) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.anchorVirt = c.nowLocked() + c.anchorReal = time.Now() + c.rate = rate +} + +// hold freezes the clock where it is. The explorer mirrors the effective rate, so it +// stops moving too rather than drifting ahead of what the replay has actually served. +func (c *clock) hold() { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.anchorVirt = c.nowLocked() + c.anchorReal = time.Now() + c.holds++ +} + +// release lifts one hold, resuming at the configured rate once the last one is gone. +func (c *clock) release() { + c.mutex.Lock() + defer c.mutex.Unlock() + + if c.holds == 0 { + return + } + + c.anchorVirt = c.nowLocked() + c.anchorReal = time.Now() + c.holds-- +} + +func (c *clock) isHeld() bool { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.holds > 0 +} + +// state returns the current virtual time and the rate it is moving at, as served to +// the explorer. +func (c *clock) state() (time.Time, float64) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.nowLocked(), c.effectiveRateLocked() +} + +// realDelayUntil returns how long to wait in real time for the virtual clock to reach +// the given point, at the rate it is configured to run at. It returns 0 when the clock +// is paused or already past it, since a paused clock would never get there on its own. +func (c *clock) realDelayUntil(virt time.Time) time.Duration { + c.mutex.RLock() + defer c.mutex.RUnlock() + + if c.rate <= 0 { + return 0 + } + + remaining := virt.Sub(c.nowLocked()) + if remaining <= 0 { + return 0 + } + + return time.Duration(float64(remaining) / c.rate) +} diff --git a/replay/clock_test.go b/replay/clock_test.go new file mode 100644 index 00000000..5c8248b7 --- /dev/null +++ b/replay/clock_test.go @@ -0,0 +1,72 @@ +package replay + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPausedClockHoldsItsAnchor(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + + time.Sleep(20 * time.Millisecond) + + require.Equal(t, anchor, clock.now()) + + now, rate := clock.state() + require.Equal(t, anchor, now) + require.Zero(t, rate) +} + +func TestSetJumpsTheClock(t *testing.T) { + clock := newClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + + target := time.Date(2026, 1, 1, 0, 0, 12, 0, time.UTC) + clock.set(target) + + require.Equal(t, target, clock.now()) +} + +func TestPlayingClockAdvancesWithRealTime(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + clock.setRate(100) + + time.Sleep(20 * time.Millisecond) + + elapsed := clock.now().Sub(anchor) + require.Greater(t, elapsed, time.Second, "100x rate should cover >1s of virtual time in 20ms") +} + +func TestSetRateKeepsElapsedVirtualTime(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + clock.setRate(100) + + time.Sleep(20 * time.Millisecond) + + clock.setRate(0) + frozen := clock.now() + + time.Sleep(20 * time.Millisecond) + + require.Equal(t, frozen, clock.now(), "a paused clock must not drift") + require.True(t, frozen.After(anchor), "the virtual time gained while playing must be kept") +} + +func TestRealDelayUntilScalesWithRate(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + + // a paused clock will never reach the target on its own + require.Zero(t, clock.realDelayUntil(anchor.Add(time.Minute))) + + clock.setRate(4) + + delay := clock.realDelayUntil(anchor.Add(4 * time.Second)) + require.InDelta(t, float64(time.Second), float64(delay), float64(50*time.Millisecond)) + + require.Zero(t, clock.realDelayUntil(anchor.Add(-time.Second))) +} diff --git a/replay/clproxy.go b/replay/clproxy.go new file mode 100644 index 00000000..e1b886a3 --- /dev/null +++ b/replay/clproxy.go @@ -0,0 +1,299 @@ +package replay + +import ( + "context" + "encoding/hex" + "encoding/json" + "net/http" + "strconv" + "strings" + + "github.com/ethpandaops/go-eth2-client/spec/phase0" +) + +// blockIDPaths are the beacon API paths whose first variable segment identifies a +// block, and stateIDPaths the ones whose first variable segment identifies a state. +// Everything else is proxied unchanged. +var ( + blockIDPaths = []string{ + "/eth/v1/beacon/headers/", + "/eth/v1/beacon/blocks/", + "/eth/v2/beacon/blocks/", + "/eth/v1/beacon/blinded_blocks/", + "/eth/v2/beacon/blinded_blocks/", + "/eth/v1/beacon/blob_sidecars/", + "/eth/v1/beacon/blobs/", + "/eth/v1/beacon/execution_payload_envelopes/", + "/eth/v1/beacon/execution_proofs/", + "/eth/v1/beacon/inclusion_lists/", + } + + stateIDPaths = []string{ + "/eth/v1/beacon/states/", + "/eth/v2/beacon/states/", + "/eth/v1/debug/beacon/states/", + "/eth/v2/debug/beacon/states/", + } +) + +// clHandler is the fake beacon node. Everything is proxied to the real upstream except +// where the virtual head matters: the sync status is synthesized, the event stream is +// generated locally, `head` aliases resolve to the head at the virtual slot, and +// anything beyond the virtual slot is reported as missing. +func (r *Replay) clHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/eth/v1/node/syncing", r.serveSyncing) + mux.HandleFunc("/eth/v1/events", r.events.serveHTTP) + mux.HandleFunc("/eth/v1/beacon/states/head/finality_checkpoints", r.serveHeadFinality) + mux.HandleFunc("/", r.serveProxied) + + return mux +} + +func (r *Replay) serveSyncing(w http.ResponseWriter, _ *http.Request) { + r.mutex.RLock() + headSlot := uint64(0) + if r.head != nil { + headSlot = r.head.Slot + } + r.mutex.RUnlock() + + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "head_slot": strconv.FormatUint(headSlot, 10), + "sync_distance": "0", + "is_syncing": false, + "is_optimistic": false, + "el_offline": false, + }, + }) +} + +// serveHeadFinality answers from the finality the replay tracks rather than proxying. +// Reading finality upstream forces the node to load the head state, which for a +// replayed (historical) head is expensive at best and pruned away at worst. +func (r *Replay) serveHeadFinality(w http.ResponseWriter, _ *http.Request) { + r.mutex.RLock() + finality := r.finality + r.mutex.RUnlock() + + if finality == nil || len(finality.Raw) == 0 { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + if _, err := w.Write(finality.Raw); err != nil { + r.logger.WithError(err).Debug("error writing head finality response") + } +} + +func (r *Replay) serveProxied(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + writeAPIError(w, http.StatusNotImplemented, "the replay proxy only serves read requests") + return + } + + path, err := r.rewritePath(req.Context(), req.URL.Path) + if err != nil { + if err == errNotFound { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + writeAPIError(w, http.StatusBadGateway, err.Error()) + + return + } + + if query := req.URL.RawQuery; query != "" { + if err := r.checkQueryCutoff(req); err != nil { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + path += "?" + query + } + + // a full beacon state blocks the explorer for as long as it takes to arrive; the + // driver waits for these rather than replaying slots the explorer cannot see yet + if isStatePath(path) { + defer r.states.begin()() + } + + r.upstream.serve(w, req, path) +} + +// rewritePath resolves the block/state identifier in a beacon API path against the +// virtual head, so the explorer can never see past the slot the replay has reached. +func (r *Replay) rewritePath(ctx context.Context, path string) (string, error) { + for _, prefix := range blockIDPaths { + if strings.HasPrefix(path, prefix) { + return r.rewriteID(ctx, path, prefix, false) + } + } + + for _, prefix := range stateIDPaths { + if strings.HasPrefix(path, prefix) { + return r.rewriteID(ctx, path, prefix, true) + } + } + + return path, nil +} + +func (r *Replay) rewriteID(ctx context.Context, path, prefix string, isState bool) (string, error) { + rest := strings.TrimPrefix(path, prefix) + + id, suffix, _ := strings.Cut(rest, "/") + if suffix != "" { + suffix = "/" + suffix + } + + resolved, err := r.resolveID(ctx, id, isState) + if err != nil { + return "", err + } + + return prefix + resolved + suffix, nil +} + +// resolveID maps a beacon API identifier to something the upstream can answer with the +// same meaning it had at the virtual head slot. +func (r *Replay) resolveID(ctx context.Context, id string, isState bool) (string, error) { + switch id { + case "head": + header := r.currentHead() + if header == nil { + return "", errNotFound + } + + if isState { + return header.StateRoot, nil + } + + return header.Root, nil + + case "finalized", "justified": + return r.resolveCheckpointID(ctx, id, isState) + + case "genesis": + return id, nil + } + + if strings.HasPrefix(id, "0x") { + // roots are only ever learned from artifacts the replay already released + return id, nil + } + + slot, err := strconv.ParseUint(id, 10, 64) + if err != nil { + // an identifier the replay does not understand is passed through unchanged + return id, nil + } + + r.mutex.RLock() + virtualSlot := r.virtualSlot + r.mutex.RUnlock() + + if slot > virtualSlot { + return "", errNotFound + } + + return id, nil +} + +// resolveCheckpointID turns the `finalized` and `justified` aliases into the concrete +// roots the chain had at the virtual head, rather than the ones the upstream has today. +func (r *Replay) resolveCheckpointID(ctx context.Context, id string, isState bool) (string, error) { + r.mutex.RLock() + finality := r.finality + r.mutex.RUnlock() + + if finality == nil { + return "", errNotFound + } + + root := finality.FinalizedRoot + if id == "justified" { + root = finality.JustifiedRoot + } + + if !isState { + return root, nil + } + + header, err := r.upstream.headerByRoot(ctx, root) + if err != nil || header == nil { + return "", errNotFound + } + + return header.StateRoot, nil +} + +// checkQueryCutoff rejects queries that select data beyond the virtual head slot. +func (r *Replay) checkQueryCutoff(req *http.Request) error { + slotParam := req.URL.Query().Get("slot") + if slotParam == "" { + return nil + } + + slot, err := strconv.ParseUint(slotParam, 10, 64) + if err != nil { + return nil + } + + r.mutex.RLock() + virtualSlot := r.virtualSlot + r.mutex.RUnlock() + + if slot > virtualSlot { + return errNotFound + } + + return nil +} + +func (r *Replay) currentHead() *blockHeader { + r.mutex.RLock() + defer r.mutex.RUnlock() + + return r.head +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + + if err := json.NewEncoder(w).Encode(body); err != nil { + return + } +} + +// writeAPIError answers in the shape the beacon API uses for errors, so clients report +// something meaningful instead of a decode failure. +func writeAPIError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]any{ + "code": status, + "message": message, + }) +} + +// parseRoot decodes a 0x-prefixed root for use in an event payload. A malformed root +// yields the zero root, which is what the upstream would have to have served for this +// to happen at all. +func parseRoot(value string) phase0.Root { + root := phase0.Root{} + + decoded, err := hex.DecodeString(strings.TrimPrefix(value, "0x")) + if err != nil || len(decoded) != len(root) { + return root + } + + copy(root[:], decoded) + + return root +} diff --git a/replay/clproxy_test.go b/replay/clproxy_test.go new file mode 100644 index 00000000..a61fd654 --- /dev/null +++ b/replay/clproxy_test.go @@ -0,0 +1,123 @@ +package replay + +import ( + "context" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func testReplay(virtualSlot uint64) *Replay { + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + return &Replay{ + logger: logger, + events: newEventHub(logger), + control: newLossyEventHub(logger), + states: newStateLoads(), + chain: &chainInfo{ + slotsPerEpoch: 32, + slotDuration: 12 * time.Second, + }, + upstream: &upstream{logger: logger}, + virtualSlot: virtualSlot, + head: &blockHeader{ + Slot: virtualSlot - 1, + Root: "0x1111111111111111111111111111111111111111111111111111111111111111", + StateRoot: "0x2222222222222222222222222222222222222222222222222222222222222222", + }, + finality: &finalityCheckpoints{ + JustifiedEpoch: 10, + JustifiedRoot: "0x3333333333333333333333333333333333333333333333333333333333333333", + FinalizedEpoch: 9, + FinalizedRoot: "0x4444444444444444444444444444444444444444444444444444444444444444", + }, + } +} + +func TestRewritePath(t *testing.T) { + replay := testReplay(100) + ctx := context.Background() + + tests := []struct { + name string + path string + want string + wantErr bool + }{ + { + name: "head header resolves to the head block root", + path: "/eth/v1/beacon/headers/head", + want: "/eth/v1/beacon/headers/" + replay.head.Root, + }, + { + name: "head finality resolves to the head state root", + path: "/eth/v1/beacon/states/head/finality_checkpoints", + want: "/eth/v1/beacon/states/" + replay.head.StateRoot + "/finality_checkpoints", + }, + { + name: "finalized block id resolves to the tracked checkpoint", + path: "/eth/v2/beacon/blocks/finalized", + want: "/eth/v2/beacon/blocks/" + replay.finality.FinalizedRoot, + }, + { + name: "roots are passed through", + path: "/eth/v2/debug/beacon/states/0x9999999999999999999999999999999999999999999999999999999999999999", + want: "/eth/v2/debug/beacon/states/0x9999999999999999999999999999999999999999999999999999999999999999", + }, + { + name: "genesis is passed through", + path: "/eth/v2/debug/beacon/states/genesis", + want: "/eth/v2/debug/beacon/states/genesis", + }, + { + name: "a slot at the virtual head is served", + path: "/eth/v1/beacon/headers/100", + want: "/eth/v1/beacon/headers/100", + }, + { + name: "a slot beyond the virtual head is hidden", + path: "/eth/v1/beacon/headers/101", + wantErr: true, + }, + { + name: "unrelated paths are untouched", + path: "/eth/v1/config/spec", + want: "/eth/v1/config/spec", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := replay.rewritePath(ctx, test.path) + + if test.wantErr { + require.ErrorIs(t, err, errNotFound) + return + } + + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} + +func TestRewritePathWithoutHead(t *testing.T) { + replay := testReplay(100) + replay.head = nil + + _, err := replay.rewritePath(context.Background(), "/eth/v1/beacon/headers/head") + require.ErrorIs(t, err, errNotFound) +} + +func TestParseRoot(t *testing.T) { + root := parseRoot("0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + require.Equal(t, byte(0x01), root[0]) + require.Equal(t, byte(0x20), root[31]) + + // a malformed root degrades to the zero root rather than panicking + require.Equal(t, parseRoot("nonsense"), parseRoot("0x00")) +} diff --git a/replay/config.go b/replay/config.go new file mode 100644 index 00000000..4874bd85 --- /dev/null +++ b/replay/config.go @@ -0,0 +1,95 @@ +package replay + +import ( + "fmt" + "time" +) + +// Config describes a replay run: where the real chain data comes from, where the +// fake nodes listen, and where in the chain the replay starts. +type Config struct { + // UpstreamURL is the beacon node the consensus proxy reads from. It must serve + // historical blocks and states for the replayed range, so point it at a single + // known-archive node rather than at a load balancer over mixed-retention nodes. + UpstreamURL string + + // ExecutionURL is the execution node the JSON-RPC proxy reads from. When empty + // the execution proxy is not started. + ExecutionURL string + + // TracoorURL and TracoorNetwork enable the tracoor fallback for artifacts the + // beacon node has pruned. Tracoor indexes by root, so it can only serve blocks + // and states whose root is already known from a header. + TracoorURL string + TracoorNetwork string + + CLListen string + ELListen string + ControlListen string + + // StartSlot is the first slot the replay serves. The head is initialized to the + // newest block at or before this slot. + StartSlot uint64 + + // Speed is the initial playback rate in virtual seconds per real second. A value + // of 0 starts the replay paused. + Speed float64 + + // CacheDir, when set, stores every immutable artifact fetched from upstream so a + // range is downloaded once and replays offline afterwards. + CacheDir string + + // EmitBids controls whether the winning execution payload bid of each Gloas block + // is replayed on the event stream. It costs one extra block fetch per slot. + EmitBids bool + + // StepSettle is the real time the driver pauses at each phase of a stepped slot, + // giving the explorer a chance to observe the slot boundary and process events. + StepSettle time.Duration + + // StateHoldTimeout bounds how long the replay freezes its clock waiting for the + // explorer to finish loading a beacon state, so a stuck read cannot wedge a run. + StateHoldTimeout time.Duration +} + +// DefaultConfig returns a config with the non-chain-specific defaults filled in. +func DefaultConfig() Config { + return Config{ + CLListen: "127.0.0.1:15052", + ELListen: "127.0.0.1:15545", + ControlListen: "127.0.0.1:15000", + Speed: 0, + EmitBids: true, + StepSettle: 250 * time.Millisecond, + StateHoldTimeout: 2 * time.Minute, + } +} + +// Validate checks the config for the combinations the replay cannot run with. +func (c *Config) Validate() error { + if c.UpstreamURL == "" { + return fmt.Errorf("no beacon upstream configured (--upstream): tracoor alone cannot resolve slots") + } + + if c.TracoorURL != "" && c.TracoorNetwork == "" { + return fmt.Errorf("--tracoor-network is required when --tracoor is set") + } + + if c.StartSlot == 0 { + return fmt.Errorf("no start slot configured (--start-slot / --start-epoch)") + } + + if c.Speed < 0 { + return fmt.Errorf("speed must not be negative") + } + + if c.StepSettle <= 0 { + c.StepSettle = 250 * time.Millisecond + } + + if c.StateHoldTimeout <= 0 { + c.StateHoldTimeout = 2 * time.Minute + } + + return nil +} diff --git a/replay/console.go b/replay/console.go new file mode 100644 index 00000000..1f2f509c --- /dev/null +++ b/replay/console.go @@ -0,0 +1,219 @@ +package replay + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" +) + +const consoleHelp = `commands: + status show where the replay currently stands + step [n] advance n slots (default 1), then pause + forward [speed] + run to a slot, emitting every slot on the way; without a + speed it advances as fast as upstream allows + play [speed] run continuously at x real time (default 1) + stop pause the replay and freeze the virtual clock + start resume at the last speed + help show this help + quit stop the replay and exit +` + +// RunConsole reads commands from stdin until the input ends or `quit` is entered. It +// returns when the replay should shut down. +func (r *Replay) RunConsole(ctx context.Context, in io.Reader, out io.Writer) { + lines := make(chan string) + + go func() { + defer close(lines) + + scanner := bufio.NewScanner(in) + for scanner.Scan() { + lines <- scanner.Text() + } + }() + + consolePrint(out, consoleHelp) + r.printStatus(out) + consolePrint(out, "replay> ") + + for { + select { + case <-ctx.Done(): + return + case line, ok := <-lines: + if !ok { + return + } + + if r.runCommand(out, line) { + return + } + + consolePrint(out, "replay> ") + } + } +} + +// runCommand executes one console line and reports whether the replay should exit. +func (r *Replay) runCommand(out io.Writer, line string) bool { + fields := strings.Fields(line) + if len(fields) == 0 { + return false + } + + command, args := fields[0], fields[1:] + + switch command { + case "status", "s": + r.printStatus(out) + + case "step": + slots := uint64(1) + + if len(args) > 0 { + parsed, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + consolePrintf(out, "invalid slot count %q\n", args[0]) + + return false + } + + slots = parsed + } + + r.Step(slots) + + case "forward", "fwd": + if len(args) == 0 { + consolePrint(out, "usage: forward [speed]\n") + + return false + } + + slot, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + consolePrintf(out, "invalid slot %q\n", args[0]) + + return false + } + + speed := 0.0 + + if len(args) > 1 { + speed, err = parseSpeed(args[1]) + if err != nil { + consolePrintf(out, "%v\n", err) + + return false + } + } + + if err := r.Forward(slot, speed); err != nil { + consolePrintf(out, "%v\n", err) + } + + case "play": + speed := 1.0 + + if len(args) > 0 { + parsed, err := parseSpeed(args[0]) + if err != nil { + consolePrintf(out, "%v\n", err) + + return false + } + + speed = parsed + } + + r.Play(speed) + + case "stop", "pause": + r.Pause() + r.printStatus(out) + + case "start", "resume": + r.Resume() + + case "help", "?": + consolePrint(out, consoleHelp) + + case "quit", "exit", "q": + return true + + default: + consolePrintf(out, "unknown command %q (try `help`)\n", command) + } + + return false +} + +func (r *Replay) printStatus(out io.Writer) { + status := r.Status() + + mode := "paused" + + switch { + case status.Holding: + mode = fmt.Sprintf("holding for %v state load(s)", status.StateLoads) + case status.Running && status.Speed > 0: + mode = fmt.Sprintf("playing %gx", status.Speed) + case status.Running && status.TargetSlot > 0: + mode = fmt.Sprintf("stepping to %v", status.TargetSlot) + case status.Running: + mode = "running at max speed" + } + + upstream := status.Upstream + if status.Tracoor { + upstream += " (+tracoor)" + } + + consolePrintf(out, " slot %v epoch %v head %v [%v]\n", + status.VirtualSlot, status.VirtualEpoch, status.HeadSlot, shortRoot(status.HeadRoot)) + consolePrintf(out, " justified %v finalized %v el block %v\n", + status.JustifiedEpoch, status.FinalizedEpoch, status.ExecutionBlock) + consolePrintf(out, " %v time %v streams %v\n", + mode, status.VirtualTime.Format(time.RFC3339), status.Subscribers) + consolePrintf(out, " upstream %v\n", upstream) +} + +// parseSpeed reads a playback rate, accepting both `4` and `4x`. +func parseSpeed(value string) (float64, error) { + speed, err := strconv.ParseFloat(strings.TrimSuffix(value, "x"), 64) + if err != nil || speed <= 0 { + return 0, fmt.Errorf("invalid speed %q", value) + } + + return speed, nil +} + +// consolePrint and consolePrintf write to the console; a broken console is not worth +// aborting a replay for, so write errors are deliberately ignored. +func consolePrint(out io.Writer, text string) { + _, _ = io.WriteString(out, text) +} + +func consolePrintf(out io.Writer, format string, args ...any) { + _, _ = fmt.Fprintf(out, format, args...) +} + +func shortRoot(root string) string { + if len(root) <= 12 { + return root + } + + return root[:10] + "…" +} + +// Stdio returns the console's default streams, kept in one place so the caller does not +// have to reach into os from its own package. +func Stdio() (io.Reader, io.Writer) { + return os.Stdin, os.Stdout +} diff --git a/replay/console_test.go b/replay/console_test.go new file mode 100644 index 00000000..2ced0761 --- /dev/null +++ b/replay/console_test.go @@ -0,0 +1,136 @@ +package replay + +import ( + "bytes" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func testConsoleReplay(virtualSlot uint64) *Replay { + replay := testReplay(virtualSlot) + replay.clock = newClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + replay.wake = make(chan struct{}, 1) + replay.cfg = DefaultConfig() + + return replay +} + +func TestConsoleCommands(t *testing.T) { + tests := []struct { + name string + line string + wantRunning bool + wantSpeed float64 + wantTarget uint64 + wantOutput string + }{ + { + name: "step advances one slot by default", + line: "step", + wantRunning: true, + wantTarget: 101, + }, + { + name: "step takes a slot count", + line: "step 32", + wantRunning: true, + wantTarget: 132, + }, + { + name: "forward sets an absolute target", + line: "forward 4096", + wantRunning: true, + wantTarget: 4096, + }, + { + name: "forward takes an optional speed", + line: "forward 4096 6x", + wantRunning: true, + wantSpeed: 6, + wantTarget: 4096, + }, + { + name: "forward refuses to rewind", + line: "forward 50", + wantRunning: false, + wantOutput: "cannot rewind", + }, + { + name: "play defaults to real time", + line: "play", + wantRunning: true, + wantSpeed: 1, + }, + { + name: "play takes an x-suffixed speed", + line: "play 8x", + wantRunning: true, + wantSpeed: 8, + }, + { + name: "an invalid speed is rejected", + line: "play banana", + wantOutput: "invalid speed", + }, + { + name: "an unknown command is reported", + line: "frobnicate", + wantOutput: "unknown command", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + replay := testConsoleReplay(100) + out := &bytes.Buffer{} + + require.False(t, replay.runCommand(out, test.line)) + + require.Equal(t, test.wantRunning, replay.running) + require.Equal(t, test.wantSpeed, replay.speed) + require.Equal(t, test.wantTarget, replay.target) + + if test.wantOutput != "" { + require.Contains(t, out.String(), test.wantOutput) + } + }) + } +} + +func TestConsoleQuit(t *testing.T) { + replay := testConsoleReplay(100) + require.True(t, replay.runCommand(&bytes.Buffer{}, "quit")) +} + +func TestConsoleStopAndResume(t *testing.T) { + replay := testConsoleReplay(100) + out := &bytes.Buffer{} + + replay.runCommand(out, "play 4x") + require.True(t, replay.running) + + replay.runCommand(out, "stop") + require.False(t, replay.running) + + // the clock must not keep running while the replay is paused + frozen := replay.clock.now() + time.Sleep(20 * time.Millisecond) + require.Equal(t, frozen, replay.clock.now()) + + replay.runCommand(out, "start") + require.True(t, replay.running) + require.Equal(t, 4.0, replay.speed) +} + +func TestConsoleStatus(t *testing.T) { + replay := testConsoleReplay(100) + out := &bytes.Buffer{} + + replay.runCommand(out, "status") + + require.Contains(t, out.String(), "slot 100") + require.Contains(t, out.String(), "finalized 9") + require.Contains(t, out.String(), "paused") +} diff --git a/replay/control.go b/replay/control.go new file mode 100644 index 00000000..7c28ccdd --- /dev/null +++ b/replay/control.go @@ -0,0 +1,147 @@ +package replay + +import ( + _ "embed" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// controlUI is the replay control panel the explorer side-loads. Serving it from here +// keeps the explorer's side of the integration down to a single script tag. +// +//go:embed assets/replay-ui.js +var controlUI []byte + +// Command is a control instruction, as posted to /replay/command. It mirrors what the +// interactive console offers, so the explorer UI and the console drive the same replay +// through the same code. +type Command struct { + Action string `json:"action"` + Speed float64 `json:"speed,omitempty"` + Slots uint64 `json:"slots,omitempty"` + Slot uint64 `json:"slot,omitempty"` +} + +// controlHandler serves the replay's own API: the virtual clock the explorer follows, a +// status snapshot, a status event stream, and the commands that drive the replay. +func (r *Replay) controlHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/replay/clock", r.serveClock) + mux.HandleFunc("/replay/status", r.serveStatus) + mux.HandleFunc("/replay/events", r.control.serveHTTP) + mux.HandleFunc("/replay/command", r.serveCommand) + mux.HandleFunc("/replay/ui.js", serveControlUI) + + return withCORS(mux) +} + +// withCORS lets the control API be called from the explorer's own origin, which is a +// different host and port than this server. +func withCORS(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Access-Control-Max-Age", "86400") + + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + + next.ServeHTTP(w, r) + }) +} + +func serveControlUI(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/javascript; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + + if _, err := w.Write(controlUI); err != nil { + return + } +} + +func (r *Replay) serveClock(w http.ResponseWriter, _ *http.Request) { + now, rate := r.clock.state() + + writeJSON(w, http.StatusOK, map[string]any{ + "time_ms": now.UnixMilli(), + "rate": rate, + }) +} + +func (r *Replay) serveStatus(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, r.Status()) +} + +func (r *Replay) serveCommand(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + writeAPIError(w, http.StatusMethodNotAllowed, "commands are posted") + return + } + + command := Command{} + if err := json.NewDecoder(req.Body).Decode(&command); err != nil { + writeAPIError(w, http.StatusBadRequest, fmt.Sprintf("invalid command: %v", err)) + return + } + + if err := r.Execute(command); err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + + writeJSON(w, http.StatusOK, r.Status()) +} + +// Execute applies a control command. +func (r *Replay) Execute(command Command) error { + switch strings.ToLower(command.Action) { + case "play": + // a speed of 0 means "as fast as upstream allows", which is what the UI sends + // for its `max` setting + r.Play(command.Speed) + + case "speed": + r.SetSpeed(command.Speed) + + case "step": + r.Step(command.Slots) + + case "forward": + return r.Forward(command.Slot, command.Speed) + + case "stop", "pause": + r.Pause() + + case "start", "resume": + r.Resume() + + default: + return fmt.Errorf("unknown action %q", command.Action) + } + + return nil +} + +// notifyStatus pushes the current status to everything watching the control stream. It +// is lossy by design: a status is a snapshot, so a client that fell behind wants the +// newest one rather than the backlog, and the driver must never wait on a browser tab. +func (r *Replay) notifyStatus() { + if r.control.subscriberCount() == 0 { + return + } + + data, err := json.Marshal(r.Status()) + if err != nil { + r.logger.WithError(err).Debug("could not encode replay status") + return + } + + r.control.publish(sseEvent{topic: "status", data: data}) +} diff --git a/replay/control_test.go b/replay/control_test.go new file mode 100644 index 00000000..df8c0d32 --- /dev/null +++ b/replay/control_test.go @@ -0,0 +1,257 @@ +package replay + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func controlServer(t *testing.T, replay *Replay) *httptest.Server { + t.Helper() + + server := httptest.NewServer(replay.controlHandler()) + t.Cleanup(server.Close) + + return server +} + +func postCommand(t *testing.T, server *httptest.Server, body string) (*http.Response, Status) { + t.Helper() + + rsp, err := http.Post(server.URL+"/replay/command", "application/json", strings.NewReader(body)) + require.NoError(t, err) + + t.Cleanup(func() { _ = rsp.Body.Close() }) + + status := Status{} + if rsp.StatusCode == http.StatusOK { + require.NoError(t, json.NewDecoder(rsp.Body).Decode(&status)) + } + + return rsp, status +} + +func TestCommandsDriveTheReplay(t *testing.T) { + tests := []struct { + name string + body string + wantRunning bool + wantSpeed float64 + wantTarget uint64 + }{ + { + name: "play at a speed", + body: `{"action":"play","speed":4}`, + wantRunning: true, + wantSpeed: 4, + }, + { + name: "play at max speed", + body: `{"action":"play","speed":0}`, + wantRunning: true, + wantSpeed: 0, + }, + { + name: "step a number of slots", + body: `{"action":"step","slots":32}`, + wantRunning: true, + wantTarget: 132, + }, + { + name: "forward to a slot", + body: `{"action":"forward","slot":500,"speed":6}`, + wantRunning: true, + wantSpeed: 6, + wantTarget: 500, + }, + { + name: "speed alone does not start the replay", + body: `{"action":"speed","speed":8}`, + wantRunning: false, + wantSpeed: 8, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + rsp, status := postCommand(t, server, test.body) + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, test.wantRunning, status.Running) + require.Equal(t, test.wantSpeed, status.Speed) + require.Equal(t, test.wantTarget, status.TargetSlot) + }) + } +} + +func TestCommandRejectsUnknownAction(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + rsp, _ := postCommand(t, server, `{"action":"detonate"}`) + require.Equal(t, http.StatusBadRequest, rsp.StatusCode) + + rsp, _ = postCommand(t, server, `{"action":"forward","slot":50}`) + require.Equal(t, http.StatusBadRequest, rsp.StatusCode, "forward must refuse to rewind") +} + +func TestResumeKeepsTargetAndSpeed(t *testing.T) { + replay := testConsoleReplay(100) + + require.NoError(t, replay.Forward(500, 4)) + replay.Pause() + replay.Resume() + + status := replay.Status() + require.True(t, status.Running) + require.Equal(t, 4.0, status.Speed, "resuming must keep the speed it was running at") + require.Equal(t, uint64(500), status.TargetSlot, "resuming must keep running towards the target") +} + +func TestResumeDropsAReachedTarget(t *testing.T) { + replay := testConsoleReplay(100) + + replay.Step(1) + replay.virtualSlot = 101 // the driver would have advanced here + replay.Pause() + replay.Resume() + + require.Zero(t, replay.Status().TargetSlot, "a target already reached must not pause the replay again") + require.True(t, replay.Status().Running) +} + +func TestStatusReportsChainContext(t *testing.T) { + replay := testConsoleReplay(100) + replay.cfg.StartSlot = 90 + replay.upstreamSlot = 4000 + replay.chain.genesisTime = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + status := replay.Status() + + require.Equal(t, uint64(90), status.StartSlot) + require.Equal(t, uint64(4000), status.UpstreamSlot) + require.Equal(t, uint64(125), status.UpstreamEpoch) + require.Equal(t, uint64(12000), status.SlotDurationMs) + require.Equal(t, uint64(32), status.SlotsPerEpoch) +} + +func TestControlUIIsServed(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + rsp, err := http.Get(server.URL + "/replay/ui.js") + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Contains(t, rsp.Header.Get("Content-Type"), "javascript") + + body := &bytes.Buffer{} + _, err = body.ReadFrom(rsp.Body) + require.NoError(t, err) + + require.Contains(t, body.String(), "replay-callout", "the side-loaded UI must build its callout") + require.Contains(t, body.String(), "/replay/command", "the side-loaded UI must call the control API") + require.Contains(t, body.String(), "window.doraReplayApi", + "the side-loaded UI must take the control address from the explorer, not from its own script url") + require.Contains(t, body.String(), "window.doraIndexRefreshInterval", + "the side-loaded UI must retune the explorer's polling to the replay's pace") +} + +// TestControlAllowsCrossOriginCalls guards the reason the UI works at all: it is served +// by the replay but runs on the explorer's origin, so the control API has to accept +// cross-origin calls and their preflight. +func TestControlAllowsCrossOriginCalls(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + req, err := http.NewRequest(http.MethodOptions, server.URL+"/replay/command", http.NoBody) + require.NoError(t, err) + + req.Header.Set("Origin", "http://localhost:8083") + req.Header.Set("Access-Control-Request-Method", "POST") + + rsp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusNoContent, rsp.StatusCode) + require.Equal(t, "*", rsp.Header.Get("Access-Control-Allow-Origin")) + require.Contains(t, rsp.Header.Get("Access-Control-Allow-Methods"), "POST") +} + +func TestStatusStreamPushesOnChange(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/replay/events", http.NoBody) + require.NoError(t, err) + + rsp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "text/event-stream", rsp.Header.Get("Content-Type")) + + require.Eventually(t, func() bool { return replay.control.subscriberCount() == 1 }, 5*time.Second, 5*time.Millisecond) + + replay.Play(8) + + reader := bufio.NewReader(rsp.Body) + + eventLine, err := reader.ReadString('\n') + require.NoError(t, err) + require.Equal(t, "event: status", strings.TrimSpace(eventLine)) + + dataLine, err := reader.ReadString('\n') + require.NoError(t, err) + + status := Status{} + require.NoError(t, json.Unmarshal([]byte(strings.TrimPrefix(strings.TrimSpace(dataLine), "data: ")), &status)) + require.True(t, status.Running) + require.Equal(t, 8.0, status.Speed) +} + +// TestStatusStreamNeverBlocksTheDriver is the property that keeps a stalled browser tab +// from stalling the replay: status updates are dropped, not queued. +func TestStatusStreamNeverBlocksTheDriver(t *testing.T) { + replay := testConsoleReplay(100) + + stuck := newEventSubscriber(map[string]bool{"status": true}) + replay.control.add(stuck) + + // fill the subscriber's buffer well past capacity + done := make(chan struct{}) + + go func() { + defer close(done) + + for i := 0; i < 1000; i++ { + replay.notifyStatus() + } + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("notifyStatus blocked on a subscriber that is not reading") + } +} diff --git a/replay/elproxy.go b/replay/elproxy.go new file mode 100644 index 00000000..37ffe599 --- /dev/null +++ b/replay/elproxy.go @@ -0,0 +1,703 @@ +package replay + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// elBlockHistory is how many recent block hashes are kept for block filters. It only +// has to cover the gap between two filter polls. +const elBlockHistory = 4096 + +// blockTagParam maps the JSON-RPC methods that take a block tag to the index of that +// parameter, so `latest` can be pinned to the virtual execution head. +var blockTagParam = map[string]int{ + "eth_getBlockByNumber": 0, + "eth_getBlockTransactionCountByNumber": 0, + "eth_getUncleCountByBlockNumber": 0, + "eth_getBlockReceipts": 0, + "eth_getBalance": 1, + "eth_getCode": 1, + "eth_getTransactionCount": 1, + "eth_call": 1, + "eth_estimateGas": 1, + "eth_createAccessList": 1, + "eth_getStorageAt": 2, + "eth_getProof": 2, +} + +// blockTags are the symbolic block references that must be pinned to the virtual head. +// `finalized` and `safe` are approximated by the head, which is the closest the replay +// can get without tracking execution finality separately. +var blockTags = map[string]bool{ + "latest": true, + "pending": true, + "safe": true, + "finalized": true, +} + +// elBlock is the part of an execution block the replay tracks. +type elBlock struct { + number uint64 + hash string + timestamp time.Time +} + +// elProxy is the fake execution node: a JSON-RPC proxy with a virtual head that follows +// the replayed consensus head, so the explorer never sees a block from the future. +type elProxy struct { + logger logrus.FieldLogger + url string + client *http.Client + + mutex sync.RWMutex + headNumber uint64 + headHash string + blockHashes map[uint64]string + filters map[string]uint64 + filterSerial uint64 + requestID uint64 +} + +var _ http.Handler = (*elProxy)(nil) + +func newELProxy(logger logrus.FieldLogger, url string) *elProxy { + return &elProxy{ + logger: logger, + url: strings.TrimSuffix(url, "/"), + client: newPooledClient(5 * time.Minute), + blockHashes: make(map[uint64]string, elBlockHistory), + filters: make(map[string]uint64), + } +} + +// init positions the virtual execution head at the last block produced at or before the +// replay's start time, found by bisecting the upstream chain. +func (p *elProxy) init(ctx context.Context, startTime time.Time) error { + latest, err := p.blockByTag(ctx, "latest") + if err != nil { + return err + } + + if latest == nil { + return fmt.Errorf("execution upstream has no latest block") + } + + if !latest.timestamp.After(startTime) { + p.setHead(latest) + return nil + } + + low, high := uint64(0), latest.number + + for low < high { + mid := (low + high + 1) / 2 + + block, err := p.blockByNumber(ctx, mid) + if err != nil { + return err + } + + if block == nil || block.timestamp.After(startTime) { + high = mid - 1 + } else { + low = mid + } + } + + block, err := p.blockByNumber(ctx, low) + if err != nil { + return err + } + + if block == nil { + return fmt.Errorf("could not resolve execution block %v", low) + } + + p.setHead(block) + + p.logger.WithFields(logrus.Fields{ + "block": block.number, + "time": block.timestamp.UTC().Format(time.RFC3339), + }).Info("resolved execution head") + + return nil +} + +// advanceTo moves the virtual head forward over every block produced at or before the +// given slot time, and returns the blocks it took in. +func (p *elProxy) advanceTo(ctx context.Context, slotTime time.Time) ([]elBlock, error) { + added := []elBlock{} + + for { + next, err := p.blockByNumber(ctx, p.head()+1) + if err != nil { + return added, err + } + + if next == nil || next.timestamp.After(slotTime) { + return added, nil + } + + p.setHead(next) + added = append(added, *next) + } +} + +func (p *elProxy) head() uint64 { + p.mutex.RLock() + defer p.mutex.RUnlock() + + return p.headNumber +} + +func (p *elProxy) setHead(block *elBlock) { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.headNumber = block.number + p.headHash = block.hash + p.blockHashes[block.number] = block.hash + + if block.number >= elBlockHistory { + delete(p.blockHashes, block.number-elBlockHistory) + } +} + +// -- JSON-RPC ---------------------------------------------------------------------- + +type rpcRequest struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params []json.RawMessage `json:"params"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +func (p *elProxy) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + http.Error(w, "only POST is supported", http.StatusMethodNotAllowed) + return + } + + body, err := io.ReadAll(req.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + http.Error(w, "empty request", http.StatusBadRequest) + return + } + + if trimmed[0] == '[' { + requests := []rpcRequest{} + if err := json.Unmarshal(trimmed, &requests); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + writeJSON(w, http.StatusOK, p.handleBatch(req.Context(), requests)) + + return + } + + request := rpcRequest{} + if err := json.Unmarshal(trimmed, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + writeJSON(w, http.StatusOK, p.handle(req.Context(), &request)) +} + +// handleBatch answers a batched call, forwarding everything it cannot answer locally as +// one batch. Splitting a batch into individual upstream calls would turn a single round +// trip into one per entry, which is exactly what the caller batched to avoid. +func (p *elProxy) handleBatch(ctx context.Context, requests []rpcRequest) []*rpcResponse { + responses := make([]*rpcResponse, len(requests)) + forwarded := make([]*rpcRequest, 0, len(requests)) + positions := make([]int, 0, len(requests)) + + for i := range requests { + local, forward := p.prepare(&requests[i]) + if local != nil { + responses[i] = local + continue + } + + forwarded = append(forwarded, forward) + positions = append(positions, i) + } + + if len(forwarded) == 0 { + return responses + } + + results := p.forwardBatch(ctx, forwarded) + for n, position := range positions { + responses[position] = results[n] + } + + return responses +} + +func (p *elProxy) handle(ctx context.Context, request *rpcRequest) *rpcResponse { + local, forward := p.prepare(request) + if local != nil { + return local + } + + return p.forward(ctx, forward) +} + +// prepare answers a call from the replay's own state where it can, and otherwise +// returns the (possibly rewritten) request to forward upstream. +func (p *elProxy) prepare(request *rpcRequest) (*rpcResponse, *rpcRequest) { + switch request.Method { + case "eth_blockNumber": + return p.reply(request, hexUint(p.head())), nil + + case "eth_syncing": + return p.reply(request, false), nil + + case "eth_newBlockFilter": + return p.reply(request, p.newBlockFilter()), nil + + case "eth_getFilterChanges": + return p.handleFilterChanges(request), nil + + case "eth_uninstallFilter": + return p.handleUninstallFilter(request), nil + + case "eth_getLogs": + return p.clampGetLogs(request) + } + + if index, hasTag := blockTagParam[request.Method]; hasTag { + pinned, visible, err := p.pinBlockTag(request, index) + if err != nil { + return p.replyError(request, -32602, err.Error()), nil + } + + if !visible { + // the requested block is beyond the virtual head; a node that does not + // have it yet answers with no result + return p.reply(request, nil), nil + } + + return nil, pinned + } + + return nil, request +} + +// pinBlockTag rewrites a symbolic block tag to the virtual head and reports whether a +// numeric block reference is at or below it. +func (p *elProxy) pinBlockTag(request *rpcRequest, index int) (*rpcRequest, bool, error) { + if index > len(request.Params) { + return nil, false, fmt.Errorf("%v is missing parameter %v", request.Method, index) + } + + if index == len(request.Params) { + // the tag is optional and was omitted, which means `latest`; appending the + // pinned head keeps the meaning the caller intended + pinned := *request + pinned.Params = append(append([]json.RawMessage{}, request.Params...), quotedHex(p.head())) + + return &pinned, true, nil + } + + raw := request.Params[index] + + tag := "" + if err := json.Unmarshal(raw, &tag); err != nil { + // not a string: eth_getBlockByNumber never sees this, but block-hash objects + // (eth_getProof style) are passed through untouched + return request, true, nil + } + + if blockTags[tag] { + pinned := *request + pinned.Params = append([]json.RawMessage{}, request.Params...) + pinned.Params[index] = quotedHex(p.head()) + + return &pinned, true, nil + } + + if tag == "earliest" || !strings.HasPrefix(tag, "0x") { + return request, true, nil + } + + number, err := strconv.ParseUint(strings.TrimPrefix(tag, "0x"), 16, 64) + if err != nil { + return nil, false, fmt.Errorf("invalid block number %q", tag) + } + + return request, number <= p.head(), nil +} + +func (p *elProxy) newBlockFilter() string { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.filterSerial++ + id := fmt.Sprintf("0x%016x", p.filterSerial) + p.filters[id] = p.headNumber + + return id +} + +// handleFilterChanges returns the hashes of the blocks that entered the virtual chain +// since the filter was last polled. +func (p *elProxy) handleFilterChanges(request *rpcRequest) *rpcResponse { + id, err := stringParam(request, 0) + if err != nil { + return p.replyError(request, -32602, err.Error()) + } + + p.mutex.Lock() + delivered, exists := p.filters[id] + if !exists { + p.mutex.Unlock() + + return p.replyError(request, -32000, "filter not found") + } + + hashes := []string{} + + for number := delivered + 1; number <= p.headNumber; number++ { + if hash, ok := p.blockHashes[number]; ok { + hashes = append(hashes, hash) + } + } + + p.filters[id] = p.headNumber + p.mutex.Unlock() + + return p.reply(request, hashes) +} + +func (p *elProxy) handleUninstallFilter(request *rpcRequest) *rpcResponse { + id, err := stringParam(request, 0) + if err != nil { + return p.replyError(request, -32602, err.Error()) + } + + p.mutex.Lock() + _, exists := p.filters[id] + delete(p.filters, id) + p.mutex.Unlock() + + return p.reply(request, exists) +} + +// clampGetLogs clamps the requested range to the virtual head before forwarding, so a +// scan that asks for `latest` stops where the replay currently stands. +func (p *elProxy) clampGetLogs(request *rpcRequest) (*rpcResponse, *rpcRequest) { + if len(request.Params) == 0 { + return nil, request + } + + filter := map[string]json.RawMessage{} + if err := json.Unmarshal(request.Params[0], &filter); err != nil { + return p.replyError(request, -32602, "invalid log filter"), nil + } + + if _, byHash := filter["blockHash"]; byHash { + return nil, request + } + + head := p.head() + + from, err := blockNumberFromFilter(filter, "fromBlock", 0) + if err != nil { + return p.replyError(request, -32602, err.Error()), nil + } + + if from > head { + return p.reply(request, []any{}), nil + } + + to, err := blockNumberFromFilter(filter, "toBlock", head) + if err != nil { + return p.replyError(request, -32602, err.Error()), nil + } + + if to > head { + to = head + } + + filter["fromBlock"] = quotedHex(from) + filter["toBlock"] = quotedHex(to) + + encoded, err := json.Marshal(filter) + if err != nil { + return p.replyError(request, -32603, err.Error()), nil + } + + clamped := *request + clamped.Params = append([]json.RawMessage{encoded}, request.Params[1:]...) + + return nil, &clamped +} + +// blockNumberFromFilter reads a log filter bound, mapping symbolic tags to fallback. +func blockNumberFromFilter(filter map[string]json.RawMessage, field string, fallback uint64) (uint64, error) { + raw, exists := filter[field] + if !exists { + return fallback, nil + } + + value := "" + if err := json.Unmarshal(raw, &value); err != nil { + return 0, fmt.Errorf("invalid %v in log filter", field) + } + + if value == "earliest" { + return 0, nil + } + + if blockTags[value] { + return fallback, nil + } + + number, err := strconv.ParseUint(strings.TrimPrefix(value, "0x"), 16, 64) + if err != nil { + return 0, fmt.Errorf("invalid %v %q in log filter", field, value) + } + + return number, nil +} + +// forward passes a request to the real execution node and returns its answer verbatim. +func (p *elProxy) forward(ctx context.Context, request *rpcRequest) *rpcResponse { + body, err := p.post(ctx, request) + if err != nil { + return p.replyError(request, -32603, err.Error()) + } + + response := rpcResponse{} + if err := json.Unmarshal(body, &response); err != nil { + return p.replyError(request, -32603, fmt.Sprintf("invalid upstream response: %s", truncate(body, 200))) + } + + response.ID = request.ID + response.JSONRPC = "2.0" + + return &response +} + +// post sends a JSON-RPC payload to the execution upstream and returns the raw answer. +func (p *elProxy) post(ctx context.Context, payload any) ([]byte, error) { + encoded, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.url, bytes.NewReader(encoded)) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + + rsp, err := p.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = rsp.Body.Close() }() + + return io.ReadAll(rsp.Body) +} + +// forwardBatch passes several requests upstream in a single call and puts the answers +// back in the order they were asked, matching by id. +func (p *elProxy) forwardBatch(ctx context.Context, requests []*rpcRequest) []*rpcResponse { + if len(requests) == 1 { + return []*rpcResponse{p.forward(ctx, requests[0])} + } + + body, err := p.post(ctx, requests) + if err != nil { + return p.batchError(requests, err) + } + + upstreamResponses := []*rpcResponse{} + if err := json.Unmarshal(body, &upstreamResponses); err != nil { + return p.batchError(requests, fmt.Errorf("invalid upstream batch response: %s", truncate(body, 200))) + } + + byID := make(map[string]*rpcResponse, len(upstreamResponses)) + for _, response := range upstreamResponses { + byID[string(response.ID)] = response + } + + responses := make([]*rpcResponse, len(requests)) + + for i, request := range requests { + response, matched := byID[string(request.ID)] + if !matched { + if i < len(upstreamResponses) { + // an upstream that does not echo ids is still answering in order + response = upstreamResponses[i] + } else { + responses[i] = p.replyError(request, -32603, "upstream did not answer this batch entry") + continue + } + } + + response.ID = request.ID + response.JSONRPC = "2.0" + responses[i] = response + } + + return responses +} + +func (p *elProxy) batchError(requests []*rpcRequest, err error) []*rpcResponse { + responses := make([]*rpcResponse, len(requests)) + for i, request := range requests { + responses[i] = p.replyError(request, -32603, err.Error()) + } + + return responses +} + +// call issues a request of the replay's own to the execution upstream. +func (p *elProxy) call(ctx context.Context, method string, params ...any) (json.RawMessage, error) { + encoded := make([]json.RawMessage, 0, len(params)) + + for _, param := range params { + raw, err := json.Marshal(param) + if err != nil { + return nil, err + } + + encoded = append(encoded, raw) + } + + p.mutex.Lock() + p.requestID++ + id := p.requestID + p.mutex.Unlock() + + request := &rpcRequest{ + JSONRPC: "2.0", + ID: json.RawMessage(strconv.FormatUint(id, 10)), + Method: method, + Params: encoded, + } + + response := p.forward(ctx, request) + if response.Error != nil { + return nil, fmt.Errorf("%v: %v", method, response.Error.Message) + } + + return response.Result, nil +} + +func (p *elProxy) blockByTag(ctx context.Context, tag string) (*elBlock, error) { + return p.decodeBlock(p.call(ctx, "eth_getBlockByNumber", tag, false)) +} + +func (p *elProxy) blockByNumber(ctx context.Context, number uint64) (*elBlock, error) { + return p.decodeBlock(p.call(ctx, "eth_getBlockByNumber", fmt.Sprintf("0x%x", number), false)) +} + +func (p *elProxy) decodeBlock(result json.RawMessage, err error) (*elBlock, error) { + if err != nil { + return nil, err + } + + if len(result) == 0 || string(result) == "null" { + return nil, nil + } + + header := struct { + Number string `json:"number"` + Hash string `json:"hash"` + Timestamp string `json:"timestamp"` + }{} + + if err := json.Unmarshal(result, &header); err != nil { + return nil, fmt.Errorf("error parsing execution block: %w", err) + } + + number, err := strconv.ParseUint(strings.TrimPrefix(header.Number, "0x"), 16, 64) + if err != nil { + return nil, fmt.Errorf("invalid block number %q: %w", header.Number, err) + } + + timestamp, err := strconv.ParseInt(strings.TrimPrefix(header.Timestamp, "0x"), 16, 64) + if err != nil { + return nil, fmt.Errorf("invalid block timestamp %q: %w", header.Timestamp, err) + } + + return &elBlock{ + number: number, + hash: header.Hash, + timestamp: time.Unix(timestamp, 0).UTC(), + }, nil +} + +func (p *elProxy) reply(request *rpcRequest, result any) *rpcResponse { + encoded, err := json.Marshal(result) + if err != nil { + return p.replyError(request, -32603, err.Error()) + } + + return &rpcResponse{JSONRPC: "2.0", ID: request.ID, Result: encoded} +} + +func (p *elProxy) replyError(request *rpcRequest, code int, message string) *rpcResponse { + return &rpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + Error: &rpcError{Code: code, Message: message}, + } +} + +func stringParam(request *rpcRequest, index int) (string, error) { + if index >= len(request.Params) { + return "", fmt.Errorf("missing parameter %v", index) + } + + value := "" + if err := json.Unmarshal(request.Params[index], &value); err != nil { + return "", fmt.Errorf("parameter %v is not a string", index) + } + + return value, nil +} + +func hexUint(value uint64) string { + return fmt.Sprintf("0x%x", value) +} + +func quotedHex(value uint64) json.RawMessage { + return json.RawMessage(fmt.Sprintf("%q", hexUint(value))) +} diff --git a/replay/elproxy_test.go b/replay/elproxy_test.go new file mode 100644 index 00000000..df4b6641 --- /dev/null +++ b/replay/elproxy_test.go @@ -0,0 +1,277 @@ +package replay + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func testELProxy(head uint64) *elProxy { + proxy := newELProxy(logrus.New(), "http://localhost:0") + proxy.setHead(&elBlock{number: head, hash: "0xdeadbeef"}) + + return proxy +} + +func rawParams(t *testing.T, values ...any) []json.RawMessage { + t.Helper() + + params := make([]json.RawMessage, 0, len(values)) + + for _, value := range values { + encoded, err := json.Marshal(value) + require.NoError(t, err) + + params = append(params, encoded) + } + + return params +} + +func TestPinBlockTag(t *testing.T) { + proxy := testELProxy(500) + + tests := []struct { + name string + method string + params []any + wantParam string + wantVisible bool + }{ + { + name: "latest is pinned to the virtual head", + method: "eth_getBlockByNumber", + params: []any{"latest", false}, + wantParam: "0x1f4", + wantVisible: true, + }, + { + name: "finalized is approximated by the virtual head", + method: "eth_getBlockByNumber", + params: []any{"finalized", false}, + wantParam: "0x1f4", + wantVisible: true, + }, + { + name: "a block at the head stays visible", + method: "eth_getBlockByNumber", + params: []any{"0x1f4", false}, + wantParam: "0x1f4", + wantVisible: true, + }, + { + name: "a block beyond the head is hidden", + method: "eth_getBlockByNumber", + params: []any{"0x1f5", false}, + wantVisible: false, + }, + { + name: "earliest is left alone", + method: "eth_getBlockByNumber", + params: []any{"earliest", false}, + wantParam: "earliest", + wantVisible: true, + }, + { + name: "an omitted tag is appended as the head", + method: "eth_getBalance", + params: []any{"0x0000000000000000000000000000000000000001"}, + wantParam: "0x1f4", + wantVisible: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := &rpcRequest{Method: test.method, Params: rawParams(t, test.params...)} + index := blockTagParam[test.method] + + pinned, visible, err := proxy.pinBlockTag(request, index) + require.NoError(t, err) + require.Equal(t, test.wantVisible, visible) + + if !visible { + return + } + + value := "" + require.NoError(t, json.Unmarshal(pinned.Params[index], &value)) + require.Equal(t, test.wantParam, value) + }) + } +} + +func TestBlockNumberFromFilter(t *testing.T) { + filter := map[string]json.RawMessage{ + "fromBlock": json.RawMessage(`"0x10"`), + "toBlock": json.RawMessage(`"latest"`), + } + + from, err := blockNumberFromFilter(filter, "fromBlock", 0) + require.NoError(t, err) + require.Equal(t, uint64(16), from) + + to, err := blockNumberFromFilter(filter, "toBlock", 999) + require.NoError(t, err) + require.Equal(t, uint64(999), to) + + missing, err := blockNumberFromFilter(filter, "blockHash", 42) + require.NoError(t, err) + require.Equal(t, uint64(42), missing) + + _, err = blockNumberFromFilter(map[string]json.RawMessage{ + "fromBlock": json.RawMessage(`"zzz"`), + }, "fromBlock", 0) + require.Error(t, err) +} + +func TestBlockFilterDeliversNewBlocks(t *testing.T) { + proxy := testELProxy(100) + + filterID := proxy.newBlockFilter() + + // nothing has been produced since the filter was created + response := proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, filterID)}) + require.Nil(t, response.Error) + require.JSONEq(t, `[]`, string(response.Result)) + + proxy.setHead(&elBlock{number: 101, hash: "0xaa"}) + proxy.setHead(&elBlock{number: 102, hash: "0xbb"}) + + response = proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, filterID)}) + require.Nil(t, response.Error) + require.JSONEq(t, `["0xaa","0xbb"]`, string(response.Result)) + + // a second poll without new blocks returns nothing again + response = proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, filterID)}) + require.JSONEq(t, `[]`, string(response.Result)) + + // an unknown filter is reported the way a node reports an expired one + response = proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, "0xdead")}) + require.NotNil(t, response.Error) + require.Contains(t, response.Error.Message, "filter not found") +} + +func TestClampGetLogsToVirtualHead(t *testing.T) { + proxy := testELProxy(500) + + tests := []struct { + name string + filter string + wantFrom string + wantTo string + wantLocal string + }{ + { + name: "latest is clamped to the virtual head", + filter: `{"fromBlock":"0x10","toBlock":"latest"}`, + wantFrom: `"0x10"`, + wantTo: `"0x1f4"`, + }, + { + name: "a range beyond the head is truncated", + filter: `{"fromBlock":"0x10","toBlock":"0x999"}`, + wantFrom: `"0x10"`, + wantTo: `"0x1f4"`, + }, + { + name: "a missing toBlock defaults to the head", + filter: `{"fromBlock":"0x10"}`, + wantFrom: `"0x10"`, + wantTo: `"0x1f4"`, + }, + { + name: "a range entirely beyond the head yields no logs", + filter: `{"fromBlock":"0x500","toBlock":"0x600"}`, + wantLocal: `[]`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := &rpcRequest{ + Method: "eth_getLogs", + Params: []json.RawMessage{json.RawMessage(test.filter)}, + } + + local, forward := proxy.clampGetLogs(request) + + if test.wantLocal != "" { + require.NotNil(t, local) + require.JSONEq(t, test.wantLocal, string(local.Result)) + + return + } + + require.Nil(t, local) + require.NotNil(t, forward) + + clamped := map[string]json.RawMessage{} + require.NoError(t, json.Unmarshal(forward.Params[0], &clamped)) + require.Equal(t, test.wantFrom, string(clamped["fromBlock"])) + require.Equal(t, test.wantTo, string(clamped["toBlock"])) + }) + } +} + +// TestBatchIsForwardedAsOneCall guards the property that makes batching worth anything: +// a batched call must reach the upstream as a single request, not as one per entry. +func TestBatchIsForwardedAsOneCall(t *testing.T) { + upstreamCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamCalls++ + + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + + requests := []rpcRequest{} + require.NoError(t, json.Unmarshal(body, &requests)) + + responses := make([]rpcResponse, 0, len(requests)) + for _, request := range requests { + responses = append(responses, rpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + Result: json.RawMessage(`"ok"`), + }) + } + + writeJSON(w, http.StatusOK, responses) + })) + defer server.Close() + + proxy := newELProxy(logrus.New(), server.URL) + proxy.setHead(&elBlock{number: 500, hash: "0xdead"}) + + batch := []rpcRequest{ + {JSONRPC: "2.0", ID: json.RawMessage(`1`), Method: "eth_getTransactionReceipt", Params: rawParams(t, "0xaa")}, + {JSONRPC: "2.0", ID: json.RawMessage(`2`), Method: "eth_getTransactionReceipt", Params: rawParams(t, "0xbb")}, + {JSONRPC: "2.0", ID: json.RawMessage(`3`), Method: "eth_blockNumber"}, + } + + responses := proxy.handleBatch(context.Background(), batch) + + require.Len(t, responses, 3) + require.Equal(t, 1, upstreamCalls, "the two forwarded entries must share one upstream call") + require.JSONEq(t, `"ok"`, string(responses[0].Result)) + require.JSONEq(t, `"ok"`, string(responses[1].Result)) + require.JSONEq(t, `"0x1f4"`, string(responses[2].Result), "eth_blockNumber is answered locally") +} + +func TestUninstallFilter(t *testing.T) { + proxy := testELProxy(100) + filterID := proxy.newBlockFilter() + + response := proxy.handleUninstallFilter(&rpcRequest{Params: rawParams(t, filterID)}) + require.JSONEq(t, `true`, string(response.Result)) + + response = proxy.handleUninstallFilter(&rpcRequest{Params: rawParams(t, filterID)}) + require.JSONEq(t, `false`, string(response.Result)) +} diff --git a/replay/events.go b/replay/events.go new file mode 100644 index 00000000..3f215d39 --- /dev/null +++ b/replay/events.go @@ -0,0 +1,181 @@ +package replay + +import ( + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// sseEvent is one entry of the beacon node event stream. +type sseEvent struct { + topic string + data []byte +} + +// eventHub is the fake node's /eth/v1/events endpoint: a fan-out of synthesized chain +// events to every connected subscriber, filtered by the topics it asked for. +type eventHub struct { + logger logrus.FieldLogger + + // lossy hubs drop an event for a subscriber whose buffer is full instead of + // waiting for it. Chain events must never be lost, but status snapshots should + // never hold the replay up either. + lossy bool + + mutex sync.Mutex + subscribers map[*eventSubscriber]struct{} +} + +type eventSubscriber struct { + topics map[string]bool + events chan sseEvent + + // done is closed when the subscriber disconnects, so a publisher waiting on a + // slow reader is released instead of blocking forever. + done chan struct{} +} + +func newEventSubscriber(topics map[string]bool) *eventSubscriber { + return &eventSubscriber{ + topics: topics, + events: make(chan sseEvent, 256), + done: make(chan struct{}), + } +} + +func newEventHub(logger logrus.FieldLogger) *eventHub { + return &eventHub{ + logger: logger, + subscribers: make(map[*eventSubscriber]struct{}), + } +} + +// newLossyEventHub returns a hub that never blocks its publisher. +func newLossyEventHub(logger logrus.FieldLogger) *eventHub { + hub := newEventHub(logger) + hub.lossy = true + + return hub +} + +// publish delivers an event to every subscriber that asked for its topic. It waits for +// a subscriber that is behind rather than dropping the event: the replay sets the pace, +// so a slow explorer should slow the replay down, not silently lose a block. This is +// what makes "advance as fast as upstream allows" self-throttle to what the explorer +// can actually index. +func (h *eventHub) publish(event sseEvent) { + h.mutex.Lock() + targets := make([]*eventSubscriber, 0, len(h.subscribers)) + + for sub := range h.subscribers { + if sub.topics[event.topic] { + targets = append(targets, sub) + } + } + h.mutex.Unlock() + + for _, sub := range targets { + if h.lossy { + select { + case sub.events <- event: + default: + } + + continue + } + + select { + case sub.events <- event: + case <-sub.done: + h.logger.Debugf("subscriber disconnected while waiting to take a %v event", event.topic) + } + } +} + +// subscriberCount reports how many event streams are currently connected. +func (h *eventHub) subscriberCount() int { + h.mutex.Lock() + defer h.mutex.Unlock() + + return len(h.subscribers) +} + +func (h *eventHub) add(sub *eventSubscriber) { + h.mutex.Lock() + defer h.mutex.Unlock() + + h.subscribers[sub] = struct{}{} +} + +func (h *eventHub) remove(sub *eventSubscriber) { + h.mutex.Lock() + delete(h.subscribers, sub) + h.mutex.Unlock() + + close(sub.done) +} + +// serveHTTP implements the /eth/v1/events endpoint. Every requested topic is accepted; +// topics the replay cannot synthesize (inclusion lists, fast confirmations) simply +// never fire, which clients already tolerate. +func (h *eventHub) serveHTTP(w http.ResponseWriter, r *http.Request) { + flusher, canFlush := w.(http.Flusher) + if !canFlush { + writeAPIError(w, http.StatusInternalServerError, "streaming not supported") + return + } + + topics := map[string]bool{} + for _, topic := range strings.Split(r.URL.Query().Get("topics"), ",") { + if topic = strings.TrimSpace(topic); topic != "" { + topics[topic] = true + } + } + + if len(topics) == 0 { + if !h.lossy { + writeAPIError(w, http.StatusBadRequest, "no topics requested") + return + } + + // the control stream carries one topic, so asking for it is optional + topics["status"] = true + } + + sub := newEventSubscriber(topics) + + h.add(sub) + defer h.remove(sub) + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flusher.Flush() + + keepalive := time.NewTicker(15 * time.Second) + defer keepalive.Stop() + + for { + select { + case <-r.Context().Done(): + return + case <-keepalive.C: + if _, err := fmt.Fprint(w, ":keepalive\n\n"); err != nil { + return + } + + flusher.Flush() + case event := <-sub.events: + if _, err := fmt.Fprintf(w, "event: %s\ndata: %s\n\n", event.topic, event.data); err != nil { + return + } + + flusher.Flush() + } + } +} diff --git a/replay/events_test.go b/replay/events_test.go new file mode 100644 index 00000000..176de0bc --- /dev/null +++ b/replay/events_test.go @@ -0,0 +1,78 @@ +package replay + +import ( + "bufio" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestEventHubFiltersByTopic(t *testing.T) { + hub := newEventHub(logrus.New()) + + blocks := newEventSubscriber(map[string]bool{"block": true}) + heads := newEventSubscriber(map[string]bool{"head": true}) + + hub.add(blocks) + hub.add(heads) + + require.Equal(t, 2, hub.subscriberCount()) + + hub.publish(sseEvent{topic: "block", data: []byte(`{"slot":"1"}`)}) + + require.Len(t, blocks.events, 1) + require.Empty(t, heads.events, "a subscriber must not receive topics it did not ask for") + + hub.remove(blocks) + require.Equal(t, 1, hub.subscriberCount()) +} + +func TestEventStreamServesSSE(t *testing.T) { + hub := newEventHub(logrus.New()) + + server := httptest.NewServer(http.HandlerFunc(hub.serveHTTP)) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"?topics=head,block", http.NoBody) + require.NoError(t, err) + + rsp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "text/event-stream", rsp.Header.Get("Content-Type")) + + require.Eventually(t, func() bool { return hub.subscriberCount() == 1 }, 5*time.Second, 5*time.Millisecond) + + hub.publish(sseEvent{topic: "head", data: []byte(`{"slot":"42"}`)}) + + reader := bufio.NewReader(rsp.Body) + + eventLine, err := reader.ReadString('\n') + require.NoError(t, err) + require.Equal(t, "event: head", strings.TrimSpace(eventLine)) + + dataLine, err := reader.ReadString('\n') + require.NoError(t, err) + require.Equal(t, `data: {"slot":"42"}`, strings.TrimSpace(dataLine)) +} + +func TestEventStreamRejectsEmptyTopics(t *testing.T) { + hub := newEventHub(logrus.New()) + + recorder := httptest.NewRecorder() + hub.serveHTTP(recorder, httptest.NewRequest(http.MethodGet, "/eth/v1/events", http.NoBody)) + + require.Equal(t, http.StatusBadRequest, recorder.Code) +} diff --git a/replay/example-config.yaml b/replay/example-config.yaml new file mode 100644 index 00000000..2ed94b15 --- /dev/null +++ b/replay/example-config.yaml @@ -0,0 +1,96 @@ +# Example explorer config for a dora-replay run. Copy it to config-.yaml, point +# the endpoints at your dora-replay listeners, and start the explorer with it. +logging: + outputLevel: "info" + +chain: + displayName: "devnet (replay)" + +server: + host: "0.0.0.0" + port: "8083" + +frontend: + enabled: true + debug: true + pprof: true + minimize: false + + siteName: "Dora the Explorer" + siteSubtitle: "replay" + + ethExplorerLink: "" + + validatorNamesInventory: "https://config.glamsterdam-devnet-8.ethpandaops.io/api/v1/nodes/validator-ranges" + + publicRpcUrl: "" + showSubmitDeposit: true + showSubmitElRequests: true + showValidatorSummary: true + + # the replay steps slot by slot, so cached pages would hide what just changed + disablePageCache: true + + tracoorUrl: "https://tracoor.glamsterdam-devnet-8.ethpandaops.io" + tracoorNetwork: "glamsterdam-devnet-8" + +api: + enabled: true + +# drive every "now" in the explorer off the dora-replay control server instead of the +# real wall clock. the endpoints below must point at the matching dora-replay proxies. +replay: + enabled: true + controlUrl: "http://127.0.0.1:15000" + +beaconapi: + endpoints: + - url: "http://127.0.0.1:15052" + name: "replay-cl" + localCacheSize: 100 + +executionapi: + endpoints: + - url: "http://127.0.0.1:15545" + name: "replay-el" + logBatchSize: 1000 + genesisConfig: "https://config.glamsterdam-devnet-8.ethpandaops.io/el/genesis.json" + +mevIndexer: + relays: [] + refreshInterval: 5m + +indexer: + inMemoryEpochs: 6 + + # the replay starts mid-chain, so there is no history to walk back to: without this + # the synchronizer would try to backfill from epoch 0, and the upstreams no longer + # have the states to do it with. + # + # to repair a hole (e.g. after the explorer was restarted mid-finalization), flip + # this to false and add `resyncFromEpoch: ` for one run: the + # synchronizer then walks from there up to the live finalized epoch and stops. + disableSynchronizer: true + + pubkeyCachePath: "./temp/pubkeys-glamsterdam8-replay.db" + + stateCache: + enabled: true + path: "./temp/statecache-glamsterdam8-replay" + +executionIndexer: + enabled: true + retention: 336h + detailsEnabled: false + tracesEnabled: false + +database: + engine: "sqlite" + sqlite: + file: "tmp-db-glamsterdam8-replay.sqlite" + +blockDb: + engine: "pebble" + pebble: + path: "./temp/blockdb-glamsterdam8-replay.peb" + cacheSize: 100 diff --git a/replay/replay.go b/replay/replay.go new file mode 100644 index 00000000..0524587a --- /dev/null +++ b/replay/replay.go @@ -0,0 +1,843 @@ +package replay + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "sync" + "time" + + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" +) + +const ( + // headSearchDepth bounds how far back the replay looks for the newest block at or + // before the start slot before giving up. + headSearchDepth = 256 + + // finalityFallbackEpochs bounds how far back the replay looks for a state it can + // read finality from when the head state itself has been pruned upstream. + finalityFallbackEpochs = 8 + + // upstreamPollInterval is how often the head of the real chain is re-read, purely + // so the UI can show how far the replay still has to go. + upstreamPollInterval = 30 * time.Second +) + +// Replay steps a past slot range through a fake beacon/execution node pair, driving a +// virtual clock so an explorer pointed at it sees the range unfold as if it were live. +type Replay struct { + logger logrus.FieldLogger + cfg Config + chain *chainInfo + upstream *upstream + events *eventHub + control *eventHub + clock *clock + el *elProxy + states *stateLoads + + // wake nudges the driver whenever the drive state changed. + wake chan struct{} + + mutex sync.RWMutex + virtualSlot uint64 + head *blockHeader + finality *finalityCheckpoints + running bool + speed float64 + target uint64 + upstreamSlot uint64 + upstreamAtUTC time.Time + + servers []*http.Server +} + +// New prepares a replay: it reads genesis and the timing specs from upstream, resolves +// the head at the start slot and positions the virtual clock there. +func New(ctx context.Context, logger logrus.FieldLogger, cfg Config) (*Replay, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + + up, err := newUpstream(logger.WithField("module", "upstream"), &cfg) + if err != nil { + return nil, err + } + + chain, err := loadChainInfo(ctx, up) + if err != nil { + return nil, err + } + + up.chain = chain + + logger.WithFields(logrus.Fields{ + "genesis": chain.genesisTime.Format(time.RFC3339), + "slotDuration": chain.slotDuration, + "slotsPerEpoch": chain.slotsPerEpoch, + }).Info("loaded chain info from upstream") + + replay := &Replay{ + logger: logger, + cfg: cfg, + chain: chain, + upstream: up, + events: newEventHub(logger.WithField("module", "events")), + control: newLossyEventHub(logger.WithField("module", "control")), + states: newStateLoads(), + clock: newClock(chain.slotTime(cfg.StartSlot).Add(chain.payloadOffset())), + wake: make(chan struct{}, 1), + virtualSlot: cfg.StartSlot, + } + + if cfg.ExecutionURL != "" { + replay.el = newELProxy(logger.WithField("module", "el"), cfg.ExecutionURL) + } + + if err := replay.initHead(ctx); err != nil { + return nil, err + } + + if replay.el != nil { + if err := replay.el.init(ctx, chain.slotTime(cfg.StartSlot)); err != nil { + return nil, fmt.Errorf("error initializing execution head: %w", err) + } + } + + return replay, nil +} + +// blockOffset is how far into a slot the block and head events are replayed, and +// payloadOffset when the payload becomes available. Both approximate real node timing. +func (c *chainInfo) blockOffset() time.Duration { + return c.slotDuration / 3 +} + +func (c *chainInfo) payloadOffset() time.Duration { + return c.slotDuration * 2 / 3 +} + +// stateGateOffset is how far into a slot the replay checks whether the explorer is +// still loading a beacon state. +func (c *chainInfo) stateGateOffset() time.Duration { + return c.slotDuration / 2 +} + +// initHead walks back from the start slot to the newest block, which becomes the head +// the fake node reports before the first step. +func (r *Replay) initHead(ctx context.Context) error { + slot := r.cfg.StartSlot + + for depth := 0; depth < headSearchDepth; depth++ { + header, err := r.upstream.headerBySlot(ctx, slot) + if err != nil { + return fmt.Errorf("error resolving head at slot %v: %w", slot, err) + } + + if header != nil { + r.mutex.Lock() + r.head = header + r.mutex.Unlock() + + if err := r.refreshFinality(ctx, header); err != nil { + return err + } + + r.logger.WithFields(logrus.Fields{ + "slot": header.Slot, + "root": header.Root, + "finalized": r.finality.FinalizedEpoch, + }).Info("resolved replay head") + + return nil + } + + if slot == 0 { + break + } + + slot-- + } + + return fmt.Errorf("no block found within %v slots before slot %v", headSearchDepth, r.cfg.StartSlot) +} + +// Start brings up the fake nodes and the control server and starts the driver. +func (r *Replay) Start(ctx context.Context) error { + if err := r.listen(ctx, r.cfg.CLListen, r.clHandler(), "consensus"); err != nil { + return err + } + + if r.el != nil { + if err := r.listen(ctx, r.cfg.ELListen, r.el, "execution"); err != nil { + return err + } + } + + if err := r.listen(ctx, r.cfg.ControlListen, r.controlHandler(), "control"); err != nil { + return err + } + + go r.runDriver(ctx) + go r.runUpstreamPoller(ctx) + + if r.cfg.Speed > 0 { + r.Play(r.cfg.Speed) + } + + return nil +} + +func (r *Replay) listen(ctx context.Context, addr string, handler http.Handler, name string) error { + listener, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("could not listen on %v for the %v endpoint: %w", addr, name, err) + } + + server := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 30 * time.Second, + } + + r.servers = append(r.servers, server) + + go func() { + if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { + r.logger.WithError(err).Errorf("%v endpoint stopped", name) + } + }() + + go func() { + <-ctx.Done() + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + r.logger.WithError(err).Debugf("error shutting down %v endpoint", name) + } + }() + + r.logger.Infof("%v endpoint listening on %v", name, addr) + + return nil +} + +// Stop shuts the fake nodes down. +func (r *Replay) Stop() error { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, server := range r.servers { + if err := server.Shutdown(shutdownCtx); err != nil { + return err + } + } + + return nil +} + +// -- drive state ------------------------------------------------------------------ + +// Play runs the replay continuously at the given multiple of real time. +func (r *Replay) Play(speed float64) { + r.mutex.Lock() + r.running = true + r.speed = speed + r.target = 0 + r.mutex.Unlock() + + r.clock.setRate(speed) + r.nudge() + r.notifyStatus() +} + +// Step advances a fixed number of slots as fast as upstream allows, then pauses. +func (r *Replay) Step(slots uint64) { + if slots == 0 { + slots = 1 + } + + r.mutex.Lock() + r.running = true + r.speed = 0 + r.target = r.virtualSlot + slots + r.mutex.Unlock() + + r.clock.setRate(0) + r.nudge() + r.notifyStatus() +} + +// Forward runs to a slot, emitting every slot on the way, then pauses. A speed of 0 +// advances as fast as upstream allows; anything else plays at that multiple of real +// time, which gives the explorer a predictable amount of time per slot to keep up. +func (r *Replay) Forward(slot uint64, speed float64) error { + r.mutex.Lock() + if slot <= r.virtualSlot { + current := r.virtualSlot + r.mutex.Unlock() + + return fmt.Errorf("slot %v is not ahead of the current head slot %v (the replay cannot rewind)", slot, current) + } + + r.running = true + r.speed = speed + r.target = slot + r.mutex.Unlock() + + r.clock.setRate(speed) + r.nudge() + r.notifyStatus() + + return nil +} + +// Pause freezes the replay and the virtual clock where they are. +func (r *Replay) Pause() { + r.mutex.Lock() + r.running = false + r.mutex.Unlock() + + r.clock.setRate(0) + r.nudge() + r.notifyStatus() +} + +// Resume continues what the replay was doing before it was paused, keeping both the +// speed and any target it was running towards. A target that has already been reached +// is dropped, so resuming after a completed step runs on rather than pausing again. +func (r *Replay) Resume() { + r.mutex.Lock() + r.running = true + if r.target != 0 && r.target <= r.virtualSlot { + r.target = 0 + } + speed := r.speed + r.mutex.Unlock() + + r.clock.setRate(speed) + r.nudge() + r.notifyStatus() +} + +// SetSpeed changes the playback rate without starting or stopping the replay. +func (r *Replay) SetSpeed(speed float64) { + if speed < 0 { + speed = 0 + } + + r.mutex.Lock() + r.speed = speed + running := r.running + r.mutex.Unlock() + + if running { + r.clock.setRate(speed) + } + + r.nudge() + r.notifyStatus() +} + +func (r *Replay) nudge() { + select { + case r.wake <- struct{}{}: + default: + } +} + +func (r *Replay) isRunning() bool { + r.mutex.RLock() + defer r.mutex.RUnlock() + + return r.running +} + +// Status is a snapshot of the replay for the console, the control endpoint and the +// explorer's own replay UI. +type Status struct { + VirtualSlot uint64 `json:"virtual_slot"` + VirtualEpoch uint64 `json:"virtual_epoch"` + VirtualTime time.Time `json:"virtual_time"` + + HeadSlot uint64 `json:"head_slot"` + HeadRoot string `json:"head_root"` + FinalizedEpoch uint64 `json:"finalized_epoch"` + JustifiedEpoch uint64 `json:"justified_epoch"` + ExecutionBlock uint64 `json:"execution_block"` + + Running bool `json:"running"` + Speed float64 `json:"speed"` + Rate float64 `json:"rate"` + StartSlot uint64 `json:"start_slot"` + TargetSlot uint64 `json:"target_slot"` + + // StateLoads is how many beacon states the explorer is pulling right now, and + // Holding says the replay's clock is frozen waiting for them. + StateLoads int `json:"state_loads"` + Holding bool `json:"holding"` + + // UpstreamSlot is the head of the real chain, so the UI can show how much of it is + // still ahead of the replay. + UpstreamSlot uint64 `json:"upstream_slot"` + UpstreamEpoch uint64 `json:"upstream_epoch"` + UpstreamSeen time.Time `json:"upstream_seen"` + + GenesisTime time.Time `json:"genesis_time"` + SlotDurationMs uint64 `json:"slot_duration_ms"` + SlotsPerEpoch uint64 `json:"slots_per_epoch"` + + Subscribers int `json:"event_subscribers"` + Upstream string `json:"upstream"` + Tracoor bool `json:"tracoor"` +} + +// Status returns a snapshot of where the replay currently stands. +func (r *Replay) Status() Status { + r.mutex.RLock() + defer r.mutex.RUnlock() + + virtualTime, rate := r.clock.state() + + status := Status{ + VirtualSlot: r.virtualSlot, + VirtualEpoch: r.chain.epochOf(r.virtualSlot), + VirtualTime: virtualTime.UTC(), + Running: r.running, + Speed: r.speed, + Rate: rate, + StartSlot: r.cfg.StartSlot, + TargetSlot: r.target, + UpstreamSlot: r.upstreamSlot, + UpstreamEpoch: r.chain.epochOf(r.upstreamSlot), + UpstreamSeen: r.upstreamAtUTC, + GenesisTime: r.chain.genesisTime, + SlotDurationMs: uint64(r.chain.slotDuration / time.Millisecond), + SlotsPerEpoch: r.chain.slotsPerEpoch, + StateLoads: r.states.count(), + Holding: r.clock.isHeld(), + Subscribers: r.events.subscriberCount(), + Upstream: redactURL(r.cfg.UpstreamURL), + Tracoor: r.upstream.tracoor != nil, + } + + if r.head != nil { + status.HeadSlot = r.head.Slot + status.HeadRoot = r.head.Root + } + + if r.finality != nil { + status.FinalizedEpoch = r.finality.FinalizedEpoch + status.JustifiedEpoch = r.finality.JustifiedEpoch + } + + if r.el != nil { + status.ExecutionBlock = r.el.head() + } + + return status +} + +// -- driver ----------------------------------------------------------------------- + +func (r *Replay) runDriver(ctx context.Context) { + for { + if !r.isRunning() { + if err := r.awaitResume(ctx); err != nil { + return + } + + continue + } + + r.mutex.RLock() + next := r.virtualSlot + 1 + target := r.target + r.mutex.RUnlock() + + if target != 0 && next > target { + r.logger.Infof("reached slot %v, pausing", target) + r.Pause() + + continue + } + + if err := r.advanceSlot(ctx, next); err != nil { + if ctx.Err() != nil { + return + } + + r.logger.WithError(err).Errorf("failed advancing to slot %v, pausing", next) + r.Pause() + } + } +} + +// runUpstreamPoller keeps track of where the real chain currently is, which is what the +// replay is progressing through. It is only informational, so a failed read is ignored. +func (r *Replay) runUpstreamPoller(ctx context.Context) { + ticker := time.NewTicker(upstreamPollInterval) + defer ticker.Stop() + + for { + if header, err := r.upstream.headerByRoot(ctx, "head"); err == nil && header != nil { + r.mutex.Lock() + changed := r.upstreamSlot != header.Slot + r.upstreamSlot = header.Slot + r.upstreamAtUTC = time.Now().UTC() + r.mutex.Unlock() + + if changed { + r.notifyStatus() + } + } + + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (r *Replay) awaitResume(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.wake: + if r.isRunning() { + return nil + } + } + } +} + +// advanceSlot moves the replay to the given slot in three phases, mirroring how a real +// node experiences a slot: the boundary, the block, and the payload becoming available. +func (r *Replay) advanceSlot(ctx context.Context, slot uint64) error { + boundary := r.chain.slotTime(slot) + + if err := r.waitForVirtual(ctx, boundary); err != nil { + return err + } + + r.mutex.Lock() + r.virtualSlot = slot + r.mutex.Unlock() + + header, err := r.upstream.headerBySlot(ctx, slot) + if err != nil { + return fmt.Errorf("error fetching header: %w", err) + } + + if err := r.waitForVirtual(ctx, boundary.Add(r.chain.blockOffset())); err != nil { + return err + } + + if header != nil { + if err := r.emitBlock(ctx, header); err != nil { + return err + } + } + + // halfway through the slot, wait for any beacon state the explorer is loading. The + // clock is frozen while waiting, so a slow state read costs real time but no + // virtual time: without this the replay would run on and the explorer would be + // several slots behind the moment the state finally arrived. + if err := r.waitForVirtual(ctx, boundary.Add(r.chain.stateGateOffset())); err != nil { + return err + } + + if err := r.awaitStateLoads(ctx); err != nil { + return err + } + + if err := r.waitForVirtual(ctx, boundary.Add(r.chain.payloadOffset())); err != nil { + return err + } + + if err := r.advanceExecution(ctx, slot, boundary, header); err != nil { + return err + } + + r.notifyStatus() + + if !r.isPlaying() { + if err := r.settle(ctx); err != nil { + return err + } + } + + return nil +} + +// awaitStateLoads holds the replay until the explorer has finished loading the beacon +// states it asked for. It gives up after StateHoldTimeout so a stuck read cannot wedge +// the replay for good. +func (r *Replay) awaitStateLoads(ctx context.Context) error { + idle := r.states.idleChan() + + select { + case <-idle: + return nil + default: + } + + r.clock.hold() + r.notifyStatus() + + defer func() { + r.clock.release() + r.notifyStatus() + }() + + started := time.Now() + pending := r.states.count() + + select { + case <-ctx.Done(): + return ctx.Err() + + case <-idle: + r.logger.Debugf("held %v for %v state load(s)", time.Since(started).Round(time.Millisecond), pending) + + case <-time.After(r.cfg.StateHoldTimeout): + r.logger.Warnf("%v state load(s) did not finish within %v, continuing", pending, r.cfg.StateHoldTimeout) + } + + return nil +} + +func (r *Replay) isPlaying() bool { + r.mutex.RLock() + defer r.mutex.RUnlock() + + return r.running && r.speed > 0 +} + +func (r *Replay) settle(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(r.cfg.StepSettle): + return nil + } +} + +// waitForVirtual blocks until the virtual clock reaches a point in time. While playing +// that means sleeping the corresponding amount of real time; while stepping the clock +// simply jumps there, and while paused it waits for the replay to be resumed. +func (r *Replay) waitForVirtual(ctx context.Context, target time.Time) error { + for { + if !r.isRunning() { + if err := r.awaitResume(ctx); err != nil { + return err + } + + continue + } + + if !r.isPlaying() { + if r.clock.now().Before(target) { + r.clock.set(target) + } + + return nil + } + + delay := r.clock.realDelayUntil(target) + if delay <= 0 { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + case <-r.wake: + } + } +} + +// emitBlock publishes the events a real node emits for a new block, after making the +// block the head so that anything the explorer requests in response already resolves. +func (r *Replay) emitBlock(ctx context.Context, header *blockHeader) error { + if r.cfg.EmitBids && r.chain.bidsActiveAt(header.Slot) { + bid, err := r.upstream.payloadBid(ctx, header.Root) + if err != nil { + r.logger.WithError(err).Debugf("could not read payload bid of block %v", header.Root) + } else if bid != nil { + r.events.publish(sseEvent{topic: "execution_payload_bid", data: bid}) + } + } + + r.mutex.Lock() + previousEpoch := uint64(0) + if r.head != nil { + previousEpoch = r.chain.epochOf(r.head.Slot) + } + r.head = header + r.mutex.Unlock() + + blockEvent := &v1.BlockEvent{ + Slot: phase0.Slot(header.Slot), + Block: parseRoot(header.Root), + } + r.publishJSON("block", blockEvent) + + headEvent := &v1.HeadEvent{ + Slot: phase0.Slot(header.Slot), + Block: parseRoot(header.Root), + State: parseRoot(header.StateRoot), + EpochTransition: header.Slot%r.chain.slotsPerEpoch == 0, + } + r.publishJSON("head", headEvent) + + // finality only ever moves at an epoch boundary, so it is re-read once per epoch + // rather than for every block + if r.chain.epochOf(header.Slot) != previousEpoch { + if err := r.refreshFinality(ctx, header); err != nil { + r.logger.WithError(err).Warnf("could not refresh finality at slot %v", header.Slot) + } + } + + return nil +} + +// refreshFinality reads the finality the chain knew at a head block and emits a +// finalized_checkpoint event when it moved. +func (r *Replay) refreshFinality(ctx context.Context, header *blockHeader) error { + finality, err := r.readFinality(ctx, header) + if err != nil { + return err + } + + r.mutex.Lock() + previous := r.finality + r.finality = finality + r.mutex.Unlock() + + // the initial read only establishes the baseline, it is not a new checkpoint + if previous == nil || previous.FinalizedEpoch == finality.FinalizedEpoch { + return nil + } + + finalizedHeader, err := r.upstream.headerByRoot(ctx, finality.FinalizedRoot) + if err != nil || finalizedHeader == nil { + r.logger.Debugf("could not resolve finalized header %v", finality.FinalizedRoot) + return nil + } + + event := &v1.FinalizedCheckpointEvent{ + Block: parseRoot(finality.FinalizedRoot), + State: parseRoot(finalizedHeader.StateRoot), + Epoch: phase0.Epoch(finality.FinalizedEpoch), + } + r.publishJSON("finalized_checkpoint", event) + + r.logger.Infof("finalized checkpoint moved to epoch %v", finality.FinalizedEpoch) + + return nil +} + +// readFinality asks the upstream what the chain had finalized at a head block. Reading +// finality forces the node to load that state, and nodes keep only a shallow window of +// them, so a pruned head state falls back to the epoch boundaries below it: finality +// only moves at an epoch boundary, so the epoch's first block answers for the whole +// epoch. +func (r *Replay) readFinality(ctx context.Context, header *blockHeader) (*finalityCheckpoints, error) { + finality, err := r.upstream.finality(ctx, header.StateRoot) + if err == nil { + return finality, nil + } + + if err != errNotFound { + return nil, err + } + + epoch := r.chain.epochOf(header.Slot) + + for attempt := 0; attempt < finalityFallbackEpochs && epoch > 0; attempt++ { + boundary, headerErr := r.upstream.headerBySlot(ctx, epoch*r.chain.slotsPerEpoch) + epoch-- + + if headerErr != nil || boundary == nil { + continue + } + + finality, err = r.upstream.finality(ctx, boundary.StateRoot) + if err == nil { + r.logger.Debugf("read finality from epoch boundary slot %v (head state was pruned)", boundary.Slot) + + return finality, nil + } + + if err != errNotFound { + return nil, err + } + } + + return nil, fmt.Errorf("upstream has no state to read finality from at slot %v; use an archive node or a node that keeps historical states", header.Slot) +} + +// advanceExecution moves the virtual execution head to the last block produced at or +// before this slot. A new execution block whose timestamp is exactly this slot's time +// means the payload for this slot was revealed, which is what the availability event +// reports post-Gloas. +func (r *Replay) advanceExecution(ctx context.Context, slot uint64, boundary time.Time, header *blockHeader) error { + if r.el == nil { + return nil + } + + added, err := r.el.advanceTo(ctx, boundary) + if err != nil { + return fmt.Errorf("error advancing execution head: %w", err) + } + + if header == nil { + return nil + } + + for _, block := range added { + if block.timestamp.Equal(boundary) { + event := &v1.ExecutionPayloadAvailableEvent{ + BlockRoot: parseRoot(header.Root), + Slot: phase0.Slot(slot), + } + r.publishJSON("execution_payload_available", event) + + break + } + } + + return nil +} + +// redactURL strips credentials from an endpoint so they do not end up on the console +// or in the status endpoint. +func redactURL(endpoint string) string { + parsed, err := url.Parse(endpoint) + if err != nil { + return endpoint + } + + return parsed.Redacted() +} + +func (r *Replay) publishJSON(topic string, event any) { + data, err := json.Marshal(event) + if err != nil { + r.logger.WithError(err).Errorf("could not encode %v event", topic) + return + } + + r.events.publish(sseEvent{topic: topic, data: data}) +} diff --git a/replay/replay_test.go b/replay/replay_test.go new file mode 100644 index 00000000..9297f362 --- /dev/null +++ b/replay/replay_test.go @@ -0,0 +1,285 @@ +package replay + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +const ( + testGenesisUnix = 1700000000 + testSlotsPerEpoch = 32 + testSlotSeconds = 12 +) + +// fakeBeacon is a minimal beacon node serving a synthetic chain, used to drive the +// replay end to end without a real upstream. +type fakeBeacon struct { + // emptySlots are slots that carry no block. + emptySlots map[uint64]bool + + // finalizedEpoch is what every finality read reports. + finalizedEpoch uint64 +} + +func (f *fakeBeacon) rootOf(slot uint64) string { + return fmt.Sprintf("0x%064x", slot) +} + +func (f *fakeBeacon) stateRootOf(slot uint64) string { + return fmt.Sprintf("0x%064x", slot+1_000_000) +} + +func (f *fakeBeacon) start() *httptest.Server { + mux := http.NewServeMux() + + mux.HandleFunc("/eth/v1/beacon/genesis", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{"genesis_time": strconv.Itoa(testGenesisUnix)}, + }) + }) + + mux.HandleFunc("/eth/v1/config/spec", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]string{ + "SECONDS_PER_SLOT": strconv.Itoa(testSlotSeconds), + "SLOTS_PER_EPOCH": strconv.Itoa(testSlotsPerEpoch), + }, + }) + }) + + mux.HandleFunc("/eth/v1/beacon/headers/", func(w http.ResponseWriter, r *http.Request) { + id := strings.TrimPrefix(r.URL.Path, "/eth/v1/beacon/headers/") + + slot, err := strconv.ParseUint(id, 10, 64) + if err != nil { + // resolve by root, which the synthetic chain encodes as the slot number + parsed, parseErr := strconv.ParseUint(strings.TrimLeft(strings.TrimPrefix(id, "0x"), "0"), 16, 64) + if parseErr != nil { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + slot = parsed + } + + if f.emptySlots[slot] { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "root": f.rootOf(slot), + "header": map[string]any{ + "message": map[string]any{ + "slot": strconv.FormatUint(slot, 10), + "parent_root": f.rootOf(slot - 1), + "state_root": f.stateRootOf(slot), + }, + }, + }, + }) + }) + + mux.HandleFunc("/eth/v1/beacon/states/", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "current_justified": map[string]any{ + "epoch": strconv.FormatUint(f.finalizedEpoch+1, 10), + "root": f.rootOf(1), + }, + "finalized": map[string]any{ + "epoch": strconv.FormatUint(f.finalizedEpoch, 10), + "root": f.rootOf(2), + }, + }, + }) + }) + + mux.HandleFunc("/eth/v2/beacon/blocks/", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "message": map[string]any{ + "body": map[string]any{ + "signed_execution_payload_bid": map[string]any{"message": map[string]any{"slot": "1"}}, + }, + }, + }, + }) + }) + + return httptest.NewServer(mux) +} + +func newTestReplay(t *testing.T, beacon *fakeBeacon, startSlot uint64) (*Replay, *httptest.Server) { + t.Helper() + + server := beacon.start() + t.Cleanup(server.Close) + + cfg := DefaultConfig() + cfg.UpstreamURL = server.URL + cfg.StartSlot = startSlot + cfg.EmitBids = false + cfg.StepSettle = time.Millisecond + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + replay, err := New(context.Background(), logger, cfg) + require.NoError(t, err) + + return replay, server +} + +// collectEvents subscribes to the hub the way an event stream client would. +func collectEvents(replay *Replay, topics ...string) *eventSubscriber { + filter := make(map[string]bool, len(topics)) + for _, topic := range topics { + filter[topic] = true + } + + sub := newEventSubscriber(filter) + replay.events.add(sub) + + return sub +} + +func TestNewResolvesHeadAtStartSlot(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{100: true, 99: true}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + + require.Equal(t, uint64(100), replay.virtualSlot) + require.NotNil(t, replay.head) + require.Equal(t, uint64(98), replay.head.Slot, "head must be the newest block at or before the start slot") + require.Equal(t, uint64(2), replay.finality.FinalizedEpoch) +} + +func TestStepEmitsBlockAndHead(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + sub := collectEvents(replay, "block", "head") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go replay.runDriver(ctx) + replay.Step(1) + + blockEvent := awaitEvent(t, sub, "block") + require.Equal(t, "101", jsonField(t, blockEvent.data, "slot")) + require.Equal(t, beacon.rootOf(101), jsonField(t, blockEvent.data, "block")) + + headEvent := awaitEvent(t, sub, "head") + require.Equal(t, "101", jsonField(t, headEvent.data, "slot")) + require.Equal(t, beacon.stateRootOf(101), jsonField(t, headEvent.data, "state")) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause after the step") + require.Equal(t, uint64(101), replay.Status().VirtualSlot) + require.Equal(t, uint64(101), replay.Status().HeadSlot) +} + +func TestEmptySlotAdvancesWithoutEvents(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{101: true}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + sub := collectEvents(replay, "block", "head") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go replay.runDriver(ctx) + replay.Step(1) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause after the step") + + require.Equal(t, uint64(101), replay.Status().VirtualSlot) + require.Equal(t, uint64(100), replay.Status().HeadSlot, "an empty slot must not move the head") + require.Empty(t, sub.events, "an empty slot must not emit events") +} + +func TestForwardRunsToTargetSlot(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + go replay.runDriver(ctx) + require.NoError(t, replay.Forward(110, 0)) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause at the target") + require.Equal(t, uint64(110), replay.Status().VirtualSlot) +} + +func TestVirtualClockTracksReplayedSlot(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + + // before stepping, the clock sits inside the start slot + require.Equal(t, uint64(100), slotOfVirtualTime(replay)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go replay.runDriver(ctx) + replay.Step(3) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause after the step") + require.Equal(t, uint64(103), slotOfVirtualTime(replay)) +} + +func slotOfVirtualTime(replay *Replay) uint64 { + elapsed := replay.clock.now().Sub(replay.chain.genesisTime) + + return uint64(elapsed / replay.chain.slotDuration) +} + +func awaitEvent(t *testing.T, sub *eventSubscriber, topic string) sseEvent { + t.Helper() + + deadline := time.After(10 * time.Second) + + for { + select { + case event := <-sub.events: + if event.topic == topic { + return event + } + case <-deadline: + t.Fatalf("timed out waiting for a %v event", topic) + } + } +} + +func jsonField(t *testing.T, data []byte, field string) string { + t.Helper() + + parsed := map[string]any{} + require.NoError(t, json.Unmarshal(data, &parsed)) + + value, ok := parsed[field].(string) + require.Truef(t, ok, "field %v is missing or not a string in %s", field, data) + + return value +} + +func requireEventually(t *testing.T, condition func() bool, message string) { + t.Helper() + + require.Eventually(t, condition, 10*time.Second, 5*time.Millisecond, message) +} diff --git a/replay/stateloads.go b/replay/stateloads.go new file mode 100644 index 00000000..49070061 --- /dev/null +++ b/replay/stateloads.go @@ -0,0 +1,71 @@ +package replay + +import "sync" + +// stateLoads counts the beacon states the explorer is currently pulling through the +// proxy. A full state is tens of megabytes and takes seconds to fetch, decompress and +// decode, during which the explorer cannot process anything else — so the replay uses +// this to hold its clock rather than running ahead of what the explorer has seen. +type stateLoads struct { + mutex sync.Mutex + active int + total uint64 + + // idle is closed while nothing is loading, and replaced with an open channel as + // soon as a load starts, so a waiter can select on it. + idle chan struct{} +} + +func newStateLoads() *stateLoads { + loads := &stateLoads{idle: make(chan struct{})} + close(loads.idle) + + return loads +} + +// begin records the start of a state load and returns the function that ends it. +func (s *stateLoads) begin() func() { + s.mutex.Lock() + + if s.active == 0 { + s.idle = make(chan struct{}) + } + + s.active++ + s.total++ + s.mutex.Unlock() + + ended := false + + return func() { + if ended { + return + } + + ended = true + + s.mutex.Lock() + defer s.mutex.Unlock() + + s.active-- + if s.active == 0 { + close(s.idle) + } + } +} + +// idleChan returns a channel that is closed once no state is being loaded. It is +// already closed when nothing is loading right now. +func (s *stateLoads) idleChan() <-chan struct{} { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.idle +} + +func (s *stateLoads) count() int { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.active +} diff --git a/replay/stateloads_test.go b/replay/stateloads_test.go new file mode 100644 index 00000000..b2b1aa68 --- /dev/null +++ b/replay/stateloads_test.go @@ -0,0 +1,217 @@ +package replay + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestStateLoadsTracksActiveReads(t *testing.T) { + loads := newStateLoads() + + requireClosed(t, loads.idleChan(), "nothing is loading yet") + require.Zero(t, loads.count()) + + endFirst := loads.begin() + endSecond := loads.begin() + + require.Equal(t, 2, loads.count()) + requireOpen(t, loads.idleChan(), "two reads are in flight") + + endFirst() + require.Equal(t, 1, loads.count()) + requireOpen(t, loads.idleChan(), "one read is still in flight") + + endSecond() + require.Zero(t, loads.count()) + requireClosed(t, loads.idleChan(), "the last read finished") + + // ending twice must not double-count + endSecond() + require.Zero(t, loads.count()) +} + +func TestStateLoadsBecomeBusyAgain(t *testing.T) { + loads := newStateLoads() + + end := loads.begin() + end() + + requireClosed(t, loads.idleChan(), "idle after the first read") + + loads.begin() + requireOpen(t, loads.idleChan(), "a later read must make it busy again") +} + +// TestClockDoesNotMoveWhileHeld is the whole point of the state gate: waiting for the +// explorer must cost real time but no virtual time, or the replay runs ahead of what +// the explorer has actually seen. +func TestClockDoesNotMoveWhileHeld(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + clock.setRate(100) + + time.Sleep(20 * time.Millisecond) + + clock.hold() + frozen := clock.now() + + _, rate := clock.state() + require.Zero(t, rate, "the explorer must see a stopped clock while the replay is holding") + + time.Sleep(30 * time.Millisecond) + require.Equal(t, frozen, clock.now(), "no virtual time may pass while held") + + clock.release() + + _, rate = clock.state() + require.Equal(t, 100.0, rate, "releasing must restore the rate it was running at") + + time.Sleep(20 * time.Millisecond) + require.True(t, clock.now().After(frozen), "the clock must resume after the hold") +} + +func TestClockHoldsNest(t *testing.T) { + clock := newClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + clock.setRate(10) + + clock.hold() + clock.hold() + clock.release() + + require.True(t, clock.isHeld(), "the clock stays held until every hold is released") + + clock.release() + require.False(t, clock.isHeld()) + + // releasing more often than holding must not unbalance the counter + clock.release() + require.False(t, clock.isHeld()) +} + +func TestAwaitStateLoadsHoldsUntilTheReadFinishes(t *testing.T) { + replay := testConsoleReplay(100) + replay.clock.setRate(50) + + end := replay.states.begin() + + released := make(chan struct{}) + + go func() { + defer close(released) + require.NoError(t, replay.awaitStateLoads(context.Background())) + }() + + require.Eventually(t, func() bool { return replay.clock.isHeld() }, 5*time.Second, 5*time.Millisecond, + "the gate must freeze the clock while a state is loading") + + frozen := replay.clock.now() + + // the gate must keep holding, not return + time.Sleep(50 * time.Millisecond) + + select { + case <-released: + t.Fatal("the gate returned while a state was still loading") + default: + } + + require.True(t, replay.clock.isHeld()) + require.True(t, replay.Status().Holding) + require.Equal(t, 1, replay.Status().StateLoads) + require.Equal(t, frozen, replay.clock.now(), "the 50ms wait must have cost no virtual time") + + end() + + select { + case <-released: + case <-time.After(5 * time.Second): + t.Fatal("the gate did not release when the state load finished") + } + + require.False(t, replay.clock.isHeld()) + require.False(t, replay.Status().Holding) +} + +func TestAwaitStateLoadsReturnsImmediatelyWhenIdle(t *testing.T) { + replay := testConsoleReplay(100) + + done := make(chan struct{}) + + go func() { + defer close(done) + require.NoError(t, replay.awaitStateLoads(context.Background())) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the gate must not wait when no state is loading") + } + + require.False(t, replay.clock.isHeld()) +} + +func TestAwaitStateLoadsGivesUpAfterTheTimeout(t *testing.T) { + replay := testConsoleReplay(100) + replay.cfg.StateHoldTimeout = 50 * time.Millisecond + + replay.states.begin() // never finishes + + done := make(chan struct{}) + + go func() { + defer close(done) + require.NoError(t, replay.awaitStateLoads(context.Background())) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("a stuck state load must not wedge the replay for good") + } + + require.False(t, replay.clock.isHeld(), "the hold must be lifted when the gate gives up") +} + +func TestAwaitStateLoadsStopsOnShutdown(t *testing.T) { + replay := testConsoleReplay(100) + replay.states.begin() // never finishes + + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + + go func() { done <- replay.awaitStateLoads(ctx) }() + + cancel() + + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("the gate must give up when the replay shuts down") + } +} + +func requireClosed(t *testing.T, channel <-chan struct{}, message string) { + t.Helper() + + select { + case <-channel: + default: + t.Fatalf("expected an idle (closed) channel: %v", message) + } +} + +func requireOpen(t *testing.T, channel <-chan struct{}, message string) { + t.Helper() + + select { + case <-channel: + t.Fatalf("expected a busy (open) channel: %v", message) + default: + } +} diff --git a/replay/tracoor.go b/replay/tracoor.go new file mode 100644 index 00000000..e46c79f8 --- /dev/null +++ b/replay/tracoor.go @@ -0,0 +1,169 @@ +package replay + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/klauspost/compress/zstd" +) + +// zstdMagic identifies a zstd frame. Tracoor stores its artifacts compressed and the +// object store hands them back with Content-Encoding: zstd, which the Go HTTP client +// does not unwrap on its own. +var zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} + +// tracoorArtifact is a stored artifact and the slot it belongs to. The slot is what +// lets the replay label the artifact with the right fork. +type tracoorArtifact struct { + body []byte + slot uint64 +} + +// tracoorClient resolves artifacts from a tracoor instance. Tracoor indexes by root +// rather than by slot, so it can only answer for blocks and states whose root is +// already known from a header. +type tracoorClient struct { + base string + network string + client *http.Client + decoder *zstd.Decoder +} + +func newTracoorClient(baseURL, network string) (*tracoorClient, error) { + decoder, err := zstd.NewReader(nil) + if err != nil { + return nil, fmt.Errorf("could not create zstd decoder: %w", err) + } + + return &tracoorClient{ + base: strings.TrimSuffix(baseURL, "/"), + network: network, + client: newPooledClient(5 * time.Minute), + decoder: decoder, + }, nil +} + +func (t *tracoorClient) fetchBeaconBlock(ctx context.Context, blockRoot string) (*tracoorArtifact, error) { + return t.fetch(ctx, "list-beacon-block", "block_root", blockRoot, "beacon_blocks", "beacon_block") +} + +func (t *tracoorClient) fetchBeaconState(ctx context.Context, stateRoot string) (*tracoorArtifact, error) { + return t.fetch(ctx, "list-beacon-state", "state_root", stateRoot, "beacon_states", "beacon_state") +} + +func (t *tracoorClient) fetch(ctx context.Context, endpoint, rootField, root, listField, artifactType string) (*tracoorArtifact, error) { + id, slot, err := t.lookup(ctx, endpoint, rootField, root, listField) + if err != nil { + return nil, err + } + + body, err := t.download(ctx, artifactType, id) + if err != nil { + return nil, err + } + + return &tracoorArtifact{body: body, slot: slot}, nil +} + +// lookup asks tracoor which stored artifacts match a root and returns the newest one. +func (t *tracoorClient) lookup(ctx context.Context, endpoint, rootField, root, listField string) (string, uint64, error) { + request := map[string]any{ + "network": t.network, + rootField: root, + "pagination": map[string]any{ + "limit": 1, + "offset": 0, + "order_by": "fetched_at DESC", + }, + } + + body, err := json.Marshal(request) + if err != nil { + return "", 0, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.base+"/v1/api/"+endpoint, bytes.NewReader(body)) + if err != nil { + return "", 0, err + } + + req.Header.Set("Content-Type", "application/json") + + rsp, err := t.client.Do(req) + if err != nil { + return "", 0, err + } + defer func() { _ = rsp.Body.Close() }() + + if rsp.StatusCode != http.StatusOK { + data, _ := io.ReadAll(rsp.Body) + return "", 0, fmt.Errorf("tracoor %v returned %v: %s", endpoint, rsp.StatusCode, truncate(data, 200)) + } + + // the response shape differs per endpoint only in the name of the list field + parsed := map[string][]struct { + ID string `json:"id"` + Slot string `json:"slot"` + }{} + + if err := json.NewDecoder(rsp.Body).Decode(&parsed); err != nil { + return "", 0, fmt.Errorf("error parsing tracoor response: %w", err) + } + + items := parsed[listField] + if len(items) == 0 || items[0].ID == "" { + return "", 0, errNotFound + } + + slot, err := strconv.ParseUint(items[0].Slot, 10, 64) + if err != nil { + return "", 0, fmt.Errorf("invalid slot %q in tracoor response: %w", items[0].Slot, err) + } + + return items[0].ID, slot, nil +} + +func (t *tracoorClient) download(ctx context.Context, artifactType, id string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, t.base+"/download/"+artifactType+"/"+id, http.NoBody) + if err != nil { + return nil, err + } + + rsp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = rsp.Body.Close() }() + + if rsp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("tracoor download of %v/%v returned %v", artifactType, id, rsp.StatusCode) + } + + body, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + + return t.decompress(body) +} + +// decompress unwraps a zstd frame, leaving anything else untouched. +func (t *tracoorClient) decompress(body []byte) ([]byte, error) { + if !bytes.HasPrefix(body, zstdMagic) { + return body, nil + } + + decoded, err := t.decoder.DecodeAll(body, nil) + if err != nil { + return nil, fmt.Errorf("error decompressing tracoor artifact: %w", err) + } + + return decoded, nil +} diff --git a/replay/upstream.go b/replay/upstream.go new file mode 100644 index 00000000..e09e3ad1 --- /dev/null +++ b/replay/upstream.go @@ -0,0 +1,499 @@ +package replay + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// errNotFound is returned when the upstream has no artifact for a request. It maps to +// a 404 on the proxy, which is a normal answer for empty slots. +var errNotFound = fmt.Errorf("not found") + +// upstreamError carries a non-2xx upstream answer so the proxy can hand the same status +// and body back to its own client. Clients read these codes: a 400 for "this block has +// no payload envelope" means something quite different from a gateway failure. +type upstreamError struct { + status int + body []byte + header http.Header +} + +func (e *upstreamError) Error() string { + return fmt.Sprintf("upstream returned %v: %s", e.status, truncate(e.body, 200)) +} + +// upstream is the read side of the replay: a beacon node that answers by slot or root, +// with an optional tracoor fallback for artifacts the node has already pruned. +type upstream struct { + logger logrus.FieldLogger + base *url.URL + client *http.Client + tracoor *tracoorClient + cache *artifactCache + + // chain is set once the timing specs are known; it labels tracoor artifacts with + // the fork they belong to. + chain *chainInfo +} + +// artifact is a raw upstream response, kept encoded so it can be forwarded verbatim. +// The headers matter as much as the body: SSZ responses carry their fork in +// Eth-Consensus-Version, without which the explorer cannot decode them. +type artifact struct { + body []byte + header http.Header +} + +// newPooledClient returns an HTTP client that keeps connections alive across the many +// small requests a replay makes. The default transport keeps only two idle connections +// per host, which for a TLS upstream means a fresh handshake on nearly every request — +// on a remote devnet that alone dominates the time it takes to step a slot. +func newPooledClient(timeout time.Duration) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConns = 128 + transport.MaxIdleConnsPerHost = 64 + transport.MaxConnsPerHost = 64 + transport.IdleConnTimeout = 10 * time.Minute + + return &http.Client{Transport: transport, Timeout: timeout} +} + +func newUpstream(logger logrus.FieldLogger, cfg *Config) (*upstream, error) { + base, err := url.Parse(strings.TrimSuffix(cfg.UpstreamURL, "/")) + if err != nil { + return nil, fmt.Errorf("invalid upstream url: %w", err) + } + + up := &upstream{ + logger: logger, + base: base, + client: newPooledClient(10 * time.Minute), + } + + if cfg.TracoorURL != "" { + tracoor, err := newTracoorClient(cfg.TracoorURL, cfg.TracoorNetwork) + if err != nil { + return nil, err + } + + up.tracoor = tracoor + } + + if cfg.CacheDir != "" { + cache, err := newArtifactCache(logger, cfg.CacheDir) + if err != nil { + return nil, err + } + + up.cache = cache + } + + return up, nil +} + +// get fetches a path from the beacon upstream, serving it from the artifact cache when +// the path is immutable and already recorded. +func (u *upstream) get(ctx context.Context, path string, accept string) (*artifact, error) { + cacheKey := "" + if u.cache != nil && isImmutablePath(path) { + cacheKey = u.cache.key(path, accept) + if art := u.cache.load(cacheKey); art != nil { + return art, nil + } + } + + art, err := u.fetch(ctx, path, accept) + if err != nil { + return nil, err + } + + if cacheKey != "" { + u.cache.store(cacheKey, art) + } + + return art, nil +} + +func (u *upstream) fetch(ctx context.Context, path string, accept string) (*artifact, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.base.String()+path, http.NoBody) + if err != nil { + return nil, err + } + + if accept != "" { + req.Header.Set("Accept", accept) + } + + rsp, err := u.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = rsp.Body.Close() }() + + body, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + + if rsp.StatusCode == http.StatusNotFound { + return nil, errNotFound + } + + if rsp.StatusCode != http.StatusOK { + return nil, &upstreamError{ + status: rsp.StatusCode, + body: body, + header: forwardableHeaders(rsp.Header), + } + } + + return &artifact{body: body, header: forwardableHeaders(rsp.Header)}, nil +} + +// hopByHopHeaders are the response headers that describe this particular connection +// rather than the artifact, so they must not be copied to the proxied response. +var hopByHopHeaders = map[string]bool{ + "Connection": true, + "Content-Encoding": true, + "Content-Length": true, + "Keep-Alive": true, + "Proxy-Authenticate": true, + "Proxy-Connection": true, + "Te": true, + "Trailer": true, + "Transfer-Encoding": true, + "Upgrade": true, + "Date": true, + "Server": true, +} + +func forwardableHeaders(src http.Header) http.Header { + dst := make(http.Header, len(src)) + + for key, values := range src { + if hopByHopHeaders[http.CanonicalHeaderKey(key)] { + continue + } + + dst[http.CanonicalHeaderKey(key)] = values + } + + return dst +} + +func (u *upstream) getJSON(ctx context.Context, path string, out any) error { + art, err := u.get(ctx, path, "application/json") + if err != nil { + return err + } + + if err := json.Unmarshal(art.body, out); err != nil { + return fmt.Errorf("error parsing %v: %w", path, err) + } + + return nil +} + +// blockHeader is the part of a beacon block header the replay needs to track the head +// and to rewrite `head` aliases into concrete roots. +type blockHeader struct { + Slot uint64 + Root string + StateRoot string + ParentRoot string + BlockNumber uint64 +} + +type headerResponse struct { + Data struct { + Root string `json:"root"` + Header struct { + Message struct { + Slot string `json:"slot"` + ParentRoot string `json:"parent_root"` + StateRoot string `json:"state_root"` + } `json:"message"` + } `json:"header"` + } `json:"data"` +} + +func (r *headerResponse) toHeader() (*blockHeader, error) { + slot, err := strconv.ParseUint(r.Data.Header.Message.Slot, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid slot in header response: %w", err) + } + + return &blockHeader{ + Slot: slot, + Root: r.Data.Root, + StateRoot: r.Data.Header.Message.StateRoot, + ParentRoot: r.Data.Header.Message.ParentRoot, + }, nil +} + +// headerBySlot returns the block header at a slot, or nil when the slot is empty. +func (u *upstream) headerBySlot(ctx context.Context, slot uint64) (*blockHeader, error) { + return u.header(ctx, strconv.FormatUint(slot, 10)) +} + +// headerByRoot returns the block header for a block root, or nil when it is unknown. +func (u *upstream) headerByRoot(ctx context.Context, root string) (*blockHeader, error) { + return u.header(ctx, root) +} + +func (u *upstream) header(ctx context.Context, blockID string) (*blockHeader, error) { + rsp := headerResponse{} + + err := u.getJSON(ctx, "/eth/v1/beacon/headers/"+blockID, &rsp) + if err != nil { + if err == errNotFound { + return nil, nil + } + + return nil, err + } + + return rsp.toHeader() +} + +// finalityCheckpoints is the finality the chain knew at a given block root. Raw keeps +// the upstream response so the replay can serve it back verbatim, including the +// previous justified checkpoint it does not track itself. +type finalityCheckpoints struct { + JustifiedEpoch uint64 + JustifiedRoot string + FinalizedEpoch uint64 + FinalizedRoot string + Raw json.RawMessage +} + +func (u *upstream) finality(ctx context.Context, stateID string) (*finalityCheckpoints, error) { + rsp := struct { + Data struct { + CurrentJustified struct { + Epoch string `json:"epoch"` + Root string `json:"root"` + } `json:"current_justified"` + Finalized struct { + Epoch string `json:"epoch"` + Root string `json:"root"` + } `json:"finalized"` + } `json:"data"` + }{} + + art, err := u.get(ctx, "/eth/v1/beacon/states/"+stateID+"/finality_checkpoints", "application/json") + if err != nil { + return nil, err + } + + raw := json.RawMessage(art.body) + + if err := json.Unmarshal(raw, &rsp); err != nil { + return nil, fmt.Errorf("error parsing finality checkpoints: %w", err) + } + + justifiedEpoch, err := strconv.ParseUint(rsp.Data.CurrentJustified.Epoch, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid justified epoch: %w", err) + } + + finalizedEpoch, err := strconv.ParseUint(rsp.Data.Finalized.Epoch, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid finalized epoch: %w", err) + } + + return &finalityCheckpoints{ + JustifiedEpoch: justifiedEpoch, + JustifiedRoot: rsp.Data.CurrentJustified.Root, + FinalizedEpoch: finalizedEpoch, + FinalizedRoot: rsp.Data.Finalized.Root, + Raw: raw, + }, nil +} + +// payloadBid returns the winning execution payload bid carried by a Gloas block, as +// raw JSON ready to be replayed on the event stream. It returns nil for blocks from +// forks that have no bid. +func (u *upstream) payloadBid(ctx context.Context, root string) (json.RawMessage, error) { + rsp := struct { + Data struct { + Message struct { + Body struct { + SignedExecutionPayloadBid json.RawMessage `json:"signed_execution_payload_bid"` + } `json:"body"` + } `json:"message"` + } `json:"data"` + }{} + + if err := u.getJSON(ctx, "/eth/v2/beacon/blocks/"+root, &rsp); err != nil { + return nil, err + } + + bid := rsp.Data.Message.Body.SignedExecutionPayloadBid + if len(bid) == 0 || string(bid) == "null" { + return nil, nil + } + + return bid, nil +} + +func isStatePath(path string) bool { + return strings.Contains(path, "/debug/beacon/states/") +} + +// serve answers a proxied GET for a client of the fake node. Beacon states are read +// from tracoor first when it is configured: it keeps every state, while beacon nodes +// prune all but the most recent ones, and serving a 17 MB state per epoch out of the +// archive would hammer the devnet. +func (u *upstream) serve(w http.ResponseWriter, r *http.Request, path string) { + accept := r.Header.Get("Accept") + + // states are read from tracoor first, blocks are not: the node still has every + // block of the replayed range and answers in one round trip, while a tracoor read + // costs a lookup plus a download + if isStatePath(path) { + if art := u.tryTracoor(r.Context(), path, accept); art != nil { + writeArtifact(w, art, http.StatusOK) + return + } + } + + art, err := u.get(r.Context(), path, accept) + + switch { + case err == errNotFound: + if art = u.tryTracoor(r.Context(), path, "application/octet-stream"); art == nil { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + case err != nil: + var upErr *upstreamError + if errors.As(err, &upErr) { + // hand the upstream's own answer back unchanged; the client knows what a + // 400 or a 503 from a beacon node means + u.logger.Debugf("upstream %v answered %v", path, upErr.status) + writeArtifact(w, &artifact{body: upErr.body, header: upErr.header}, upErr.status) + + return + } + + u.logger.WithError(err).Warnf("upstream request failed: %v", path) + writeAPIError(w, http.StatusBadGateway, err.Error()) + + return + } + + writeArtifact(w, art, http.StatusOK) +} + +// tryTracoor resolves a root-addressed block or state from tracoor, returning nil when +// tracoor is not configured, cannot serve this path, or the client cannot take SSZ. +func (u *upstream) tryTracoor(ctx context.Context, path, accept string) *artifact { + if u.tracoor == nil || !acceptsSSZ(accept) { + return nil + } + + segments := strings.Split(strings.Trim(path, "/"), "/") + root := segments[len(segments)-1] + + if !strings.HasPrefix(root, "0x") { + return nil + } + + cacheKey := "" + if u.cache != nil { + cacheKey = u.cache.key(path, "tracoor") + if art := u.cache.load(cacheKey); art != nil { + return art + } + } + + var ( + fetched *tracoorArtifact + err error + ) + + switch { + case strings.Contains(path, "/debug/beacon/states/"): + fetched, err = u.tracoor.fetchBeaconState(ctx, root) + case strings.Contains(path, "/beacon/blocks/"): + fetched, err = u.tracoor.fetchBeaconBlock(ctx, root) + default: + return nil + } + + if err != nil { + if err != errNotFound { + u.logger.WithError(err).Debugf("tracoor lookup failed for %v", path) + } + + return nil + } + + art := &artifact{ + body: fetched.body, + header: http.Header{ + "Content-Type": []string{"application/octet-stream"}, + "Eth-Consensus-Version": []string{u.forkAt(fetched.slot)}, + }, + } + + if cacheKey != "" { + u.cache.store(cacheKey, art) + } + + u.logger.Debugf("served %v from tracoor (slot %v, %v bytes)", path, fetched.slot, len(art.body)) + + return art +} + +func (u *upstream) forkAt(slot uint64) string { + if u.chain == nil { + return "" + } + + return u.chain.forkAt(slot) +} + +// acceptsSSZ reports whether a client will take an SSZ-encoded response. Tracoor only +// stores SSZ, so a JSON-only client has to be served from the beacon node. +func acceptsSSZ(accept string) bool { + return accept == "" || strings.Contains(accept, "application/octet-stream") || strings.Contains(accept, "*/*") +} + +func writeArtifact(w http.ResponseWriter, art *artifact, status int) { + for key, values := range art.header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + if w.Header().Get("Content-Type") == "" { + w.Header().Set("Content-Type", "application/octet-stream") + } + + w.WriteHeader(status) + + if _, err := w.Write(art.body); err != nil { + return + } +} + +func truncate(data []byte, max int) string { + if len(data) <= max { + return string(data) + } + + return string(data[:max]) + "..." +} diff --git a/static/js/page-index.js b/static/js/page-index.js index 31ed78da..76f18175 100644 --- a/static/js/page-index.js +++ b/static/js/page-index.js @@ -6,7 +6,7 @@ initCountdownTooltips(); }); - var refreshInterval = 15000; + var defaultRefreshInterval = 15000; var lastRefresh = new Date().getTime(); var loopTimer = null; var isRefreshing = false; @@ -97,10 +97,22 @@ }, }; + // refreshInterval is read on every loop rather than captured once, so an external + // driver (dora-replay, which advances slots far faster than real time) can retune + // it at runtime by setting window.doraIndexRefreshInterval. + function refreshInterval() { + var override = window.doraIndexRefreshInterval; + if (typeof override === "number" && isFinite(override) && override > 0) { + return override; + } + + return defaultRefreshInterval; + } + function scheduleLoop() { if(loopTimer) return; - var refreshTimeout = refreshInterval - ((new Date().getTime() - lastRefresh)); + var refreshTimeout = refreshInterval() - ((new Date().getTime() - lastRefresh)); if(refreshTimeout < 0) refreshTimeout = 0; else if(refreshTimeout > 1000) @@ -110,14 +122,13 @@ function refreshLoop() { loopTimer = null; - var refreshTimeout = refreshInterval - ((new Date().getTime() - lastRefresh)); + var refreshTimeout = refreshInterval() - ((new Date().getTime() - lastRefresh)); if(refreshTimeout < 0) refreshTimeout = 0; document.getElementById("update_timer").innerText = "Next update in " + Math.ceil(refreshTimeout / 1000) + "s"; if(refreshTimeout <= 0) { lastRefresh = new Date().getTime(); - refreshTimeout = refreshInterval; refresh(); } } diff --git a/templates/_layout/layout.html b/templates/_layout/layout.html index 24b47fed..091fd2fc 100644 --- a/templates/_layout/layout.html +++ b/templates/_layout/layout.html @@ -60,6 +60,10 @@ + {{ if .ReplayControlUrl }} + + + {{ end }} {{ template "js" .Data }} diff --git a/types/config.go b/types/config.go index 1c88da07..e2e39953 100644 --- a/types/config.go +++ b/types/config.go @@ -214,6 +214,16 @@ type Config struct { AllowedMethods []string `yaml:"allowedMethods" envconfig:"RPC_PROXY_ALLOWED_METHODS"` } `yaml:"rpcProxy"` + // Replay drives the explorer off the virtual clock of a dora-replay control + // server instead of the real wall clock, so a past slot range can be stepped + // through as if it were happening live. Only meaningful when the beacon and + // execution endpoints point at the matching dora-replay proxies. + Replay struct { + Enabled bool `yaml:"enabled" envconfig:"REPLAY_ENABLED"` + ControlUrl string `yaml:"controlUrl" envconfig:"REPLAY_CONTROL_URL"` + PollInterval time.Duration `yaml:"pollInterval" envconfig:"REPLAY_POLL_INTERVAL"` + } `yaml:"replay"` + // EnsResolver optionally resolves execution addresses to their primary ENS name. // Names are resolved on the local network (the chain this explorer indexes, via // the main execution pool) and on every configured remote network (each with its diff --git a/types/models.go b/types/models.go index 66247143..001b2823 100644 --- a/types/models.go +++ b/types/models.go @@ -34,6 +34,12 @@ type PageData struct { ApiEnabled bool ExecutionIndexerEnabled bool EnsSearchEnabled bool + + // ReplayControlUrl is the dora-replay control server this explorer is driven by. + // It is only set in replay mode, and it is the whole of the explorer's knowledge + // about the replay UI: the layout side-loads that UI from the replay process and + // hands it this address to talk to. + ReplayControlUrl string } type MainMenuItem struct { From 3bac96ebba698cd5a24291c7ebd7f85a713345d9 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 20 Aug 2026 22:10:54 +0200 Subject: [PATCH 3/3] fix(replay): keep serving slots during a state load instead of stalling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state gate froze the clock for the whole of a beacon state read, so the driver sat in awaitStateLoads and emitted no block or head events at all while it lasted — several seconds per epoch boundary, and far longer when reads stacked up. That was the wrong shape. The explorer loads states on a different goroutine than it indexes blocks on, so a slow read does not stop it from processing blocks; it only stops it from finishing that epoch's stats. Holding the replay for the read therefore idles the block indexer for no benefit. Allow the replay to run stateLoadLeadSlots further while a read is in flight and only hold beyond that, so blocks keep flowing through a multi-second read while the clock still cannot run away. Not the cause, for the record: the proxy does not serialize requests behind a long state read. Measured through it, a cold block fetch during a cold 17MB state fetch takes 0.124s against a 0.37s idle baseline, and headers stay at 0.13s with three concurrent state fetches in flight. --- replay/README.md | 15 ++++++--- replay/replay.go | 42 ++++++++++++++++++++--- replay/stateloads_test.go | 70 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 114 insertions(+), 13 deletions(-) diff --git a/replay/README.md b/replay/README.md index 8a5b45c9..08ea3a2e 100644 --- a/replay/README.md +++ b/replay/README.md @@ -155,10 +155,17 @@ state arrived the explorer would be several slots behind the slot the replay cla be at. So halfway through every slot the replay checks whether a state is still on its way to -the explorer, and if so **freezes the clock** until it arrives. The wait costs real time -and no virtual time: the explorer's own clock mirrors the frozen rate, so it stays -exactly where the replay left it instead of drifting. The console and the control panel -both report this as `holding for N state load(s)`. +the explorer. It does **not** stop there: the explorer indexes blocks on a different +goroutine than it loads states on, so a read of several seconds does not stop it from +processing blocks, and holding the replay outright would idle its block indexer and emit +no events at all for the duration. The replay is allowed to serve a few more slots +(`stateLoadLeadSlots`) while a read is in flight, and only **freezes the clock** once it +would get further ahead than that. + +Once it holds, the wait costs real time and no virtual time: the explorer's own clock +mirrors the frozen rate, so it stays exactly where the replay left it instead of +drifting. The console and the control panel both report this as +`holding for N state load(s)`. A read that never finishes cannot wedge a run — the gate gives up after `--state-hold-timeout` (default 2m) and logs a warning. diff --git a/replay/replay.go b/replay/replay.go index 0524587a..970ef91b 100644 --- a/replay/replay.go +++ b/replay/replay.go @@ -27,6 +27,12 @@ const ( // upstreamPollInterval is how often the head of the real chain is re-read, purely // so the UI can show how far the replay still has to go. upstreamPollInterval = 30 * time.Second + + // stateLoadLeadSlots is how many slots the replay may still serve while the explorer + // is loading a beacon state. It is the slack that keeps blocks flowing during a + // multi-second read, so the explorer's block indexer is not idled by a wait that + // only its state loader is in. + stateLoadLeadSlots = 4 ) // Replay steps a past slot range through a fake beacon/execution node pair, driving a @@ -45,6 +51,10 @@ type Replay struct { // wake nudges the driver whenever the drive state changed. wake chan struct{} + // stateLeadFrom is the slot at which the explorer last started loading a state, or + // 0 when it is not loading one. Driver-owned: only advanceSlot touches it. + stateLeadFrom uint64 + mutex sync.RWMutex virtualSlot uint64 head *blockHeader @@ -556,7 +566,7 @@ func (r *Replay) advanceSlot(ctx context.Context, slot uint64) error { return err } - if err := r.awaitStateLoads(ctx); err != nil { + if err := r.awaitStateLoads(ctx, slot); err != nil { return err } @@ -579,23 +589,45 @@ func (r *Replay) advanceSlot(ctx context.Context, slot uint64) error { return nil } -// awaitStateLoads holds the replay until the explorer has finished loading the beacon -// states it asked for. It gives up after StateHoldTimeout so a stuck read cannot wedge -// the replay for good. -func (r *Replay) awaitStateLoads(ctx context.Context) error { +// awaitStateLoads keeps the replay from running away while the explorer is loading a +// beacon state, without stalling it outright. +// +// The explorer loads states on a different goroutine than it indexes blocks on, so a +// state read of several seconds does not stop it from processing blocks — it only stops +// it from finishing that epoch's stats. Holding the whole replay for the read therefore +// idles the block indexer for nothing, and emits no block or head events at all while it +// lasts. So the replay is allowed to run stateLoadLeadSlots further while a read is in +// flight, and only holds once it would get further ahead than that. +// +// Once it does hold, the clock is frozen, so the wait costs real time and no virtual +// time. It gives up after StateHoldTimeout so a stuck read cannot wedge the replay. +func (r *Replay) awaitStateLoads(ctx context.Context, slot uint64) error { idle := r.states.idleChan() select { case <-idle: + r.stateLeadFrom = 0 + return nil default: } + // first slot of this busy period: remember where the explorer started falling behind + if r.stateLeadFrom == 0 { + r.stateLeadFrom = slot + } + + if slot-r.stateLeadFrom < stateLoadLeadSlots { + // let the replay run on; the explorer can index these blocks meanwhile + return nil + } + r.clock.hold() r.notifyStatus() defer func() { r.clock.release() + r.stateLeadFrom = 0 r.notifyStatus() }() diff --git a/replay/stateloads_test.go b/replay/stateloads_test.go index b2b1aa68..565fb569 100644 --- a/replay/stateloads_test.go +++ b/replay/stateloads_test.go @@ -99,9 +99,13 @@ func TestAwaitStateLoadsHoldsUntilTheReadFinishes(t *testing.T) { released := make(chan struct{}) + // the gate only holds once the replay is more than stateLoadLeadSlots ahead of the + // slot at which the read started + replay.stateLeadFrom = 100 + go func() { defer close(released) - require.NoError(t, replay.awaitStateLoads(context.Background())) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100+stateLoadLeadSlots)) }() require.Eventually(t, func() bool { return replay.clock.isHeld() }, 5*time.Second, 5*time.Millisecond, @@ -142,7 +146,7 @@ func TestAwaitStateLoadsReturnsImmediatelyWhenIdle(t *testing.T) { go func() { defer close(done) - require.NoError(t, replay.awaitStateLoads(context.Background())) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100)) }() select { @@ -159,12 +163,13 @@ func TestAwaitStateLoadsGivesUpAfterTheTimeout(t *testing.T) { replay.cfg.StateHoldTimeout = 50 * time.Millisecond replay.states.begin() // never finishes + replay.stateLeadFrom = 100 done := make(chan struct{}) go func() { defer close(done) - require.NoError(t, replay.awaitStateLoads(context.Background())) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100+stateLoadLeadSlots)) }() select { @@ -179,12 +184,13 @@ func TestAwaitStateLoadsGivesUpAfterTheTimeout(t *testing.T) { func TestAwaitStateLoadsStopsOnShutdown(t *testing.T) { replay := testConsoleReplay(100) replay.states.begin() // never finishes + replay.stateLeadFrom = 100 ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- replay.awaitStateLoads(ctx) }() + go func() { done <- replay.awaitStateLoads(ctx, 100+stateLoadLeadSlots) }() cancel() @@ -215,3 +221,59 @@ func requireOpen(t *testing.T, channel <-chan struct{}, message string) { default: } } + +// TestStateLoadGateLetsTheReplayRunOnBriefly is the behaviour that keeps blocks flowing +// during a multi-second state read: the explorer indexes blocks on a different goroutine +// than it loads states on, so the replay serves a few more slots before it holds. +func TestStateLoadGateLetsTheReplayRunOnBriefly(t *testing.T) { + replay := testConsoleReplay(100) + replay.clock.setRate(10) + + replay.states.begin() // never finishes + + for slot := uint64(100); slot < 100+stateLoadLeadSlots; slot++ { + done := make(chan struct{}) + + go func() { + defer close(done) + require.NoError(t, replay.awaitStateLoads(context.Background(), slot)) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("the gate must not hold at slot %v, only %v slots into the read", slot, slot-100) + } + + require.False(t, replay.clock.isHeld(), "the clock must keep running within the lead window") + } + + // beyond the lead window it holds + held := make(chan struct{}) + + go func() { + defer close(held) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100+stateLoadLeadSlots)) + }() + + require.Eventually(t, func() bool { return replay.clock.isHeld() }, 5*time.Second, 5*time.Millisecond, + "the gate must hold once the replay would get further ahead than the lead window") +} + +// TestStateLoadGateResetsBetweenReads guards that the lead window is per read, not +// cumulative: a finished read must not leave the next one starting mid-window. +func TestStateLoadGateResetsBetweenReads(t *testing.T) { + replay := testConsoleReplay(100) + + end := replay.states.begin() + require.NoError(t, replay.awaitStateLoads(context.Background(), 100)) + require.Equal(t, uint64(100), replay.stateLeadFrom) + + end() + require.NoError(t, replay.awaitStateLoads(context.Background(), 101)) + require.Zero(t, replay.stateLeadFrom, "an idle tracker must clear the lead window") + + replay.states.begin() + require.NoError(t, replay.awaitStateLoads(context.Background(), 200)) + require.Equal(t, uint64(200), replay.stateLeadFrom, "the next read starts its own window") +}