diff --git a/clients/consensus/chainstate.go b/clients/consensus/chainstate.go index 67681286..5032be55 100644 --- a/clients/consensus/chainstate.go +++ b/clients/consensus/chainstate.go @@ -2,16 +2,19 @@ package consensus import ( "bytes" + "context" "crypto/sha256" "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/ethpandaops/dora/utils" "github.com/ethpandaops/ethwallclock" v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/ethpandaops/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" ) type ChainState struct { @@ -22,8 +25,13 @@ type ChainState struct { genesisMutex sync.Mutex genesis *v1.Genesis - wallclockMutex sync.Mutex - wallclock *ethwallclock.EthereumBeaconChain + wallclockMutex sync.Mutex + wallclockStarted bool + wallclock *ethwallclock.EthereumBeaconChain + + // replayClock replaces the wall clock with the virtual clock of a dora-replay + // control server, so a past slot range can be stepped through as if it were live. + replayClock atomic.Pointer[replayClock] finalityMutex sync.RWMutex finality *v1.Finality @@ -198,7 +206,7 @@ func (cs *ChainState) initWallclock() { cs.wallclockMutex.Lock() defer cs.wallclockMutex.Unlock() - if cs.wallclock != nil { + if cs.wallclockStarted { return } @@ -206,6 +214,15 @@ func (cs *ChainState) initWallclock() { return } + cs.wallclockStarted = true + + // ethwallclock reads time.Now() internally and cannot be paused, so a replay run + // drives the slot/epoch dispatchers off the virtual clock instead. + if clock := cs.replayClock.Load(); clock != nil { + go cs.runReplayWallclock(clock) + return + } + cs.wallclock = ethwallclock.NewEthereumBeaconChain(cs.genesis.GenesisTime, time.Duration(cs.specs.SlotDurationMs)*time.Millisecond, cs.specs.SlotsPerEpoch) cs.wallclock.OnEpochChanged(func(current ethwallclock.Epoch) { cs.wallclockEpochDispatcher.Fire(¤t) @@ -215,6 +232,105 @@ func (cs *ChainState) initWallclock() { }) } +// EnableReplayClock points the chain state at a dora-replay control server, so every +// "now" in the explorer (current slot, wallclock ticks, page timestamps) follows the +// replayed slot range instead of the real wall clock. It blocks until the control +// server has answered once. +func (cs *ChainState) EnableReplayClock(ctx context.Context, logger logrus.FieldLogger, controlURL string, pollInterval time.Duration) error { + clock, err := newReplayClock(ctx, logger, controlURL, pollInterval) + if err != nil { + return err + } + + cs.replayClock.Store(clock) + + logger.WithField("now", clock.now().UTC().Format(time.RFC3339)).Info("replay clock enabled") + + return nil +} + +// IsReplaying reports whether the chain state is driven by a replay clock. +func (cs *ChainState) IsReplaying() bool { + return cs.replayClock.Load() != nil +} + +// Now returns the current time as the chain sees it: the real wall clock normally, +// or the virtual replay time while a replay is running. +func (cs *ChainState) Now() time.Time { + if clock := cs.replayClock.Load(); clock != nil { + return clock.now() + } + + return time.Now() +} + +// maxReplayWallclockCatchup bounds how many slot ticks are replayed in one go after +// the virtual clock jumped, so a large seek does not flood the dispatchers. +const maxReplayWallclockCatchup = 16 + +// runReplayWallclock fires the slot and epoch dispatchers from the virtual clock, +// standing in for ethwallclock while a replay is running. +func (cs *ChainState) runReplayWallclock(clock *replayClock) { + defer utils.HandleSubroutinePanic("clients.consensus.ChainState.runReplayWallclock", func() { + cs.runReplayWallclock(clock) + }) + + genesis := cs.genesis.GenesisTime + slotDuration := time.Duration(cs.specs.SlotDurationMs) * time.Millisecond + slotsPerEpoch := cs.specs.SlotsPerEpoch + + interval := slotDuration / 20 + if interval < 50*time.Millisecond { + interval = 50 * time.Millisecond + } else if interval > 250*time.Millisecond { + interval = 250 * time.Millisecond + } + + lastSlot := uint64(cs.CurrentSlot()) + lastEpoch := lastSlot / slotsPerEpoch + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-clock.ctx.Done(): + return + case <-ticker.C: + } + + slot := uint64(cs.CurrentSlot()) + if slot <= lastSlot { + // the replay was rewound; resync without firing anything + lastSlot = slot + lastEpoch = slot / slotsPerEpoch + + continue + } + + firstSlot := lastSlot + 1 + if slot-lastSlot > maxReplayWallclockCatchup { + firstSlot = slot + } + + for tickSlot := firstSlot; tickSlot <= slot; tickSlot++ { + slotStart := genesis.Add(time.Duration(tickSlot) * slotDuration) + + if epoch := tickSlot / slotsPerEpoch; epoch != lastEpoch { + epochStart := genesis.Add(time.Duration(epoch*slotsPerEpoch) * slotDuration) + wallclockEpoch := ethwallclock.NewEpoch(epoch, epochStart, epochStart.Add(slotDuration*time.Duration(slotsPerEpoch))) + cs.wallclockEpochDispatcher.Fire(&wallclockEpoch) + lastEpoch = epoch + } + + wallclockSlot := ethwallclock.NewSlot(tickSlot, slotStart, slotStart.Add(slotDuration)) + cs.wallclockSlotDispatcher.Fire(&wallclockSlot) + } + + lastSlot = slot + } +} + func (cs *ChainState) setFinalizedCheckpoint(finality *v1.Finality) { cs.finalityMutex.Lock() if cs.finality != nil && finality.Justified.Epoch <= cs.finality.Justified.Epoch && finality.Finalized.Epoch <= cs.finality.Finalized.Epoch { @@ -300,7 +416,7 @@ func (cs *ChainState) GetFinalizedSlot() phase0.Slot { } func (cs *ChainState) CurrentSlot() phase0.Slot { - return cs.TimeToSlot(time.Now()) + return cs.TimeToSlot(cs.Now()) } func (cs *ChainState) CurrentEpoch() phase0.Epoch { diff --git a/clients/consensus/replayclock.go b/clients/consensus/replayclock.go new file mode 100644 index 00000000..04c98bbb --- /dev/null +++ b/clients/consensus/replayclock.go @@ -0,0 +1,166 @@ +package consensus + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +const ( + // defaultReplayPollInterval is how often the virtual clock is refreshed from the + // control server. Between polls the clock is interpolated locally from the last + // reported rate, so a coarse interval does not make the clock jumpy. + defaultReplayPollInterval = 100 * time.Millisecond + + // replayConnectTimeout bounds how long EnableReplayClock waits for the control + // server to become reachable before giving up. + replayConnectTimeout = 2 * time.Minute +) + +// replayClockState is the payload served by a dora-replay control server at +// GET /replay/clock. Rate is the number of virtual milliseconds that pass per real +// millisecond; it is 0 while the replay is paused. +type replayClockState struct { + TimeMs int64 `json:"time_ms"` + Rate float64 `json:"rate"` +} + +// replayClock mirrors the virtual clock of a dora-replay control server, so the whole +// explorer sees a simulated "now" while a past slot range is stepped through. +type replayClock struct { + ctx context.Context + logger logrus.FieldLogger + url string + client *http.Client + + mutex sync.RWMutex + anchorReal time.Time + anchorVirt time.Time + rate float64 +} + +// newReplayClock connects to the control server and blocks until the first clock +// state has been read, so callers never observe an uninitialized virtual time. +func newReplayClock(ctx context.Context, logger logrus.FieldLogger, controlURL string, pollInterval time.Duration) (*replayClock, error) { + if controlURL == "" { + return nil, fmt.Errorf("no replay control url configured") + } + + if pollInterval <= 0 { + pollInterval = defaultReplayPollInterval + } + + clock := &replayClock{ + ctx: ctx, + logger: logger, + url: strings.TrimSuffix(controlURL, "/") + "/replay/clock", + client: &http.Client{Timeout: 10 * time.Second}, + } + + if err := clock.awaitFirstPoll(); err != nil { + return nil, err + } + + go clock.runPollLoop(pollInterval) + + return clock, nil +} + +// awaitFirstPoll retries the control server until it answers or the connect timeout +// expires. The explorer cannot compute any slot before this succeeds. +func (c *replayClock) awaitFirstPoll() error { + deadline := time.Now().Add(replayConnectTimeout) + lastLog := time.Time{} + + for { + err := c.poll() + if err == nil { + return nil + } + + if time.Now().After(deadline) { + return fmt.Errorf("could not reach replay control server at %v: %w", c.url, err) + } + + if time.Since(lastLog) > 5*time.Second { + c.logger.WithError(err).Warnf("waiting for replay control server at %v", c.url) + lastLog = time.Now() + } + + select { + case <-c.ctx.Done(): + return c.ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + +func (c *replayClock) runPollLoop(pollInterval time.Duration) { + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + if err := c.poll(); err != nil { + c.logger.WithError(err).Debugf("failed polling replay clock") + } + } + } +} + +func (c *replayClock) poll() error { + ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url, http.NoBody) + if err != nil { + return err + } + + rsp, err := c.client.Do(req) + if err != nil { + return err + } + defer func() { _ = rsp.Body.Close() }() + + if rsp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status %v", rsp.StatusCode) + } + + state := replayClockState{} + if err := json.NewDecoder(rsp.Body).Decode(&state); err != nil { + return fmt.Errorf("error parsing replay clock response: %w", err) + } + + c.mutex.Lock() + c.anchorReal = time.Now() + c.anchorVirt = time.UnixMilli(state.TimeMs) + c.rate = state.Rate + c.mutex.Unlock() + + return nil +} + +// now returns the current virtual time, interpolated from the last polled state so +// the clock keeps moving smoothly between polls while the replay is playing. +func (c *replayClock) now() time.Time { + c.mutex.RLock() + defer c.mutex.RUnlock() + + if c.rate == 0 { + return c.anchorVirt + } + + elapsed := time.Since(c.anchorReal) + + return c.anchorVirt.Add(time.Duration(float64(elapsed) * c.rate)) +} diff --git a/cmd/dora-explorer/main.go b/cmd/dora-explorer/main.go index c36dabc3..b9fad57e 100644 --- a/cmd/dora-explorer/main.go +++ b/cmd/dora-explorer/main.go @@ -63,6 +63,14 @@ func main() { services.InitChainService(ctx, logger) + if cfg.Replay.Enabled { + chainState := services.GlobalBeaconService.GetChainState() + err = chainState.EnableReplayClock(ctx, logger.WithField("service", "replay-clock"), cfg.Replay.ControlUrl, cfg.Replay.PollInterval) + if err != nil { + logger.Fatalf("error connecting to replay control server: %v", err) + } + } + var webserver *http.Server if cfg.Frontend.Enabled || cfg.Api.Enabled { websrv, err := startWebserver(logger) diff --git a/cmd/dora-replay/main.go b/cmd/dora-replay/main.go new file mode 100644 index 00000000..60cd0240 --- /dev/null +++ b/cmd/dora-replay/main.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/ethpandaops/dora/replay" +) + +func main() { + if err := newRootCmd().Execute(); err != nil { + os.Exit(1) + } +} + +func newRootCmd() *cobra.Command { + cfg := replay.DefaultConfig() + + var ( + startEpoch uint64 + logLevel string + noBids bool + ) + + cmd := &cobra.Command{ + Use: "dora-replay", + Short: "Replay a past slot range through Dora as if it were happening live", + Long: "dora-replay serves a fake beacon/execution node pair backed by a real upstream and\n" + + "drives a virtual clock, so an explorer pointed at it steps through a past slot range\n" + + "with pause, step, seek and play. Point dora at the two listeners and set\n" + + "`replay.enabled: true` with `replay.controlUrl` pointing at the control listener.", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + logger := logrus.New() + logger.SetOutput(os.Stderr) + + level, err := logrus.ParseLevel(logLevel) + if err != nil { + return fmt.Errorf("invalid log level %q: %w", logLevel, err) + } + + logger.SetLevel(level) + + if startEpoch > 0 && cfg.StartSlot == 0 { + slotsPerEpoch, err := replay.SlotsPerEpoch(cmd.Context(), cfg.UpstreamURL) + if err != nil { + return err + } + + cfg.StartSlot = startEpoch * slotsPerEpoch + } + + cfg.EmitBids = !noBids + + return run(cmd.Context(), logger, cfg) + }, + } + + flags := cmd.Flags() + flags.StringVar(&cfg.UpstreamURL, "upstream", "", "beacon node HTTP API to read the chain from (required, must serve historical states)") + flags.StringVar(&cfg.ExecutionURL, "el-upstream", "", "execution node JSON-RPC to read from; omit to run without an execution proxy") + flags.StringVar(&cfg.TracoorURL, "tracoor", "", "tracoor instance used as a fallback for artifacts the beacon node has pruned") + flags.StringVar(&cfg.TracoorNetwork, "tracoor-network", "", "network name to query tracoor with") + flags.StringVar(&cfg.CLListen, "cl-listen", cfg.CLListen, "listen address of the fake beacon node") + flags.StringVar(&cfg.ELListen, "el-listen", cfg.ELListen, "listen address of the fake execution node") + flags.StringVar(&cfg.ControlListen, "control-listen", cfg.ControlListen, "listen address of the control endpoint dora polls the clock from") + flags.Uint64Var(&cfg.StartSlot, "start-slot", 0, "first slot to serve") + flags.Uint64Var(&startEpoch, "start-epoch", 0, "first epoch to serve (alternative to --start-slot)") + flags.Float64Var(&cfg.Speed, "speed", 0, "start playing at this multiple of real time; 0 starts paused") + flags.StringVar(&cfg.CacheDir, "cache-dir", "", "record fetched artifacts here so the range replays offline afterwards") + flags.DurationVar(&cfg.StepSettle, "step-settle", cfg.StepSettle, "real time to pause at each phase of a stepped slot") + flags.DurationVar(&cfg.StateHoldTimeout, "state-hold-timeout", cfg.StateHoldTimeout, "how long to freeze the clock waiting for the explorer to load a beacon state") + flags.BoolVar(&noBids, "no-bids", false, "do not replay execution payload bids (saves one block fetch per slot)") + flags.StringVar(&logLevel, "log-level", "info", "log level (trace, debug, info, warn, error)") + + return cmd +} + +func run(ctx context.Context, logger logrus.FieldLogger, cfg replay.Config) error { + ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer cancel() + + instance, err := replay.New(ctx, logger, cfg) + if err != nil { + return err + } + + if err := instance.Start(ctx); err != nil { + return err + } + + in, out := replay.Stdio() + instance.RunConsole(ctx, in, out) + + if err := instance.Stop(); err != nil { + logger.WithError(err).Warn("error shutting down replay") + } + + return nil +} diff --git a/go.mod b/go.mod index a3a68e6a..df995ebf 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/jackc/pgx/v4 v4.18.3 github.com/jmoiron/sqlx v1.4.0 github.com/kelseyhightower/envconfig v1.4.0 + github.com/klauspost/compress v1.19.1 github.com/lib/pq v1.12.3 github.com/libp2p/go-libp2p v0.43.0 github.com/mashingan/smapping v0.1.19 @@ -85,7 +86,6 @@ require ( github.com/ipfs/go-log/v2 v2.8.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.19.1 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/koron/go-ssdp v0.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/handlers/index.go b/handlers/index.go index 885260b0..92166617 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -66,7 +66,7 @@ func IndexData(w http.ResponseWriter, r *http.Request) { return } // stamp live server time outside of cached page data so clients can correct local clock drift - w.Header().Set("X-Server-Time", strconv.FormatInt(time.Now().UnixMilli(), 10)) + w.Header().Set("X-Server-Time", strconv.FormatInt(services.GlobalBeaconService.GetChainState().Now().UnixMilli(), 10)) w.Header().Set("Content-Type", "application/json") err := json.NewEncoder(w).Encode(pageData) if err != nil { diff --git a/handlers/pageData.go b/handlers/pageData.go index 33b27d34..dad15349 100644 --- a/handlers/pageData.go +++ b/handlers/pageData.go @@ -38,6 +38,8 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st siteDomain = r.Host } + chainState := services.GlobalBeaconService.GetChainState() + data := &types.PageData{ Meta: &types.Meta{ Title: fullTitle, @@ -50,7 +52,7 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st Data: &types.Empty{}, Version: utils.GetExplorerVersion(), BuildTime: fmt.Sprintf("%v", buildTime.Unix()), - ServerTime: time.Now().UnixMilli(), + ServerTime: chainState.Now().UnixMilli(), Year: time.Now().UTC().Year(), ExplorerTitle: utils.Config.Frontend.SiteName, ExplorerSubtitle: utils.Config.Frontend.SiteSubtitle, @@ -61,9 +63,9 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st ApiEnabled: utils.Config.Api.Enabled && !utils.Config.Api.RequireAuth, ExecutionIndexerEnabled: utils.Config.ExecutionIndexer.Enabled, EnsSearchEnabled: ensSearchEnabled(), + ReplayControlUrl: replayControlURL(), } - chainState := services.GlobalBeaconService.GetChainState() if specs := chainState.GetSpecs(); specs != nil { data.IsReady = true data.ChainSlotsPerEpoch = specs.SlotsPerEpoch @@ -94,6 +96,16 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st return data } +// replayControlURL returns the dora-replay control server, or "" when the explorer is +// not running a replay. +func replayControlURL() string { + if !utils.Config.Replay.Enabled { + return "" + } + + return strings.TrimSuffix(utils.Config.Replay.ControlUrl, "/") +} + func createMenuItems(active string) []types.MainMenuItem { chainState := services.GlobalBeaconService.GetChainState() specs := chainState.GetSpecs() diff --git a/indexer/beacon/epochcache.go b/indexer/beacon/epochcache.go index add1d0bb..30a1fade 100644 --- a/indexer/beacon/epochcache.go +++ b/indexer/beacon/epochcache.go @@ -144,8 +144,9 @@ func (cache *epochCache) getPendingEpochStats() []*EpochStats { pendingStats := make([]*EpochStats, 0) for _, stats := range cache.statsMap { - if stats.dependentState != nil && stats.dependentState.loadingStatus == 0 { - if loadingRoots[stats.dependentState.slotRoot] { + dependentState := stats.dependentState + if dependentState != nil && dependentState.loadingStatus == 0 { + if loadingRoots[dependentState.slotRoot] { continue // another epochState with same root is already loading } pendingStats = append(pendingStats, stats) @@ -238,9 +239,9 @@ func (cache *epochCache) removeEpochStats(epochStats *EpochStats) { delete(cache.statsMap, statsKey) - if epochStats.dependentState != nil { + if dependentState := epochStats.dependentState; dependentState != nil { stateKey := getEpochStatsKey(epochStats.epoch, epochStats.dependentRoot) - epochStats.dependentState.dispose() + dependentState.dispose() delete(cache.stateMap, stateKey) } } @@ -393,6 +394,14 @@ func (cache *epochCache) runLoaderLoop() { func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { defer utils.HandleSubroutinePanic("indexer.beacon.epochCache.loadEpochStats", nil) + // pruning clears dependentState on epochs that leave the in-memory window, and it + // can do so while this load is in flight. Work from one read of the field so a + // prune halfway through cannot turn a later use into a nil dereference. + dependentState := epochStats.dependentState + if dependentState == nil { + return false + } + clients := []*Client{} preferArchive := epochStats.epoch < cache.indexer.lastFinalizedEpoch for _, client := range cache.indexer.GetReadyClientsByBlockRoot(epochStats.dependentRoot, preferArchive) { @@ -427,7 +436,7 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { if len(clients) == 0 { cache.indexer.logger.Debugf("no clients available to load epoch %v stats (dep: %v)", epochStats.epoch, epochStats.dependentRoot.String()) - epochStats.dependentState.retryCount++ + dependentState.retryCount++ return false } @@ -456,25 +465,25 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { return bytes.Compare(hashA[:], hashB[:]) < 0 }) - client := clients[int(epochStats.dependentState.retryCount)%len(clients)] + client := clients[int(dependentState.retryCount)%len(clients)] log := cache.indexer.logger.WithField("client", client.client.GetName()) - if epochStats.dependentState.retryCount > 0 { - log = log.WithField("retry", epochStats.dependentState.retryCount) + if dependentState.retryCount > 0 { + log = log.WithField("retry", dependentState.retryCount) } log.Infof("loading epoch %v stats (dep: %v, req: %v)", epochStats.epoch, epochStats.dependentRoot.String(), len(epochStats.requestedBy)) t1 := time.Now() - state, err := epochStats.dependentState.loadState(client.getContext(), client, cache) - if err != nil && epochStats.dependentState.loadingStatus == 0 { + state, err := dependentState.loadState(client.getContext(), client, cache) + if err != nil && dependentState.loadingStatus == 0 { client.logger.Warnf("failed loading epoch %v stats (dep: %v): %v", epochStats.epoch, epochStats.dependentRoot.String(), err) } loadDuration := time.Since(t1) - if epochStats.dependentState.loadingStatus != 2 { + if dependentState.loadingStatus != 2 { // epoch state could not be loaded - epochStats.dependentState.retryCount++ + dependentState.retryCount++ return false } @@ -501,14 +510,16 @@ func (cache *epochCache) loadEpochStats(epochStats *EpochStats) bool { if stats == epochStats { continue } - if stats.dependentState == nil || stats.dependentState.loadingStatus != 0 { + // same read-once rule as above: pruning may clear this concurrently + pendingState := stats.dependentState + if pendingState == nil || pendingState.loadingStatus != 0 { continue } - if stats.dependentState.slotRoot != epochStats.dependentState.slotRoot { + if pendingState.slotRoot != dependentState.slotRoot { continue } pendingOthers = append(pendingOthers, pendingEntry{ - epochState: stats.dependentState, + epochState: pendingState, stats: stats, }) } diff --git a/replay/README.md b/replay/README.md new file mode 100644 index 00000000..08ea3a2e --- /dev/null +++ b/replay/README.md @@ -0,0 +1,264 @@ +# dora-replay + +Step a past slot range through Dora as if it were happening live. + +`dora-replay` serves a fake beacon node and a fake execution node backed by a real +upstream, and drives a virtual clock that the explorer follows. A past slot range then +unfolds slot by slot with pause, step, seek and play, while the normal Dora UI stays +live and inspectable — the same indexer, service and handler code paths that run against +a real chain. + +``` + ┌───────────────┐ ┌──────────────────────┐ + dora-explorer ──►│ dora-replay │──────► │ beacon node (archive)│ + (replay: on) │ CL :15052 │ │ execution node │ + ▲ │ EL :15545 │──────► │ tracoor (states) │ + └───clock───│ ctl :15000 │ └──────────────────────┘ + └───────────────┘ + replay> step 32 +``` + +## Running one + +```sh +go build -o bin/dora-replay ./cmd/dora-replay + +./bin/dora-replay \ + --upstream https://user:pass@bn-archive-1.example \ + --el-upstream https://user:pass@rpc-1.example \ + --tracoor https://tracoor.example \ + --tracoor-network my-devnet \ + --start-slot 48000 \ + --cache-dir ./temp/replay-cache +``` + +Then start Dora with a config whose `beaconapi`/`executionapi` endpoints point at the +two listeners and whose `replay` block points at the control listener: + +```yaml +replay: + enabled: true + controlUrl: "http://127.0.0.1:15000" + +beaconapi: + endpoints: + - url: "http://127.0.0.1:15052" + name: "replay-cl" + +executionapi: + endpoints: + - url: "http://127.0.0.1:15545" + name: "replay-el" +``` + +`replay/example-config.yaml` is a complete working example. + +## Controlling it from the explorer + +With `replay.controlUrl` set, the explorer side-loads a control panel from the replay +process and hangs it off the header: a collapsed pill showing the replayed epoch and +what the replay is doing, expanding into live state and controls. + +``` + ● REPLAY epoch 1442 · playing 6x ▼ + ┌──────────────────────────────────────────────┐ + │ Slot 46154 → 48960 │ + │ Epoch 1442 of 1624 upstream │ + │ Time 2026-08-19 15:50:48Z │ + │ Head 46154 0x28a442f2… │ + │ Checkpoints justified 1440 · finalized 1439 │ + │ EL block 45376 │ + │ ▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░│░░░░░░░░░░░░░░░░░░░░░░░ │ + │ from 44800 chain head 51968 │ + │ [ Stop ] [+1] [+1 epoch] [ 6x ▾ ] │ + │ [ 48960 ] [ Forward to ] │ + └──────────────────────────────────────────────┘ +``` + +The explorer's whole share of this is one template block that sets +`window.doraReplayApi` and loads `/replay/ui.js`. The UI itself — markup, +styling, API calls — is embedded in the replay binary and talks to its control API +directly, so it can be changed without rebuilding or restarting the explorer. + +The panel also retunes how often the explorer's polling pages refresh, by setting +`window.doraIndexRefreshInterval` from the replay's current pace. At 16x a slot goes by +in well under a second, and the stock 15-second refresh would leave the index page +several epochs behind what the replay is actually showing; while paused it falls back to +the stock interval. + +## Control API + +Served on `--control-listen`, with CORS open so the browser can reach it from the +explorer's origin. + +| Endpoint | Purpose | +|---|---| +| `GET /replay/clock` | `{time_ms, rate}` — the virtual clock the explorer follows | +| `GET /replay/status` | full state snapshot (see below) | +| `GET /replay/events` | SSE stream of `status` events, pushed on every change | +| `POST /replay/command` | drive the replay; returns the new status | +| `GET /replay/ui.js` | the side-loaded control panel | + +Commands mirror the console: + +```jsonc +{"action": "play", "speed": 6} // speed 0 = as fast as upstream allows +{"action": "speed", "speed": 4} // change the rate without starting or stopping +{"action": "step", "slots": 32} +{"action": "forward", "slot": 48960, "speed": 6} +{"action": "stop"} +{"action": "start"} // resume, keeping the speed and any target +``` + +The status carries the replayed position (`virtual_slot`, `virtual_epoch`, +`virtual_time`, `rate`), the chain state at that point (`head_slot`, `head_root`, +`justified_epoch`, `finalized_epoch`, `execution_block`), the drive state (`running`, +`speed`, `start_slot`, `target_slot`) and the chain context needed to render it +(`upstream_slot`/`upstream_epoch` — the head of the *real* chain, re-read every 30s — +plus `genesis_time`, `slot_duration_ms`, `slots_per_epoch`). + +Status events are lossy on purpose: a snapshot that a stalled browser tab did not read +is dropped rather than queued, so the UI can never hold the replay up. Chain events on +the fake beacon node are the opposite — never dropped, see below. + +## The console + +``` +replay> status + slot 48160 epoch 1505 head 48160 [0x513df450…] + justified 1503 finalized 1502 el block 47370 + playing 6x time 2026-08-20T04:32:08Z streams 2 + upstream https://eth:xxxxx@bn-grandine-besu-1.example (+tracoor) + +replay> step # advance one slot, then pause +replay> step 32 # advance an epoch +replay> forward 48960 # run to a slot as fast as upstream allows +replay> forward 48960 6x # run to a slot at 6x real time +replay> play 4x # run on without a target +replay> stop # pause; the virtual clock freezes where it is +replay> start # resume, keeping the speed and any target +replay> quit +``` + +Pick the speed from what the explorer has to keep up with. Stepping runs as fast as the +upstream answers, which is far quicker than the explorer can index; a `play`/`forward` +speed gives it a fixed budget per slot instead. On a 12-second-slot devnet, 6x leaves +two real seconds per slot, which is comfortable for block indexing and epoch stats. + +## Waiting for state loads + +Once per epoch the explorer pulls a full beacon state — tens of megabytes to fetch, +decompress and decode, during which it cannot process anything else. Nothing is lost +when that happens (every link from the replay to the indexer applies backpressure rather +than dropping), but the virtual clock would otherwise keep running, so by the time the +state arrived the explorer would be several slots behind the slot the replay claims to +be at. + +So halfway through every slot the replay checks whether a state is still on its way to +the explorer. It does **not** stop there: the explorer indexes blocks on a different +goroutine than it loads states on, so a read of several seconds does not stop it from +processing blocks, and holding the replay outright would idle its block indexer and emit +no events at all for the duration. The replay is allowed to serve a few more slots +(`stateLoadLeadSlots`) while a read is in flight, and only **freezes the clock** once it +would get further ahead than that. + +Once it holds, the wait costs real time and no virtual time: the explorer's own clock +mirrors the frozen rate, so it stays exactly where the replay left it instead of +drifting. The console and the control panel both report this as +`holding for N state load(s)`. + +A read that never finishes cannot wedge a run — the gate gives up after +`--state-hold-timeout` (default 2m) and logs a warning. + +## Upstreams + +**Beacon node** (`--upstream`, required) answers everything: genesis, specs, headers, +blocks, blob sidecars, payload envelopes, finality. Point it at **one** node that keeps +historical data, not at a load balancer — a fan-out over mixed-retention backends fails +intermittently mid-replay. Nodes differ a lot here: many prune all but the most recent +states even when they are labelled "archive". + +**Tracoor** (`--tracoor`, optional but recommended) keeps every beacon state and block it +sampled, addressed by root. **States** are read from it first: it has the depth a replay +needs, where beacon nodes prune all but the most recent, and it spares the devnet from +serving a ~17 MB state per epoch. **Blocks are not** — the node still has every block of +the replayed range and answers in one round trip, where a tracoor read costs a lookup +plus a download; tracoor only backs blocks up when the node 404s one. It stores SSZ +only, so a JSON-only client falls back to the node. A tracoor-only run is rejected — +tracoor cannot resolve "the block at slot N". + +**Execution node** (`--el-upstream`, optional) backs the JSON-RPC proxy. Without it the +execution proxy is not started and the explorer runs consensus-only. + +**Cache** (`--cache-dir`, optional) records every immutable artifact fetched, so a range +is downloaded once and replays offline and reproducibly afterwards. + +## What the fake nodes do + +The consensus proxy passes everything through except where the virtual head matters: + +* `/eth/v1/node/syncing` is synthesized: head at the replayed slot, never syncing +* `head`, `finalized` and `justified` identifiers resolve to the roots the chain had at + the virtual head, not the ones the upstream has today +* any slot beyond the virtual head answers 404, the same as a node that has not seen it +* `/eth/v1/beacon/states/head/finality_checkpoints` is answered from the finality the + replay tracks — reading it upstream would force a historical state load +* `/eth/v1/events` is generated locally: `execution_payload_bid` → `block` → `head` → + `execution_payload_available`, plus `finalized_checkpoint` when it moves + +The execution proxy tracks a virtual head derived from block timestamps (a payload's +timestamp is its slot's time, so the execution head follows the consensus head exactly): + +* `latest`/`safe`/`finalized` are pinned to the virtual head, and a numeric block beyond + it answers as if the node did not have it yet +* `eth_getLogs` is clamped to the virtual head +* `eth_newBlockFilter`/`eth_getFilterChanges`/`eth_uninstallFilter` are served locally, + handing out the blocks that appeared since the last poll +* everything else is forwarded, with batches forwarded as batches + +## Dora's side + +Everything the explorer knows about "now" runs through `ChainState`. In replay mode it +polls `GET /replay/clock` and interpolates between polls, `CurrentSlot()` follows that +virtual time, and a small ticker replaces `ethwallclock` (which reads `time.Now()` +internally and cannot be paused) to fire the slot and epoch dispatchers. Genesis stays +the real one, so `SlotToTime` keeps returning true historical timestamps — only "now" +moves. Page timestamps use the same clock, so "x ago" is computed against the simulated +present. + +The only other thing Dora contributes is one block in the page layout, which sets +`window.doraReplayApi` and side-loads `/replay/ui.js`. Everything the +control panel is lives in the replay binary. + +That is the whole change on Dora's side. No indexer, service or handler code is aware of +the replay, which is the point: the run has to exercise the real code paths. + +## Starting mid-chain + +A replay starts in the middle of the chain, so use a fresh database, `blockdb`, +`statecache` and pubkey-cache path per run. Dora takes its finalized epoch from the +chain, so it will not try to finalize from epoch 0 — but the synchronizer starts from +whatever `indexer.syncstate` says. Two options: + +* `indexer.disableSynchronizer: true` — index only from the start slot forward. Fast to + get going; everything before the start slot is simply absent. +* leave the synchronizer on — it backfills epochs 0..start in the background while the + replay steps forward, ending with a complete database. Much slower, but the result can + be copied and reused as the starting point for later runs. + +## Fidelity limits + +* **Reorgs are not replayed.** The proxy serves the canonical chain as it exists today, + so blocks that were briefly head during the original run never appear and fork handling + is not exercised. +* **Intra-slot timing is approximated.** Events are emitted at fixed points in the slot + (block at ⅓, payload at ⅔) rather than when they actually arrived, so anything + timing-sensitive within a slot will not reproduce. +* **Real timers keep running.** Dora's retry and backoff timers run on real time, so a + long pause still logs client-health retries. Client readiness is not wallclock-gated, + so pausing is otherwise safe. +* **Execution head is inferred** from block timestamps rather than observed, so a slot + whose payload was never revealed shows up as "no new execution block" — which matches + reality, but by derivation. +* **Historical execution state** (`eth_getBalance` at an old block) needs an archive + execution node; a full node only keeps a shallow window. diff --git a/replay/assets/replay-ui.js b/replay/assets/replay-ui.js new file mode 100644 index 00000000..9f6ff4c4 --- /dev/null +++ b/replay/assets/replay-ui.js @@ -0,0 +1,407 @@ +// Replay control UI. +// +// This file is served by the dora-replay process and side-loaded into the explorer, so +// the explorer itself carries nothing but a script tag. Everything here talks to the +// replay's own control API on the origin this script came from. +(function () { + "use strict"; + + // the explorer hands us the control server's address; deriving it from this + // script's own URL would depend on document.currentScript, which is not something + // to rely on across browsers and script loading modes + var base = (window.doraReplayApi || "").replace(/\/+$/, ""); + if (!base) { + return; + } + + var STORAGE_KEY = "dora-replay-open"; + + // how often pages that poll for updates should refresh, relative to how fast the + // replay is producing slots + var REFRESH_MIN_MS = 1000; + var REFRESH_MAX_MS = 15000; + var REFRESH_STEPPING_MS = 2000; + var SPEEDS = [ + { value: 0, label: "max" }, + { value: 0.5, label: "0.5x" }, + { value: 1, label: "1x" }, + { value: 2, label: "2x" }, + { value: 4, label: "4x" }, + { value: 8, label: "8x" }, + { value: 16, label: "16x" }, + { value: 32, label: "32x" }, + ]; + + var CSS = [ + ".replay-callout{position:fixed;top:70px;right:16px;z-index:1010;max-width:calc(100vw - 32px);font-size:.8125rem}", + ".replay-toggle{display:flex;align-items:center;gap:.5rem;margin-left:auto;padding:.25rem .75rem;border:1px solid var(--bs-border-color,#dee2e6);border-top:none;border-radius:0 0 .5rem .5rem;background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);box-shadow:0 .25rem .75rem rgba(0,0,0,.15);cursor:pointer;white-space:nowrap}", + ".replay-toggle:hover{background:var(--bs-tertiary-bg,#f8f9fa)}", + ".replay-tag{font-weight:600;letter-spacing:.04em;font-size:.6875rem;text-transform:uppercase;color:var(--bs-warning-text-emphasis,#997404)}", + ".replay-dot{width:.5rem;height:.5rem;border-radius:50%;background:var(--bs-secondary,#6c757d);flex:none}", + ".replay-dot.is-playing{background:var(--bs-success,#198754);animation:replay-pulse 1.6s ease-in-out infinite}", + ".replay-dot.is-stepping{background:var(--bs-info,#0dcaf0)}", + ".replay-dot.is-waiting{background:var(--bs-warning,#ffc107);animation:replay-pulse 1s ease-in-out infinite}", + ".replay-dot.is-offline{background:var(--bs-danger,#dc3545)}", + "@keyframes replay-pulse{0%,100%{opacity:1}50%{opacity:.35}}", + ".replay-chevron{transition:transform .15s ease;font-size:.625rem}", + ".replay-callout.is-open .replay-chevron{transform:rotate(180deg)}", + ".replay-panel{display:none;width:23rem;max-width:calc(100vw - 32px);margin-top:.375rem;padding:.75rem;border:1px solid var(--bs-border-color,#dee2e6);border-radius:.5rem;background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);box-shadow:0 .5rem 1.5rem rgba(0,0,0,.2)}", + ".replay-callout.is-open .replay-panel{display:block}", + ".replay-grid{display:grid;grid-template-columns:auto 1fr;gap:.125rem .75rem;margin:0 0 .625rem}", + ".replay-grid dt{color:var(--bs-secondary-color,#6c757d);font-weight:400}", + ".replay-grid dd{margin:0;font-variant-numeric:tabular-nums;overflow:hidden;text-overflow:ellipsis}", + ".replay-track{height:.375rem;border-radius:.25rem;background:var(--bs-tertiary-bg,#e9ecef);overflow:hidden;position:relative}", + ".replay-track-fill{height:100%;background:var(--bs-primary,#0d6efd);width:0;transition:width .2s linear}", + ".replay-track-target{position:absolute;top:0;bottom:0;width:2px;background:var(--bs-warning,#ffc107);display:none}", + ".replay-range{display:flex;justify-content:space-between;color:var(--bs-secondary-color,#6c757d);font-size:.6875rem;margin:.25rem 0 .625rem}", + ".replay-row{display:flex;flex-wrap:wrap;gap:.375rem;align-items:center;margin-bottom:.5rem}", + ".replay-row .replay-btn,.replay-row select,.replay-row input{font-size:.8125rem;padding:.1875rem .5rem;border-radius:.25rem;border:1px solid var(--bs-border-color,#dee2e6);background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529)}", + ".replay-row .replay-btn{cursor:pointer}", + ".replay-row .replay-btn:hover:not(:disabled){background:var(--bs-tertiary-bg,#f8f9fa)}", + ".replay-row .replay-btn:disabled{opacity:.5;cursor:default}", + ".replay-btn-main{background:var(--bs-primary,#0d6efd)!important;border-color:var(--bs-primary,#0d6efd)!important;color:#fff!important;min-width:4.5rem}", + ".replay-seek-slot{flex:1;min-width:5rem;font-variant-numeric:tabular-nums}", + ".replay-source{color:var(--bs-secondary-color,#6c757d);font-size:.6875rem;word-break:break-all}", + ".replay-error{display:none;margin-top:.5rem;padding:.25rem .5rem;border-radius:.25rem;background:var(--bs-danger-bg-subtle,#f8d7da);color:var(--bs-danger-text-emphasis,#842029);font-size:.6875rem}", + ".replay-error.is-shown{display:block}", + "@media (max-width:575.98px){.replay-callout{right:8px;left:8px}.replay-panel{width:100%}}", + ].join(""); + + var state = null; + var stateReadAt = 0; + var connected = false; + var el = {}; + + function h(tag, className, text) { + var node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; + } + + function field(grid, label) { + grid.appendChild(h("dt", null, label)); + var value = h("dd", null, "-"); + grid.appendChild(value); + return value; + } + + function build() { + var style = h("style"); + style.textContent = CSS; + document.head.appendChild(style); + + var root = h("div", "replay-callout"); + + var toggle = h("button", "replay-toggle"); + toggle.type = "button"; + el.dot = h("span", "replay-dot"); + el.summary = h("span", "replay-summary", "connecting…"); + toggle.appendChild(el.dot); + toggle.appendChild(h("span", "replay-tag", "replay")); + toggle.appendChild(el.summary); + toggle.appendChild(h("span", "replay-chevron", "▼")); + toggle.addEventListener("click", function () { + var open = !root.classList.contains("is-open"); + root.classList.toggle("is-open", open); + try { + window.localStorage.setItem(STORAGE_KEY, open ? "1" : "0"); + } catch (err) { + /* private mode */ + } + }); + root.appendChild(toggle); + + var panel = h("div", "replay-panel"); + + var grid = h("dl", "replay-grid"); + el.slot = field(grid, "Slot"); + el.epoch = field(grid, "Epoch"); + el.time = field(grid, "Time"); + el.head = field(grid, "Head"); + el.checkpoints = field(grid, "Checkpoints"); + el.execution = field(grid, "EL block"); + panel.appendChild(grid); + + var track = h("div", "replay-track"); + el.fill = h("div", "replay-track-fill"); + el.marker = h("div", "replay-track-target"); + track.appendChild(el.fill); + track.appendChild(el.marker); + panel.appendChild(track); + + var range = h("div", "replay-range"); + el.rangeFrom = h("span", null, ""); + el.rangeTo = h("span", null, ""); + range.appendChild(el.rangeFrom); + range.appendChild(el.rangeTo); + panel.appendChild(range); + + var controls = h("div", "replay-row"); + el.toggleRun = button("Start", "replay-btn replay-btn-main", onToggleRun); + el.step1 = button("+1", "replay-btn", function () { + send({ action: "step", slots: 1 }); + }); + el.stepEpoch = button("+1 epoch", "replay-btn", function () { + send({ action: "step", slots: (state && state.slots_per_epoch) || 32 }); + }); + el.speed = h("select"); + SPEEDS.forEach(function (speed) { + var option = h("option", null, speed.label); + option.value = String(speed.value); + el.speed.appendChild(option); + }); + el.speed.addEventListener("change", function () { + send({ action: "speed", speed: parseFloat(el.speed.value) }); + }); + controls.appendChild(el.toggleRun); + controls.appendChild(el.step1); + controls.appendChild(el.stepEpoch); + controls.appendChild(el.speed); + panel.appendChild(controls); + + var seek = h("div", "replay-row"); + el.seekSlot = h("input", "replay-seek-slot"); + el.seekSlot.type = "number"; + el.seekSlot.placeholder = "slot"; + el.seekSlot.addEventListener("keydown", function (event) { + if (event.key === "Enter") onSeek(); + }); + seek.appendChild(el.seekSlot); + seek.appendChild(button("Forward to", "replay-btn", onSeek)); + panel.appendChild(seek); + + el.source = h("div", "replay-source", ""); + panel.appendChild(el.source); + + el.error = h("div", "replay-error"); + panel.appendChild(el.error); + + root.appendChild(panel); + document.body.appendChild(root); + el.root = root; + + try { + if (window.localStorage.getItem(STORAGE_KEY) === "1") { + root.classList.add("is-open"); + } + } catch (err) { + /* private mode */ + } + } + + function button(label, className, handler) { + var node = h("button", className, label); + node.type = "button"; + node.addEventListener("click", handler); + return node; + } + + function onToggleRun() { + if (!state) return; + send(state.running ? { action: "stop" } : { action: "play", speed: parseFloat(el.speed.value) }); + } + + function onSeek() { + var slot = parseInt(el.seekSlot.value, 10); + if (!isFinite(slot)) { + showError("enter a slot to run to"); + return; + } + send({ action: "forward", slot: slot, speed: parseFloat(el.speed.value) }); + } + + function send(command) { + showError(""); + + fetch(base + "/replay/command", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(command), + }) + .then(function (response) { + return response.json().then(function (body) { + if (!response.ok) throw new Error(body.message || "command failed"); + return body; + }); + }) + .then(apply) + .catch(function (err) { + showError(err.message || String(err)); + }); + } + + function showError(message) { + el.error.textContent = message; + el.error.classList.toggle("is-shown", !!message); + } + + // tuneRefresh retunes the explorer's polling pages to the replay's pace. At 16x a + // slot goes by in well under a second, so the stock 15s refresh would leave the + // index page many epochs behind what the replay is actually showing. + function tuneRefresh(status) { + var interval = REFRESH_MAX_MS; + + if (status.running) { + interval = status.rate > 0 + ? (status.slot_duration_ms || 12000) / status.rate + : REFRESH_STEPPING_MS; + } + + window.doraIndexRefreshInterval = Math.max(REFRESH_MIN_MS, Math.min(REFRESH_MAX_MS, interval)); + } + + function apply(status) { + state = status; + stateReadAt = Date.now(); + + tuneRefresh(status); + + // only follow the server's speed when the user is not mid-selection + if (document.activeElement !== el.speed) { + var speed = String(status.speed); + if (!SPEEDS.some(function (entry) { return String(entry.value) === speed; })) { + var extra = h("option", null, speed + "x"); + extra.value = speed; + el.speed.appendChild(extra); + } + el.speed.value = speed; + } + + render(); + } + + // virtualNow interpolates the replay clock between status updates, so a playing + // replay shows a moving clock rather than stepping once per event. + function virtualNow() { + if (!state) return null; + var anchor = Date.parse(state.virtual_time); + if (isNaN(anchor)) return null; + if (!state.rate) return anchor; + return anchor + (Date.now() - stateReadAt) * state.rate; + } + + function slotAt(timeMs) { + if (!state || !state.slot_duration_ms) return state ? state.virtual_slot : 0; + var genesis = Date.parse(state.genesis_time); + if (isNaN(genesis) || timeMs < genesis) return state.virtual_slot; + return Math.floor((timeMs - genesis) / state.slot_duration_ms); + } + + function mode() { + if (!connected) return { label: "disconnected", dot: "is-offline" }; + if (!state) return { label: "connecting…", dot: "" }; + if (!state.running) return { label: "paused", dot: "" }; + if (state.holding) { + return { label: "loading state" + (state.state_loads > 1 ? " (" + state.state_loads + ")" : ""), dot: "is-waiting" }; + } + if (state.speed > 0) return { label: "playing " + trim(state.speed) + "x", dot: "is-playing" }; + if (state.target_slot) return { label: "stepping", dot: "is-stepping" }; + return { label: "max speed", dot: "is-stepping" }; + } + + function trim(value) { + return String(Math.round(value * 100) / 100); + } + + function num(value) { + return (value === undefined || value === null) ? "-" : value.toLocaleString("en-US"); + } + + function render() { + var status = mode(); + el.dot.className = "replay-dot " + status.dot; + + if (!state) { + el.summary.textContent = status.label; + return; + } + + var now = virtualNow(); + var slot = state.running && state.rate && !state.holding ? slotAt(now) : state.virtual_slot; + + el.summary.textContent = "epoch " + num(state.virtual_epoch) + " · " + status.label; + + el.slot.textContent = num(slot) + + (state.target_slot ? " → " + num(state.target_slot) : ""); + el.epoch.textContent = num(state.virtual_epoch) + + (state.upstream_epoch ? " of " + num(state.upstream_epoch) + " upstream" : ""); + el.time.textContent = now ? new Date(now).toISOString().replace("T", " ").replace(/\.\d+Z$/, "Z") : "-"; + el.head.textContent = num(state.head_slot) + + (state.head_root ? " " + state.head_root.slice(0, 10) + "…" : ""); + el.checkpoints.textContent = "justified " + num(state.justified_epoch) + + " · finalized " + num(state.finalized_epoch); + el.execution.textContent = num(state.execution_block); + + var from = state.start_slot || 0; + var to = state.upstream_slot || state.target_slot || slot; + var span = to - from; + el.fill.style.width = span > 0 ? Math.max(0, Math.min(100, ((slot - from) / span) * 100)) + "%" : "0"; + + if (state.target_slot && span > 0) { + el.marker.style.display = "block"; + el.marker.style.left = Math.max(0, Math.min(100, ((state.target_slot - from) / span) * 100)) + "%"; + } else { + el.marker.style.display = "none"; + } + + el.rangeFrom.textContent = "from " + num(from); + el.rangeTo.textContent = state.upstream_slot ? "chain head " + num(state.upstream_slot) : ""; + + el.toggleRun.textContent = state.running ? "Stop" : "Start"; + el.step1.disabled = false; + el.stepEpoch.disabled = false; + + el.source.textContent = state.upstream + (state.tracoor ? " (+tracoor)" : ""); + } + + function connect() { + var stream = new EventSource(base + "/replay/events?topics=status"); + + stream.addEventListener("open", function () { + connected = true; + render(); + }); + + stream.addEventListener("status", function (event) { + connected = true; + try { + apply(JSON.parse(event.data)); + } catch (err) { + /* ignore a malformed frame and wait for the next one */ + } + }); + + stream.addEventListener("error", function () { + connected = false; + render(); + // EventSource reconnects on its own + }); + } + + function start() { + build(); + + fetch(base + "/replay/status") + .then(function (response) { return response.json(); }) + .then(function (status) { + connected = true; + apply(status); + }) + .catch(function () { + connected = false; + render(); + }); + + connect(); + + // keep the interpolated clock and progress bar moving between status events + window.setInterval(render, 500); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", start); + } else { + start(); + } +})(); diff --git a/replay/cache.go b/replay/cache.go new file mode 100644 index 00000000..4c8fa994 --- /dev/null +++ b/replay/cache.go @@ -0,0 +1,108 @@ +package replay + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +// artifactCache records the immutable artifacts a replay fetched, so a slot range is +// downloaded from the devnet once and replays offline and reproducibly afterwards. +type artifactCache struct { + logger logrus.FieldLogger + dir string +} + +func newArtifactCache(logger logrus.FieldLogger, dir string) (*artifactCache, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("could not create cache dir %v: %w", dir, err) + } + + return &artifactCache{logger: logger, dir: dir}, nil +} + +// isImmutablePath reports whether a response for this path can never change. Only +// root-addressed artifacts qualify: slot-addressed ones would go stale across a reorg, +// and everything under /node or /config reflects the upstream's live state. +func isImmutablePath(path string) bool { + segments := strings.Split(strings.Trim(path, "/"), "/") + + for _, segment := range segments { + if strings.HasPrefix(segment, "0x") && len(segment) == 66 { + return true + } + } + + return false +} + +func (c *artifactCache) key(path, accept string) string { + sum := sha256.Sum256([]byte(path + "\x00" + accept)) + + return hex.EncodeToString(sum[:]) +} + +type cacheMeta struct { + Header http.Header `json:"header"` +} + +func (c *artifactCache) paths(key string) (string, string) { + dir := filepath.Join(c.dir, key[:2]) + + return filepath.Join(dir, key+".bin"), filepath.Join(dir, key+".json") +} + +// load returns a cached artifact, or nil when it is not recorded. +func (c *artifactCache) load(key string) *artifact { + bodyPath, metaPath := c.paths(key) + + body, err := os.ReadFile(bodyPath) + if err != nil { + return nil + } + + metaData, err := os.ReadFile(metaPath) + if err != nil { + return nil + } + + meta := cacheMeta{} + if err := json.Unmarshal(metaData, &meta); err != nil { + return nil + } + + return &artifact{body: body, header: meta.Header} +} + +func (c *artifactCache) store(key string, art *artifact) { + bodyPath, metaPath := c.paths(key) + + if err := os.MkdirAll(filepath.Dir(bodyPath), 0o755); err != nil { + c.logger.WithError(err).Debugf("could not create cache subdir for %v", key) + return + } + + metaData, err := json.Marshal(cacheMeta{Header: art.header}) + if err != nil { + c.logger.WithError(err).Debugf("could not encode cache meta for %v", key) + return + } + + // write the body first: a body without meta is simply treated as a cache miss, + // while meta without a body would be too + if err := os.WriteFile(bodyPath, art.body, 0o644); err != nil { + c.logger.WithError(err).Debugf("could not write cache body for %v", key) + return + } + + if err := os.WriteFile(metaPath, metaData, 0o644); err != nil { + c.logger.WithError(err).Debugf("could not write cache meta for %v", key) + } +} diff --git a/replay/cache_test.go b/replay/cache_test.go new file mode 100644 index 00000000..54c3b079 --- /dev/null +++ b/replay/cache_test.go @@ -0,0 +1,67 @@ +package replay + +import ( + "net/http" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestIsImmutablePath(t *testing.T) { + root := "0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" + + tests := []struct { + path string + want bool + }{ + {path: "/eth/v2/debug/beacon/states/" + root, want: true}, + {path: "/eth/v2/beacon/blocks/" + root, want: true}, + {path: "/eth/v1/beacon/states/" + root + "/finality_checkpoints", want: true}, + {path: "/eth/v1/beacon/headers/49152", want: false}, + {path: "/eth/v1/beacon/headers/head", want: false}, + {path: "/eth/v1/config/spec", want: false}, + {path: "/eth/v1/node/syncing", want: false}, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + require.Equal(t, test.want, isImmutablePath(test.path)) + }) + } +} + +func TestArtifactCacheRoundTrip(t *testing.T) { + cache, err := newArtifactCache(logrus.New(), t.TempDir()) + require.NoError(t, err) + + key := cache.key("/eth/v2/beacon/blocks/0xabc", "application/octet-stream") + + require.Nil(t, cache.load(key), "an unrecorded artifact must be a cache miss") + + stored := &artifact{ + body: []byte{0x01, 0x02, 0x03}, + header: http.Header{ + "Content-Type": []string{"application/octet-stream"}, + "Eth-Consensus-Version": []string{"gloas"}, + }, + } + + cache.store(key, stored) + + loaded := cache.load(key) + require.NotNil(t, loaded) + require.Equal(t, stored.body, loaded.body) + require.Equal(t, "gloas", loaded.header.Get("Eth-Consensus-Version"), + "the fork header must survive the cache, or SSZ responses cannot be decoded") +} + +func TestArtifactCacheKeyIncludesEncoding(t *testing.T) { + cache, err := newArtifactCache(logrus.New(), t.TempDir()) + require.NoError(t, err) + + json := cache.key("/eth/v2/beacon/blocks/0xabc", "application/json") + ssz := cache.key("/eth/v2/beacon/blocks/0xabc", "application/octet-stream") + + require.NotEqual(t, json, ssz, "the same artifact in two encodings must not share a cache entry") +} diff --git a/replay/chain.go b/replay/chain.go new file mode 100644 index 00000000..a3bc70cb --- /dev/null +++ b/replay/chain.go @@ -0,0 +1,190 @@ +package replay + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/sirupsen/logrus" +) + +// forkOrder lists the forks oldest first, paired with the spec key that activates them. +// The names are the ones the beacon API reports in Eth-Consensus-Version. +var forkOrder = []struct { + name string + specKey string +}{ + {name: "altair", specKey: "ALTAIR_FORK_EPOCH"}, + {name: "bellatrix", specKey: "BELLATRIX_FORK_EPOCH"}, + {name: "capella", specKey: "CAPELLA_FORK_EPOCH"}, + {name: "deneb", specKey: "DENEB_FORK_EPOCH"}, + {name: "electra", specKey: "ELECTRA_FORK_EPOCH"}, + {name: "fulu", specKey: "FULU_FORK_EPOCH"}, + {name: "gloas", specKey: "GLOAS_FORK_EPOCH"}, + {name: "heze", specKey: "HEZE_FORK_EPOCH"}, +} + +type forkActivation struct { + name string + epoch uint64 +} + +// chainInfo holds the genesis and timing parameters the replay needs to translate +// between slots and (virtual) wall clock time. Genesis stays the real one, so every +// timestamp the explorer renders is the true historical time of the replayed slot. +type chainInfo struct { + genesisTime time.Time + slotDuration time.Duration + slotsPerEpoch uint64 + forks []forkActivation +} + +// bidsActiveAt reports whether a block at this slot can carry an execution payload bid, +// which only exists from the Gloas fork (EIP-7732) onwards. Before that, fetching the +// block to look for one would be a wasted round trip on every slot. +func (c *chainInfo) bidsActiveAt(slot uint64) bool { + epoch := c.epochOf(slot) + + for _, fork := range c.forks { + if fork.name == "gloas" { + return epoch >= fork.epoch + } + } + + return false +} + +// forkAt returns the fork a slot belongs to, as reported in Eth-Consensus-Version. +// Artifacts served from tracoor carry no such header of their own, and without it a +// client cannot tell which SSZ container it received. +func (c *chainInfo) forkAt(slot uint64) string { + epoch := c.epochOf(slot) + name := "phase0" + + for _, fork := range c.forks { + if epoch >= fork.epoch { + name = fork.name + } + } + + return name +} + +func (c *chainInfo) slotTime(slot uint64) time.Time { + return c.genesisTime.Add(time.Duration(slot) * c.slotDuration) +} + +func (c *chainInfo) epochOf(slot uint64) uint64 { + return slot / c.slotsPerEpoch +} + +// loadChainInfo reads genesis and the timing specs from the beacon upstream. +func loadChainInfo(ctx context.Context, up *upstream) (*chainInfo, error) { + genesisRsp := struct { + Data struct { + GenesisTime string `json:"genesis_time"` + } `json:"data"` + }{} + + if err := up.getJSON(ctx, "/eth/v1/beacon/genesis", &genesisRsp); err != nil { + return nil, fmt.Errorf("error fetching genesis: %w", err) + } + + genesisUnix, err := strconv.ParseInt(genesisRsp.Data.GenesisTime, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid genesis_time %q: %w", genesisRsp.Data.GenesisTime, err) + } + + specRsp := struct { + Data map[string]any `json:"data"` + }{} + + if err := up.getJSON(ctx, "/eth/v1/config/spec", &specRsp); err != nil { + return nil, fmt.Errorf("error fetching config spec: %w", err) + } + + slotDurationMs, err := specUint(specRsp.Data, "SLOT_DURATION_MS") + if err != nil { + secondsPerSlot, secErr := specUint(specRsp.Data, "SECONDS_PER_SLOT") + if secErr != nil { + return nil, fmt.Errorf("could not determine slot duration: %w", err) + } + + slotDurationMs = secondsPerSlot * 1000 + } + + slotsPerEpoch, err := specUint(specRsp.Data, "SLOTS_PER_EPOCH") + if err != nil { + return nil, err + } + + if slotDurationMs == 0 || slotsPerEpoch == 0 { + return nil, fmt.Errorf("upstream reported a zero slot duration or epoch length") + } + + return &chainInfo{ + genesisTime: time.Unix(genesisUnix, 0).UTC(), + slotDuration: time.Duration(slotDurationMs) * time.Millisecond, + slotsPerEpoch: slotsPerEpoch, + forks: parseForkSchedule(specRsp.Data), + }, nil +} + +// parseForkSchedule reads the activation epoch of every fork the upstream knows about, +// in activation order. Forks that are not scheduled are left out. +func parseForkSchedule(specs map[string]any) []forkActivation { + forks := make([]forkActivation, 0, len(forkOrder)) + + for _, fork := range forkOrder { + epoch, err := specUint(specs, fork.specKey) + if err != nil { + continue + } + + forks = append(forks, forkActivation{name: fork.name, epoch: epoch}) + } + + return forks +} + +// specUint reads a numeric spec value. The config endpoint reports numbers as decimal +// strings, but not every entry is a number (the blob schedule is a list), so anything +// that is not a plain numeric string is reported as an error rather than coerced. +func specUint(specs map[string]any, key string) (uint64, error) { + raw, ok := specs[key] + if !ok { + return 0, fmt.Errorf("spec value %v missing", key) + } + + text, ok := raw.(string) + if !ok { + return 0, fmt.Errorf("spec value %v is not a number (got %T)", key, raw) + } + + value, err := strconv.ParseUint(text, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid spec value %v=%q: %w", key, text, err) + } + + return value, nil +} + +// SlotsPerEpoch reads just the epoch length from a beacon node, so a start epoch given +// on the command line can be converted to a slot before the replay is constructed. +func SlotsPerEpoch(ctx context.Context, upstreamURL string) (uint64, error) { + up, err := newUpstream(logrus.New(), &Config{UpstreamURL: upstreamURL}) + if err != nil { + return 0, err + } + + rsp := struct { + Data map[string]any `json:"data"` + }{} + + if err := up.getJSON(ctx, "/eth/v1/config/spec", &rsp); err != nil { + return 0, fmt.Errorf("error fetching config spec: %w", err) + } + + return specUint(rsp.Data, "SLOTS_PER_EPOCH") +} diff --git a/replay/clock.go b/replay/clock.go new file mode 100644 index 00000000..97b522c9 --- /dev/null +++ b/replay/clock.go @@ -0,0 +1,133 @@ +package replay + +import ( + "sync" + "time" +) + +// clock is the virtual time the replay serves to the explorer. It is anchored to a +// point in virtual time and advances at `rate` virtual seconds per real second, so a +// paused replay (rate 0) simply holds its anchor while a playing one interpolates. +type clock struct { + mutex sync.RWMutex + anchorReal time.Time + anchorVirt time.Time + rate float64 + + // holds freezes the clock without forgetting the rate it was running at, so the + // replay can wait for the explorer without any virtual time passing. Holds nest. + holds int +} + +func newClock(virt time.Time) *clock { + return &clock{ + anchorReal: time.Now(), + anchorVirt: virt, + } +} + +// now returns the current virtual time. +func (c *clock) now() time.Time { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.nowLocked() +} + +func (c *clock) nowLocked() time.Time { + rate := c.effectiveRateLocked() + if rate == 0 { + return c.anchorVirt + } + + return c.anchorVirt.Add(time.Duration(float64(time.Since(c.anchorReal)) * rate)) +} + +// effectiveRateLocked is the rate the clock is actually running at: zero while held, +// whatever was configured otherwise. +func (c *clock) effectiveRateLocked() float64 { + if c.holds > 0 { + return 0 + } + + return c.rate +} + +// set jumps the virtual clock to an absolute point in time, keeping the current rate. +func (c *clock) set(virt time.Time) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.anchorReal = time.Now() + c.anchorVirt = virt +} + +// setRate changes the playback rate, re-anchoring so no virtual time is lost. +func (c *clock) setRate(rate float64) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.anchorVirt = c.nowLocked() + c.anchorReal = time.Now() + c.rate = rate +} + +// hold freezes the clock where it is. The explorer mirrors the effective rate, so it +// stops moving too rather than drifting ahead of what the replay has actually served. +func (c *clock) hold() { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.anchorVirt = c.nowLocked() + c.anchorReal = time.Now() + c.holds++ +} + +// release lifts one hold, resuming at the configured rate once the last one is gone. +func (c *clock) release() { + c.mutex.Lock() + defer c.mutex.Unlock() + + if c.holds == 0 { + return + } + + c.anchorVirt = c.nowLocked() + c.anchorReal = time.Now() + c.holds-- +} + +func (c *clock) isHeld() bool { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.holds > 0 +} + +// state returns the current virtual time and the rate it is moving at, as served to +// the explorer. +func (c *clock) state() (time.Time, float64) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.nowLocked(), c.effectiveRateLocked() +} + +// realDelayUntil returns how long to wait in real time for the virtual clock to reach +// the given point, at the rate it is configured to run at. It returns 0 when the clock +// is paused or already past it, since a paused clock would never get there on its own. +func (c *clock) realDelayUntil(virt time.Time) time.Duration { + c.mutex.RLock() + defer c.mutex.RUnlock() + + if c.rate <= 0 { + return 0 + } + + remaining := virt.Sub(c.nowLocked()) + if remaining <= 0 { + return 0 + } + + return time.Duration(float64(remaining) / c.rate) +} diff --git a/replay/clock_test.go b/replay/clock_test.go new file mode 100644 index 00000000..5c8248b7 --- /dev/null +++ b/replay/clock_test.go @@ -0,0 +1,72 @@ +package replay + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPausedClockHoldsItsAnchor(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + + time.Sleep(20 * time.Millisecond) + + require.Equal(t, anchor, clock.now()) + + now, rate := clock.state() + require.Equal(t, anchor, now) + require.Zero(t, rate) +} + +func TestSetJumpsTheClock(t *testing.T) { + clock := newClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + + target := time.Date(2026, 1, 1, 0, 0, 12, 0, time.UTC) + clock.set(target) + + require.Equal(t, target, clock.now()) +} + +func TestPlayingClockAdvancesWithRealTime(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + clock.setRate(100) + + time.Sleep(20 * time.Millisecond) + + elapsed := clock.now().Sub(anchor) + require.Greater(t, elapsed, time.Second, "100x rate should cover >1s of virtual time in 20ms") +} + +func TestSetRateKeepsElapsedVirtualTime(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + clock.setRate(100) + + time.Sleep(20 * time.Millisecond) + + clock.setRate(0) + frozen := clock.now() + + time.Sleep(20 * time.Millisecond) + + require.Equal(t, frozen, clock.now(), "a paused clock must not drift") + require.True(t, frozen.After(anchor), "the virtual time gained while playing must be kept") +} + +func TestRealDelayUntilScalesWithRate(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + + // a paused clock will never reach the target on its own + require.Zero(t, clock.realDelayUntil(anchor.Add(time.Minute))) + + clock.setRate(4) + + delay := clock.realDelayUntil(anchor.Add(4 * time.Second)) + require.InDelta(t, float64(time.Second), float64(delay), float64(50*time.Millisecond)) + + require.Zero(t, clock.realDelayUntil(anchor.Add(-time.Second))) +} diff --git a/replay/clproxy.go b/replay/clproxy.go new file mode 100644 index 00000000..e1b886a3 --- /dev/null +++ b/replay/clproxy.go @@ -0,0 +1,299 @@ +package replay + +import ( + "context" + "encoding/hex" + "encoding/json" + "net/http" + "strconv" + "strings" + + "github.com/ethpandaops/go-eth2-client/spec/phase0" +) + +// blockIDPaths are the beacon API paths whose first variable segment identifies a +// block, and stateIDPaths the ones whose first variable segment identifies a state. +// Everything else is proxied unchanged. +var ( + blockIDPaths = []string{ + "/eth/v1/beacon/headers/", + "/eth/v1/beacon/blocks/", + "/eth/v2/beacon/blocks/", + "/eth/v1/beacon/blinded_blocks/", + "/eth/v2/beacon/blinded_blocks/", + "/eth/v1/beacon/blob_sidecars/", + "/eth/v1/beacon/blobs/", + "/eth/v1/beacon/execution_payload_envelopes/", + "/eth/v1/beacon/execution_proofs/", + "/eth/v1/beacon/inclusion_lists/", + } + + stateIDPaths = []string{ + "/eth/v1/beacon/states/", + "/eth/v2/beacon/states/", + "/eth/v1/debug/beacon/states/", + "/eth/v2/debug/beacon/states/", + } +) + +// clHandler is the fake beacon node. Everything is proxied to the real upstream except +// where the virtual head matters: the sync status is synthesized, the event stream is +// generated locally, `head` aliases resolve to the head at the virtual slot, and +// anything beyond the virtual slot is reported as missing. +func (r *Replay) clHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/eth/v1/node/syncing", r.serveSyncing) + mux.HandleFunc("/eth/v1/events", r.events.serveHTTP) + mux.HandleFunc("/eth/v1/beacon/states/head/finality_checkpoints", r.serveHeadFinality) + mux.HandleFunc("/", r.serveProxied) + + return mux +} + +func (r *Replay) serveSyncing(w http.ResponseWriter, _ *http.Request) { + r.mutex.RLock() + headSlot := uint64(0) + if r.head != nil { + headSlot = r.head.Slot + } + r.mutex.RUnlock() + + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "head_slot": strconv.FormatUint(headSlot, 10), + "sync_distance": "0", + "is_syncing": false, + "is_optimistic": false, + "el_offline": false, + }, + }) +} + +// serveHeadFinality answers from the finality the replay tracks rather than proxying. +// Reading finality upstream forces the node to load the head state, which for a +// replayed (historical) head is expensive at best and pruned away at worst. +func (r *Replay) serveHeadFinality(w http.ResponseWriter, _ *http.Request) { + r.mutex.RLock() + finality := r.finality + r.mutex.RUnlock() + + if finality == nil || len(finality.Raw) == 0 { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + if _, err := w.Write(finality.Raw); err != nil { + r.logger.WithError(err).Debug("error writing head finality response") + } +} + +func (r *Replay) serveProxied(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + writeAPIError(w, http.StatusNotImplemented, "the replay proxy only serves read requests") + return + } + + path, err := r.rewritePath(req.Context(), req.URL.Path) + if err != nil { + if err == errNotFound { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + writeAPIError(w, http.StatusBadGateway, err.Error()) + + return + } + + if query := req.URL.RawQuery; query != "" { + if err := r.checkQueryCutoff(req); err != nil { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + path += "?" + query + } + + // a full beacon state blocks the explorer for as long as it takes to arrive; the + // driver waits for these rather than replaying slots the explorer cannot see yet + if isStatePath(path) { + defer r.states.begin()() + } + + r.upstream.serve(w, req, path) +} + +// rewritePath resolves the block/state identifier in a beacon API path against the +// virtual head, so the explorer can never see past the slot the replay has reached. +func (r *Replay) rewritePath(ctx context.Context, path string) (string, error) { + for _, prefix := range blockIDPaths { + if strings.HasPrefix(path, prefix) { + return r.rewriteID(ctx, path, prefix, false) + } + } + + for _, prefix := range stateIDPaths { + if strings.HasPrefix(path, prefix) { + return r.rewriteID(ctx, path, prefix, true) + } + } + + return path, nil +} + +func (r *Replay) rewriteID(ctx context.Context, path, prefix string, isState bool) (string, error) { + rest := strings.TrimPrefix(path, prefix) + + id, suffix, _ := strings.Cut(rest, "/") + if suffix != "" { + suffix = "/" + suffix + } + + resolved, err := r.resolveID(ctx, id, isState) + if err != nil { + return "", err + } + + return prefix + resolved + suffix, nil +} + +// resolveID maps a beacon API identifier to something the upstream can answer with the +// same meaning it had at the virtual head slot. +func (r *Replay) resolveID(ctx context.Context, id string, isState bool) (string, error) { + switch id { + case "head": + header := r.currentHead() + if header == nil { + return "", errNotFound + } + + if isState { + return header.StateRoot, nil + } + + return header.Root, nil + + case "finalized", "justified": + return r.resolveCheckpointID(ctx, id, isState) + + case "genesis": + return id, nil + } + + if strings.HasPrefix(id, "0x") { + // roots are only ever learned from artifacts the replay already released + return id, nil + } + + slot, err := strconv.ParseUint(id, 10, 64) + if err != nil { + // an identifier the replay does not understand is passed through unchanged + return id, nil + } + + r.mutex.RLock() + virtualSlot := r.virtualSlot + r.mutex.RUnlock() + + if slot > virtualSlot { + return "", errNotFound + } + + return id, nil +} + +// resolveCheckpointID turns the `finalized` and `justified` aliases into the concrete +// roots the chain had at the virtual head, rather than the ones the upstream has today. +func (r *Replay) resolveCheckpointID(ctx context.Context, id string, isState bool) (string, error) { + r.mutex.RLock() + finality := r.finality + r.mutex.RUnlock() + + if finality == nil { + return "", errNotFound + } + + root := finality.FinalizedRoot + if id == "justified" { + root = finality.JustifiedRoot + } + + if !isState { + return root, nil + } + + header, err := r.upstream.headerByRoot(ctx, root) + if err != nil || header == nil { + return "", errNotFound + } + + return header.StateRoot, nil +} + +// checkQueryCutoff rejects queries that select data beyond the virtual head slot. +func (r *Replay) checkQueryCutoff(req *http.Request) error { + slotParam := req.URL.Query().Get("slot") + if slotParam == "" { + return nil + } + + slot, err := strconv.ParseUint(slotParam, 10, 64) + if err != nil { + return nil + } + + r.mutex.RLock() + virtualSlot := r.virtualSlot + r.mutex.RUnlock() + + if slot > virtualSlot { + return errNotFound + } + + return nil +} + +func (r *Replay) currentHead() *blockHeader { + r.mutex.RLock() + defer r.mutex.RUnlock() + + return r.head +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + + if err := json.NewEncoder(w).Encode(body); err != nil { + return + } +} + +// writeAPIError answers in the shape the beacon API uses for errors, so clients report +// something meaningful instead of a decode failure. +func writeAPIError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]any{ + "code": status, + "message": message, + }) +} + +// parseRoot decodes a 0x-prefixed root for use in an event payload. A malformed root +// yields the zero root, which is what the upstream would have to have served for this +// to happen at all. +func parseRoot(value string) phase0.Root { + root := phase0.Root{} + + decoded, err := hex.DecodeString(strings.TrimPrefix(value, "0x")) + if err != nil || len(decoded) != len(root) { + return root + } + + copy(root[:], decoded) + + return root +} diff --git a/replay/clproxy_test.go b/replay/clproxy_test.go new file mode 100644 index 00000000..a61fd654 --- /dev/null +++ b/replay/clproxy_test.go @@ -0,0 +1,123 @@ +package replay + +import ( + "context" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func testReplay(virtualSlot uint64) *Replay { + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + return &Replay{ + logger: logger, + events: newEventHub(logger), + control: newLossyEventHub(logger), + states: newStateLoads(), + chain: &chainInfo{ + slotsPerEpoch: 32, + slotDuration: 12 * time.Second, + }, + upstream: &upstream{logger: logger}, + virtualSlot: virtualSlot, + head: &blockHeader{ + Slot: virtualSlot - 1, + Root: "0x1111111111111111111111111111111111111111111111111111111111111111", + StateRoot: "0x2222222222222222222222222222222222222222222222222222222222222222", + }, + finality: &finalityCheckpoints{ + JustifiedEpoch: 10, + JustifiedRoot: "0x3333333333333333333333333333333333333333333333333333333333333333", + FinalizedEpoch: 9, + FinalizedRoot: "0x4444444444444444444444444444444444444444444444444444444444444444", + }, + } +} + +func TestRewritePath(t *testing.T) { + replay := testReplay(100) + ctx := context.Background() + + tests := []struct { + name string + path string + want string + wantErr bool + }{ + { + name: "head header resolves to the head block root", + path: "/eth/v1/beacon/headers/head", + want: "/eth/v1/beacon/headers/" + replay.head.Root, + }, + { + name: "head finality resolves to the head state root", + path: "/eth/v1/beacon/states/head/finality_checkpoints", + want: "/eth/v1/beacon/states/" + replay.head.StateRoot + "/finality_checkpoints", + }, + { + name: "finalized block id resolves to the tracked checkpoint", + path: "/eth/v2/beacon/blocks/finalized", + want: "/eth/v2/beacon/blocks/" + replay.finality.FinalizedRoot, + }, + { + name: "roots are passed through", + path: "/eth/v2/debug/beacon/states/0x9999999999999999999999999999999999999999999999999999999999999999", + want: "/eth/v2/debug/beacon/states/0x9999999999999999999999999999999999999999999999999999999999999999", + }, + { + name: "genesis is passed through", + path: "/eth/v2/debug/beacon/states/genesis", + want: "/eth/v2/debug/beacon/states/genesis", + }, + { + name: "a slot at the virtual head is served", + path: "/eth/v1/beacon/headers/100", + want: "/eth/v1/beacon/headers/100", + }, + { + name: "a slot beyond the virtual head is hidden", + path: "/eth/v1/beacon/headers/101", + wantErr: true, + }, + { + name: "unrelated paths are untouched", + path: "/eth/v1/config/spec", + want: "/eth/v1/config/spec", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := replay.rewritePath(ctx, test.path) + + if test.wantErr { + require.ErrorIs(t, err, errNotFound) + return + } + + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} + +func TestRewritePathWithoutHead(t *testing.T) { + replay := testReplay(100) + replay.head = nil + + _, err := replay.rewritePath(context.Background(), "/eth/v1/beacon/headers/head") + require.ErrorIs(t, err, errNotFound) +} + +func TestParseRoot(t *testing.T) { + root := parseRoot("0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + require.Equal(t, byte(0x01), root[0]) + require.Equal(t, byte(0x20), root[31]) + + // a malformed root degrades to the zero root rather than panicking + require.Equal(t, parseRoot("nonsense"), parseRoot("0x00")) +} diff --git a/replay/config.go b/replay/config.go new file mode 100644 index 00000000..4874bd85 --- /dev/null +++ b/replay/config.go @@ -0,0 +1,95 @@ +package replay + +import ( + "fmt" + "time" +) + +// Config describes a replay run: where the real chain data comes from, where the +// fake nodes listen, and where in the chain the replay starts. +type Config struct { + // UpstreamURL is the beacon node the consensus proxy reads from. It must serve + // historical blocks and states for the replayed range, so point it at a single + // known-archive node rather than at a load balancer over mixed-retention nodes. + UpstreamURL string + + // ExecutionURL is the execution node the JSON-RPC proxy reads from. When empty + // the execution proxy is not started. + ExecutionURL string + + // TracoorURL and TracoorNetwork enable the tracoor fallback for artifacts the + // beacon node has pruned. Tracoor indexes by root, so it can only serve blocks + // and states whose root is already known from a header. + TracoorURL string + TracoorNetwork string + + CLListen string + ELListen string + ControlListen string + + // StartSlot is the first slot the replay serves. The head is initialized to the + // newest block at or before this slot. + StartSlot uint64 + + // Speed is the initial playback rate in virtual seconds per real second. A value + // of 0 starts the replay paused. + Speed float64 + + // CacheDir, when set, stores every immutable artifact fetched from upstream so a + // range is downloaded once and replays offline afterwards. + CacheDir string + + // EmitBids controls whether the winning execution payload bid of each Gloas block + // is replayed on the event stream. It costs one extra block fetch per slot. + EmitBids bool + + // StepSettle is the real time the driver pauses at each phase of a stepped slot, + // giving the explorer a chance to observe the slot boundary and process events. + StepSettle time.Duration + + // StateHoldTimeout bounds how long the replay freezes its clock waiting for the + // explorer to finish loading a beacon state, so a stuck read cannot wedge a run. + StateHoldTimeout time.Duration +} + +// DefaultConfig returns a config with the non-chain-specific defaults filled in. +func DefaultConfig() Config { + return Config{ + CLListen: "127.0.0.1:15052", + ELListen: "127.0.0.1:15545", + ControlListen: "127.0.0.1:15000", + Speed: 0, + EmitBids: true, + StepSettle: 250 * time.Millisecond, + StateHoldTimeout: 2 * time.Minute, + } +} + +// Validate checks the config for the combinations the replay cannot run with. +func (c *Config) Validate() error { + if c.UpstreamURL == "" { + return fmt.Errorf("no beacon upstream configured (--upstream): tracoor alone cannot resolve slots") + } + + if c.TracoorURL != "" && c.TracoorNetwork == "" { + return fmt.Errorf("--tracoor-network is required when --tracoor is set") + } + + if c.StartSlot == 0 { + return fmt.Errorf("no start slot configured (--start-slot / --start-epoch)") + } + + if c.Speed < 0 { + return fmt.Errorf("speed must not be negative") + } + + if c.StepSettle <= 0 { + c.StepSettle = 250 * time.Millisecond + } + + if c.StateHoldTimeout <= 0 { + c.StateHoldTimeout = 2 * time.Minute + } + + return nil +} diff --git a/replay/console.go b/replay/console.go new file mode 100644 index 00000000..1f2f509c --- /dev/null +++ b/replay/console.go @@ -0,0 +1,219 @@ +package replay + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" +) + +const consoleHelp = `commands: + status show where the replay currently stands + step [n] advance n slots (default 1), then pause + forward [speed] + run to a slot, emitting every slot on the way; without a + speed it advances as fast as upstream allows + play [speed] run continuously at x real time (default 1) + stop pause the replay and freeze the virtual clock + start resume at the last speed + help show this help + quit stop the replay and exit +` + +// RunConsole reads commands from stdin until the input ends or `quit` is entered. It +// returns when the replay should shut down. +func (r *Replay) RunConsole(ctx context.Context, in io.Reader, out io.Writer) { + lines := make(chan string) + + go func() { + defer close(lines) + + scanner := bufio.NewScanner(in) + for scanner.Scan() { + lines <- scanner.Text() + } + }() + + consolePrint(out, consoleHelp) + r.printStatus(out) + consolePrint(out, "replay> ") + + for { + select { + case <-ctx.Done(): + return + case line, ok := <-lines: + if !ok { + return + } + + if r.runCommand(out, line) { + return + } + + consolePrint(out, "replay> ") + } + } +} + +// runCommand executes one console line and reports whether the replay should exit. +func (r *Replay) runCommand(out io.Writer, line string) bool { + fields := strings.Fields(line) + if len(fields) == 0 { + return false + } + + command, args := fields[0], fields[1:] + + switch command { + case "status", "s": + r.printStatus(out) + + case "step": + slots := uint64(1) + + if len(args) > 0 { + parsed, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + consolePrintf(out, "invalid slot count %q\n", args[0]) + + return false + } + + slots = parsed + } + + r.Step(slots) + + case "forward", "fwd": + if len(args) == 0 { + consolePrint(out, "usage: forward [speed]\n") + + return false + } + + slot, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + consolePrintf(out, "invalid slot %q\n", args[0]) + + return false + } + + speed := 0.0 + + if len(args) > 1 { + speed, err = parseSpeed(args[1]) + if err != nil { + consolePrintf(out, "%v\n", err) + + return false + } + } + + if err := r.Forward(slot, speed); err != nil { + consolePrintf(out, "%v\n", err) + } + + case "play": + speed := 1.0 + + if len(args) > 0 { + parsed, err := parseSpeed(args[0]) + if err != nil { + consolePrintf(out, "%v\n", err) + + return false + } + + speed = parsed + } + + r.Play(speed) + + case "stop", "pause": + r.Pause() + r.printStatus(out) + + case "start", "resume": + r.Resume() + + case "help", "?": + consolePrint(out, consoleHelp) + + case "quit", "exit", "q": + return true + + default: + consolePrintf(out, "unknown command %q (try `help`)\n", command) + } + + return false +} + +func (r *Replay) printStatus(out io.Writer) { + status := r.Status() + + mode := "paused" + + switch { + case status.Holding: + mode = fmt.Sprintf("holding for %v state load(s)", status.StateLoads) + case status.Running && status.Speed > 0: + mode = fmt.Sprintf("playing %gx", status.Speed) + case status.Running && status.TargetSlot > 0: + mode = fmt.Sprintf("stepping to %v", status.TargetSlot) + case status.Running: + mode = "running at max speed" + } + + upstream := status.Upstream + if status.Tracoor { + upstream += " (+tracoor)" + } + + consolePrintf(out, " slot %v epoch %v head %v [%v]\n", + status.VirtualSlot, status.VirtualEpoch, status.HeadSlot, shortRoot(status.HeadRoot)) + consolePrintf(out, " justified %v finalized %v el block %v\n", + status.JustifiedEpoch, status.FinalizedEpoch, status.ExecutionBlock) + consolePrintf(out, " %v time %v streams %v\n", + mode, status.VirtualTime.Format(time.RFC3339), status.Subscribers) + consolePrintf(out, " upstream %v\n", upstream) +} + +// parseSpeed reads a playback rate, accepting both `4` and `4x`. +func parseSpeed(value string) (float64, error) { + speed, err := strconv.ParseFloat(strings.TrimSuffix(value, "x"), 64) + if err != nil || speed <= 0 { + return 0, fmt.Errorf("invalid speed %q", value) + } + + return speed, nil +} + +// consolePrint and consolePrintf write to the console; a broken console is not worth +// aborting a replay for, so write errors are deliberately ignored. +func consolePrint(out io.Writer, text string) { + _, _ = io.WriteString(out, text) +} + +func consolePrintf(out io.Writer, format string, args ...any) { + _, _ = fmt.Fprintf(out, format, args...) +} + +func shortRoot(root string) string { + if len(root) <= 12 { + return root + } + + return root[:10] + "…" +} + +// Stdio returns the console's default streams, kept in one place so the caller does not +// have to reach into os from its own package. +func Stdio() (io.Reader, io.Writer) { + return os.Stdin, os.Stdout +} diff --git a/replay/console_test.go b/replay/console_test.go new file mode 100644 index 00000000..2ced0761 --- /dev/null +++ b/replay/console_test.go @@ -0,0 +1,136 @@ +package replay + +import ( + "bytes" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func testConsoleReplay(virtualSlot uint64) *Replay { + replay := testReplay(virtualSlot) + replay.clock = newClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + replay.wake = make(chan struct{}, 1) + replay.cfg = DefaultConfig() + + return replay +} + +func TestConsoleCommands(t *testing.T) { + tests := []struct { + name string + line string + wantRunning bool + wantSpeed float64 + wantTarget uint64 + wantOutput string + }{ + { + name: "step advances one slot by default", + line: "step", + wantRunning: true, + wantTarget: 101, + }, + { + name: "step takes a slot count", + line: "step 32", + wantRunning: true, + wantTarget: 132, + }, + { + name: "forward sets an absolute target", + line: "forward 4096", + wantRunning: true, + wantTarget: 4096, + }, + { + name: "forward takes an optional speed", + line: "forward 4096 6x", + wantRunning: true, + wantSpeed: 6, + wantTarget: 4096, + }, + { + name: "forward refuses to rewind", + line: "forward 50", + wantRunning: false, + wantOutput: "cannot rewind", + }, + { + name: "play defaults to real time", + line: "play", + wantRunning: true, + wantSpeed: 1, + }, + { + name: "play takes an x-suffixed speed", + line: "play 8x", + wantRunning: true, + wantSpeed: 8, + }, + { + name: "an invalid speed is rejected", + line: "play banana", + wantOutput: "invalid speed", + }, + { + name: "an unknown command is reported", + line: "frobnicate", + wantOutput: "unknown command", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + replay := testConsoleReplay(100) + out := &bytes.Buffer{} + + require.False(t, replay.runCommand(out, test.line)) + + require.Equal(t, test.wantRunning, replay.running) + require.Equal(t, test.wantSpeed, replay.speed) + require.Equal(t, test.wantTarget, replay.target) + + if test.wantOutput != "" { + require.Contains(t, out.String(), test.wantOutput) + } + }) + } +} + +func TestConsoleQuit(t *testing.T) { + replay := testConsoleReplay(100) + require.True(t, replay.runCommand(&bytes.Buffer{}, "quit")) +} + +func TestConsoleStopAndResume(t *testing.T) { + replay := testConsoleReplay(100) + out := &bytes.Buffer{} + + replay.runCommand(out, "play 4x") + require.True(t, replay.running) + + replay.runCommand(out, "stop") + require.False(t, replay.running) + + // the clock must not keep running while the replay is paused + frozen := replay.clock.now() + time.Sleep(20 * time.Millisecond) + require.Equal(t, frozen, replay.clock.now()) + + replay.runCommand(out, "start") + require.True(t, replay.running) + require.Equal(t, 4.0, replay.speed) +} + +func TestConsoleStatus(t *testing.T) { + replay := testConsoleReplay(100) + out := &bytes.Buffer{} + + replay.runCommand(out, "status") + + require.Contains(t, out.String(), "slot 100") + require.Contains(t, out.String(), "finalized 9") + require.Contains(t, out.String(), "paused") +} diff --git a/replay/control.go b/replay/control.go new file mode 100644 index 00000000..7c28ccdd --- /dev/null +++ b/replay/control.go @@ -0,0 +1,147 @@ +package replay + +import ( + _ "embed" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// controlUI is the replay control panel the explorer side-loads. Serving it from here +// keeps the explorer's side of the integration down to a single script tag. +// +//go:embed assets/replay-ui.js +var controlUI []byte + +// Command is a control instruction, as posted to /replay/command. It mirrors what the +// interactive console offers, so the explorer UI and the console drive the same replay +// through the same code. +type Command struct { + Action string `json:"action"` + Speed float64 `json:"speed,omitempty"` + Slots uint64 `json:"slots,omitempty"` + Slot uint64 `json:"slot,omitempty"` +} + +// controlHandler serves the replay's own API: the virtual clock the explorer follows, a +// status snapshot, a status event stream, and the commands that drive the replay. +func (r *Replay) controlHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/replay/clock", r.serveClock) + mux.HandleFunc("/replay/status", r.serveStatus) + mux.HandleFunc("/replay/events", r.control.serveHTTP) + mux.HandleFunc("/replay/command", r.serveCommand) + mux.HandleFunc("/replay/ui.js", serveControlUI) + + return withCORS(mux) +} + +// withCORS lets the control API be called from the explorer's own origin, which is a +// different host and port than this server. +func withCORS(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Access-Control-Max-Age", "86400") + + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + + next.ServeHTTP(w, r) + }) +} + +func serveControlUI(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/javascript; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + + if _, err := w.Write(controlUI); err != nil { + return + } +} + +func (r *Replay) serveClock(w http.ResponseWriter, _ *http.Request) { + now, rate := r.clock.state() + + writeJSON(w, http.StatusOK, map[string]any{ + "time_ms": now.UnixMilli(), + "rate": rate, + }) +} + +func (r *Replay) serveStatus(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, r.Status()) +} + +func (r *Replay) serveCommand(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + writeAPIError(w, http.StatusMethodNotAllowed, "commands are posted") + return + } + + command := Command{} + if err := json.NewDecoder(req.Body).Decode(&command); err != nil { + writeAPIError(w, http.StatusBadRequest, fmt.Sprintf("invalid command: %v", err)) + return + } + + if err := r.Execute(command); err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + + writeJSON(w, http.StatusOK, r.Status()) +} + +// Execute applies a control command. +func (r *Replay) Execute(command Command) error { + switch strings.ToLower(command.Action) { + case "play": + // a speed of 0 means "as fast as upstream allows", which is what the UI sends + // for its `max` setting + r.Play(command.Speed) + + case "speed": + r.SetSpeed(command.Speed) + + case "step": + r.Step(command.Slots) + + case "forward": + return r.Forward(command.Slot, command.Speed) + + case "stop", "pause": + r.Pause() + + case "start", "resume": + r.Resume() + + default: + return fmt.Errorf("unknown action %q", command.Action) + } + + return nil +} + +// notifyStatus pushes the current status to everything watching the control stream. It +// is lossy by design: a status is a snapshot, so a client that fell behind wants the +// newest one rather than the backlog, and the driver must never wait on a browser tab. +func (r *Replay) notifyStatus() { + if r.control.subscriberCount() == 0 { + return + } + + data, err := json.Marshal(r.Status()) + if err != nil { + r.logger.WithError(err).Debug("could not encode replay status") + return + } + + r.control.publish(sseEvent{topic: "status", data: data}) +} diff --git a/replay/control_test.go b/replay/control_test.go new file mode 100644 index 00000000..df8c0d32 --- /dev/null +++ b/replay/control_test.go @@ -0,0 +1,257 @@ +package replay + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func controlServer(t *testing.T, replay *Replay) *httptest.Server { + t.Helper() + + server := httptest.NewServer(replay.controlHandler()) + t.Cleanup(server.Close) + + return server +} + +func postCommand(t *testing.T, server *httptest.Server, body string) (*http.Response, Status) { + t.Helper() + + rsp, err := http.Post(server.URL+"/replay/command", "application/json", strings.NewReader(body)) + require.NoError(t, err) + + t.Cleanup(func() { _ = rsp.Body.Close() }) + + status := Status{} + if rsp.StatusCode == http.StatusOK { + require.NoError(t, json.NewDecoder(rsp.Body).Decode(&status)) + } + + return rsp, status +} + +func TestCommandsDriveTheReplay(t *testing.T) { + tests := []struct { + name string + body string + wantRunning bool + wantSpeed float64 + wantTarget uint64 + }{ + { + name: "play at a speed", + body: `{"action":"play","speed":4}`, + wantRunning: true, + wantSpeed: 4, + }, + { + name: "play at max speed", + body: `{"action":"play","speed":0}`, + wantRunning: true, + wantSpeed: 0, + }, + { + name: "step a number of slots", + body: `{"action":"step","slots":32}`, + wantRunning: true, + wantTarget: 132, + }, + { + name: "forward to a slot", + body: `{"action":"forward","slot":500,"speed":6}`, + wantRunning: true, + wantSpeed: 6, + wantTarget: 500, + }, + { + name: "speed alone does not start the replay", + body: `{"action":"speed","speed":8}`, + wantRunning: false, + wantSpeed: 8, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + rsp, status := postCommand(t, server, test.body) + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, test.wantRunning, status.Running) + require.Equal(t, test.wantSpeed, status.Speed) + require.Equal(t, test.wantTarget, status.TargetSlot) + }) + } +} + +func TestCommandRejectsUnknownAction(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + rsp, _ := postCommand(t, server, `{"action":"detonate"}`) + require.Equal(t, http.StatusBadRequest, rsp.StatusCode) + + rsp, _ = postCommand(t, server, `{"action":"forward","slot":50}`) + require.Equal(t, http.StatusBadRequest, rsp.StatusCode, "forward must refuse to rewind") +} + +func TestResumeKeepsTargetAndSpeed(t *testing.T) { + replay := testConsoleReplay(100) + + require.NoError(t, replay.Forward(500, 4)) + replay.Pause() + replay.Resume() + + status := replay.Status() + require.True(t, status.Running) + require.Equal(t, 4.0, status.Speed, "resuming must keep the speed it was running at") + require.Equal(t, uint64(500), status.TargetSlot, "resuming must keep running towards the target") +} + +func TestResumeDropsAReachedTarget(t *testing.T) { + replay := testConsoleReplay(100) + + replay.Step(1) + replay.virtualSlot = 101 // the driver would have advanced here + replay.Pause() + replay.Resume() + + require.Zero(t, replay.Status().TargetSlot, "a target already reached must not pause the replay again") + require.True(t, replay.Status().Running) +} + +func TestStatusReportsChainContext(t *testing.T) { + replay := testConsoleReplay(100) + replay.cfg.StartSlot = 90 + replay.upstreamSlot = 4000 + replay.chain.genesisTime = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + status := replay.Status() + + require.Equal(t, uint64(90), status.StartSlot) + require.Equal(t, uint64(4000), status.UpstreamSlot) + require.Equal(t, uint64(125), status.UpstreamEpoch) + require.Equal(t, uint64(12000), status.SlotDurationMs) + require.Equal(t, uint64(32), status.SlotsPerEpoch) +} + +func TestControlUIIsServed(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + rsp, err := http.Get(server.URL + "/replay/ui.js") + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Contains(t, rsp.Header.Get("Content-Type"), "javascript") + + body := &bytes.Buffer{} + _, err = body.ReadFrom(rsp.Body) + require.NoError(t, err) + + require.Contains(t, body.String(), "replay-callout", "the side-loaded UI must build its callout") + require.Contains(t, body.String(), "/replay/command", "the side-loaded UI must call the control API") + require.Contains(t, body.String(), "window.doraReplayApi", + "the side-loaded UI must take the control address from the explorer, not from its own script url") + require.Contains(t, body.String(), "window.doraIndexRefreshInterval", + "the side-loaded UI must retune the explorer's polling to the replay's pace") +} + +// TestControlAllowsCrossOriginCalls guards the reason the UI works at all: it is served +// by the replay but runs on the explorer's origin, so the control API has to accept +// cross-origin calls and their preflight. +func TestControlAllowsCrossOriginCalls(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + req, err := http.NewRequest(http.MethodOptions, server.URL+"/replay/command", http.NoBody) + require.NoError(t, err) + + req.Header.Set("Origin", "http://localhost:8083") + req.Header.Set("Access-Control-Request-Method", "POST") + + rsp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusNoContent, rsp.StatusCode) + require.Equal(t, "*", rsp.Header.Get("Access-Control-Allow-Origin")) + require.Contains(t, rsp.Header.Get("Access-Control-Allow-Methods"), "POST") +} + +func TestStatusStreamPushesOnChange(t *testing.T) { + replay := testConsoleReplay(100) + server := controlServer(t, replay) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/replay/events", http.NoBody) + require.NoError(t, err) + + rsp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "text/event-stream", rsp.Header.Get("Content-Type")) + + require.Eventually(t, func() bool { return replay.control.subscriberCount() == 1 }, 5*time.Second, 5*time.Millisecond) + + replay.Play(8) + + reader := bufio.NewReader(rsp.Body) + + eventLine, err := reader.ReadString('\n') + require.NoError(t, err) + require.Equal(t, "event: status", strings.TrimSpace(eventLine)) + + dataLine, err := reader.ReadString('\n') + require.NoError(t, err) + + status := Status{} + require.NoError(t, json.Unmarshal([]byte(strings.TrimPrefix(strings.TrimSpace(dataLine), "data: ")), &status)) + require.True(t, status.Running) + require.Equal(t, 8.0, status.Speed) +} + +// TestStatusStreamNeverBlocksTheDriver is the property that keeps a stalled browser tab +// from stalling the replay: status updates are dropped, not queued. +func TestStatusStreamNeverBlocksTheDriver(t *testing.T) { + replay := testConsoleReplay(100) + + stuck := newEventSubscriber(map[string]bool{"status": true}) + replay.control.add(stuck) + + // fill the subscriber's buffer well past capacity + done := make(chan struct{}) + + go func() { + defer close(done) + + for i := 0; i < 1000; i++ { + replay.notifyStatus() + } + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("notifyStatus blocked on a subscriber that is not reading") + } +} diff --git a/replay/elproxy.go b/replay/elproxy.go new file mode 100644 index 00000000..37ffe599 --- /dev/null +++ b/replay/elproxy.go @@ -0,0 +1,703 @@ +package replay + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// elBlockHistory is how many recent block hashes are kept for block filters. It only +// has to cover the gap between two filter polls. +const elBlockHistory = 4096 + +// blockTagParam maps the JSON-RPC methods that take a block tag to the index of that +// parameter, so `latest` can be pinned to the virtual execution head. +var blockTagParam = map[string]int{ + "eth_getBlockByNumber": 0, + "eth_getBlockTransactionCountByNumber": 0, + "eth_getUncleCountByBlockNumber": 0, + "eth_getBlockReceipts": 0, + "eth_getBalance": 1, + "eth_getCode": 1, + "eth_getTransactionCount": 1, + "eth_call": 1, + "eth_estimateGas": 1, + "eth_createAccessList": 1, + "eth_getStorageAt": 2, + "eth_getProof": 2, +} + +// blockTags are the symbolic block references that must be pinned to the virtual head. +// `finalized` and `safe` are approximated by the head, which is the closest the replay +// can get without tracking execution finality separately. +var blockTags = map[string]bool{ + "latest": true, + "pending": true, + "safe": true, + "finalized": true, +} + +// elBlock is the part of an execution block the replay tracks. +type elBlock struct { + number uint64 + hash string + timestamp time.Time +} + +// elProxy is the fake execution node: a JSON-RPC proxy with a virtual head that follows +// the replayed consensus head, so the explorer never sees a block from the future. +type elProxy struct { + logger logrus.FieldLogger + url string + client *http.Client + + mutex sync.RWMutex + headNumber uint64 + headHash string + blockHashes map[uint64]string + filters map[string]uint64 + filterSerial uint64 + requestID uint64 +} + +var _ http.Handler = (*elProxy)(nil) + +func newELProxy(logger logrus.FieldLogger, url string) *elProxy { + return &elProxy{ + logger: logger, + url: strings.TrimSuffix(url, "/"), + client: newPooledClient(5 * time.Minute), + blockHashes: make(map[uint64]string, elBlockHistory), + filters: make(map[string]uint64), + } +} + +// init positions the virtual execution head at the last block produced at or before the +// replay's start time, found by bisecting the upstream chain. +func (p *elProxy) init(ctx context.Context, startTime time.Time) error { + latest, err := p.blockByTag(ctx, "latest") + if err != nil { + return err + } + + if latest == nil { + return fmt.Errorf("execution upstream has no latest block") + } + + if !latest.timestamp.After(startTime) { + p.setHead(latest) + return nil + } + + low, high := uint64(0), latest.number + + for low < high { + mid := (low + high + 1) / 2 + + block, err := p.blockByNumber(ctx, mid) + if err != nil { + return err + } + + if block == nil || block.timestamp.After(startTime) { + high = mid - 1 + } else { + low = mid + } + } + + block, err := p.blockByNumber(ctx, low) + if err != nil { + return err + } + + if block == nil { + return fmt.Errorf("could not resolve execution block %v", low) + } + + p.setHead(block) + + p.logger.WithFields(logrus.Fields{ + "block": block.number, + "time": block.timestamp.UTC().Format(time.RFC3339), + }).Info("resolved execution head") + + return nil +} + +// advanceTo moves the virtual head forward over every block produced at or before the +// given slot time, and returns the blocks it took in. +func (p *elProxy) advanceTo(ctx context.Context, slotTime time.Time) ([]elBlock, error) { + added := []elBlock{} + + for { + next, err := p.blockByNumber(ctx, p.head()+1) + if err != nil { + return added, err + } + + if next == nil || next.timestamp.After(slotTime) { + return added, nil + } + + p.setHead(next) + added = append(added, *next) + } +} + +func (p *elProxy) head() uint64 { + p.mutex.RLock() + defer p.mutex.RUnlock() + + return p.headNumber +} + +func (p *elProxy) setHead(block *elBlock) { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.headNumber = block.number + p.headHash = block.hash + p.blockHashes[block.number] = block.hash + + if block.number >= elBlockHistory { + delete(p.blockHashes, block.number-elBlockHistory) + } +} + +// -- JSON-RPC ---------------------------------------------------------------------- + +type rpcRequest struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params []json.RawMessage `json:"params"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +func (p *elProxy) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + http.Error(w, "only POST is supported", http.StatusMethodNotAllowed) + return + } + + body, err := io.ReadAll(req.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + http.Error(w, "empty request", http.StatusBadRequest) + return + } + + if trimmed[0] == '[' { + requests := []rpcRequest{} + if err := json.Unmarshal(trimmed, &requests); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + writeJSON(w, http.StatusOK, p.handleBatch(req.Context(), requests)) + + return + } + + request := rpcRequest{} + if err := json.Unmarshal(trimmed, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + writeJSON(w, http.StatusOK, p.handle(req.Context(), &request)) +} + +// handleBatch answers a batched call, forwarding everything it cannot answer locally as +// one batch. Splitting a batch into individual upstream calls would turn a single round +// trip into one per entry, which is exactly what the caller batched to avoid. +func (p *elProxy) handleBatch(ctx context.Context, requests []rpcRequest) []*rpcResponse { + responses := make([]*rpcResponse, len(requests)) + forwarded := make([]*rpcRequest, 0, len(requests)) + positions := make([]int, 0, len(requests)) + + for i := range requests { + local, forward := p.prepare(&requests[i]) + if local != nil { + responses[i] = local + continue + } + + forwarded = append(forwarded, forward) + positions = append(positions, i) + } + + if len(forwarded) == 0 { + return responses + } + + results := p.forwardBatch(ctx, forwarded) + for n, position := range positions { + responses[position] = results[n] + } + + return responses +} + +func (p *elProxy) handle(ctx context.Context, request *rpcRequest) *rpcResponse { + local, forward := p.prepare(request) + if local != nil { + return local + } + + return p.forward(ctx, forward) +} + +// prepare answers a call from the replay's own state where it can, and otherwise +// returns the (possibly rewritten) request to forward upstream. +func (p *elProxy) prepare(request *rpcRequest) (*rpcResponse, *rpcRequest) { + switch request.Method { + case "eth_blockNumber": + return p.reply(request, hexUint(p.head())), nil + + case "eth_syncing": + return p.reply(request, false), nil + + case "eth_newBlockFilter": + return p.reply(request, p.newBlockFilter()), nil + + case "eth_getFilterChanges": + return p.handleFilterChanges(request), nil + + case "eth_uninstallFilter": + return p.handleUninstallFilter(request), nil + + case "eth_getLogs": + return p.clampGetLogs(request) + } + + if index, hasTag := blockTagParam[request.Method]; hasTag { + pinned, visible, err := p.pinBlockTag(request, index) + if err != nil { + return p.replyError(request, -32602, err.Error()), nil + } + + if !visible { + // the requested block is beyond the virtual head; a node that does not + // have it yet answers with no result + return p.reply(request, nil), nil + } + + return nil, pinned + } + + return nil, request +} + +// pinBlockTag rewrites a symbolic block tag to the virtual head and reports whether a +// numeric block reference is at or below it. +func (p *elProxy) pinBlockTag(request *rpcRequest, index int) (*rpcRequest, bool, error) { + if index > len(request.Params) { + return nil, false, fmt.Errorf("%v is missing parameter %v", request.Method, index) + } + + if index == len(request.Params) { + // the tag is optional and was omitted, which means `latest`; appending the + // pinned head keeps the meaning the caller intended + pinned := *request + pinned.Params = append(append([]json.RawMessage{}, request.Params...), quotedHex(p.head())) + + return &pinned, true, nil + } + + raw := request.Params[index] + + tag := "" + if err := json.Unmarshal(raw, &tag); err != nil { + // not a string: eth_getBlockByNumber never sees this, but block-hash objects + // (eth_getProof style) are passed through untouched + return request, true, nil + } + + if blockTags[tag] { + pinned := *request + pinned.Params = append([]json.RawMessage{}, request.Params...) + pinned.Params[index] = quotedHex(p.head()) + + return &pinned, true, nil + } + + if tag == "earliest" || !strings.HasPrefix(tag, "0x") { + return request, true, nil + } + + number, err := strconv.ParseUint(strings.TrimPrefix(tag, "0x"), 16, 64) + if err != nil { + return nil, false, fmt.Errorf("invalid block number %q", tag) + } + + return request, number <= p.head(), nil +} + +func (p *elProxy) newBlockFilter() string { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.filterSerial++ + id := fmt.Sprintf("0x%016x", p.filterSerial) + p.filters[id] = p.headNumber + + return id +} + +// handleFilterChanges returns the hashes of the blocks that entered the virtual chain +// since the filter was last polled. +func (p *elProxy) handleFilterChanges(request *rpcRequest) *rpcResponse { + id, err := stringParam(request, 0) + if err != nil { + return p.replyError(request, -32602, err.Error()) + } + + p.mutex.Lock() + delivered, exists := p.filters[id] + if !exists { + p.mutex.Unlock() + + return p.replyError(request, -32000, "filter not found") + } + + hashes := []string{} + + for number := delivered + 1; number <= p.headNumber; number++ { + if hash, ok := p.blockHashes[number]; ok { + hashes = append(hashes, hash) + } + } + + p.filters[id] = p.headNumber + p.mutex.Unlock() + + return p.reply(request, hashes) +} + +func (p *elProxy) handleUninstallFilter(request *rpcRequest) *rpcResponse { + id, err := stringParam(request, 0) + if err != nil { + return p.replyError(request, -32602, err.Error()) + } + + p.mutex.Lock() + _, exists := p.filters[id] + delete(p.filters, id) + p.mutex.Unlock() + + return p.reply(request, exists) +} + +// clampGetLogs clamps the requested range to the virtual head before forwarding, so a +// scan that asks for `latest` stops where the replay currently stands. +func (p *elProxy) clampGetLogs(request *rpcRequest) (*rpcResponse, *rpcRequest) { + if len(request.Params) == 0 { + return nil, request + } + + filter := map[string]json.RawMessage{} + if err := json.Unmarshal(request.Params[0], &filter); err != nil { + return p.replyError(request, -32602, "invalid log filter"), nil + } + + if _, byHash := filter["blockHash"]; byHash { + return nil, request + } + + head := p.head() + + from, err := blockNumberFromFilter(filter, "fromBlock", 0) + if err != nil { + return p.replyError(request, -32602, err.Error()), nil + } + + if from > head { + return p.reply(request, []any{}), nil + } + + to, err := blockNumberFromFilter(filter, "toBlock", head) + if err != nil { + return p.replyError(request, -32602, err.Error()), nil + } + + if to > head { + to = head + } + + filter["fromBlock"] = quotedHex(from) + filter["toBlock"] = quotedHex(to) + + encoded, err := json.Marshal(filter) + if err != nil { + return p.replyError(request, -32603, err.Error()), nil + } + + clamped := *request + clamped.Params = append([]json.RawMessage{encoded}, request.Params[1:]...) + + return nil, &clamped +} + +// blockNumberFromFilter reads a log filter bound, mapping symbolic tags to fallback. +func blockNumberFromFilter(filter map[string]json.RawMessage, field string, fallback uint64) (uint64, error) { + raw, exists := filter[field] + if !exists { + return fallback, nil + } + + value := "" + if err := json.Unmarshal(raw, &value); err != nil { + return 0, fmt.Errorf("invalid %v in log filter", field) + } + + if value == "earliest" { + return 0, nil + } + + if blockTags[value] { + return fallback, nil + } + + number, err := strconv.ParseUint(strings.TrimPrefix(value, "0x"), 16, 64) + if err != nil { + return 0, fmt.Errorf("invalid %v %q in log filter", field, value) + } + + return number, nil +} + +// forward passes a request to the real execution node and returns its answer verbatim. +func (p *elProxy) forward(ctx context.Context, request *rpcRequest) *rpcResponse { + body, err := p.post(ctx, request) + if err != nil { + return p.replyError(request, -32603, err.Error()) + } + + response := rpcResponse{} + if err := json.Unmarshal(body, &response); err != nil { + return p.replyError(request, -32603, fmt.Sprintf("invalid upstream response: %s", truncate(body, 200))) + } + + response.ID = request.ID + response.JSONRPC = "2.0" + + return &response +} + +// post sends a JSON-RPC payload to the execution upstream and returns the raw answer. +func (p *elProxy) post(ctx context.Context, payload any) ([]byte, error) { + encoded, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.url, bytes.NewReader(encoded)) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + + rsp, err := p.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = rsp.Body.Close() }() + + return io.ReadAll(rsp.Body) +} + +// forwardBatch passes several requests upstream in a single call and puts the answers +// back in the order they were asked, matching by id. +func (p *elProxy) forwardBatch(ctx context.Context, requests []*rpcRequest) []*rpcResponse { + if len(requests) == 1 { + return []*rpcResponse{p.forward(ctx, requests[0])} + } + + body, err := p.post(ctx, requests) + if err != nil { + return p.batchError(requests, err) + } + + upstreamResponses := []*rpcResponse{} + if err := json.Unmarshal(body, &upstreamResponses); err != nil { + return p.batchError(requests, fmt.Errorf("invalid upstream batch response: %s", truncate(body, 200))) + } + + byID := make(map[string]*rpcResponse, len(upstreamResponses)) + for _, response := range upstreamResponses { + byID[string(response.ID)] = response + } + + responses := make([]*rpcResponse, len(requests)) + + for i, request := range requests { + response, matched := byID[string(request.ID)] + if !matched { + if i < len(upstreamResponses) { + // an upstream that does not echo ids is still answering in order + response = upstreamResponses[i] + } else { + responses[i] = p.replyError(request, -32603, "upstream did not answer this batch entry") + continue + } + } + + response.ID = request.ID + response.JSONRPC = "2.0" + responses[i] = response + } + + return responses +} + +func (p *elProxy) batchError(requests []*rpcRequest, err error) []*rpcResponse { + responses := make([]*rpcResponse, len(requests)) + for i, request := range requests { + responses[i] = p.replyError(request, -32603, err.Error()) + } + + return responses +} + +// call issues a request of the replay's own to the execution upstream. +func (p *elProxy) call(ctx context.Context, method string, params ...any) (json.RawMessage, error) { + encoded := make([]json.RawMessage, 0, len(params)) + + for _, param := range params { + raw, err := json.Marshal(param) + if err != nil { + return nil, err + } + + encoded = append(encoded, raw) + } + + p.mutex.Lock() + p.requestID++ + id := p.requestID + p.mutex.Unlock() + + request := &rpcRequest{ + JSONRPC: "2.0", + ID: json.RawMessage(strconv.FormatUint(id, 10)), + Method: method, + Params: encoded, + } + + response := p.forward(ctx, request) + if response.Error != nil { + return nil, fmt.Errorf("%v: %v", method, response.Error.Message) + } + + return response.Result, nil +} + +func (p *elProxy) blockByTag(ctx context.Context, tag string) (*elBlock, error) { + return p.decodeBlock(p.call(ctx, "eth_getBlockByNumber", tag, false)) +} + +func (p *elProxy) blockByNumber(ctx context.Context, number uint64) (*elBlock, error) { + return p.decodeBlock(p.call(ctx, "eth_getBlockByNumber", fmt.Sprintf("0x%x", number), false)) +} + +func (p *elProxy) decodeBlock(result json.RawMessage, err error) (*elBlock, error) { + if err != nil { + return nil, err + } + + if len(result) == 0 || string(result) == "null" { + return nil, nil + } + + header := struct { + Number string `json:"number"` + Hash string `json:"hash"` + Timestamp string `json:"timestamp"` + }{} + + if err := json.Unmarshal(result, &header); err != nil { + return nil, fmt.Errorf("error parsing execution block: %w", err) + } + + number, err := strconv.ParseUint(strings.TrimPrefix(header.Number, "0x"), 16, 64) + if err != nil { + return nil, fmt.Errorf("invalid block number %q: %w", header.Number, err) + } + + timestamp, err := strconv.ParseInt(strings.TrimPrefix(header.Timestamp, "0x"), 16, 64) + if err != nil { + return nil, fmt.Errorf("invalid block timestamp %q: %w", header.Timestamp, err) + } + + return &elBlock{ + number: number, + hash: header.Hash, + timestamp: time.Unix(timestamp, 0).UTC(), + }, nil +} + +func (p *elProxy) reply(request *rpcRequest, result any) *rpcResponse { + encoded, err := json.Marshal(result) + if err != nil { + return p.replyError(request, -32603, err.Error()) + } + + return &rpcResponse{JSONRPC: "2.0", ID: request.ID, Result: encoded} +} + +func (p *elProxy) replyError(request *rpcRequest, code int, message string) *rpcResponse { + return &rpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + Error: &rpcError{Code: code, Message: message}, + } +} + +func stringParam(request *rpcRequest, index int) (string, error) { + if index >= len(request.Params) { + return "", fmt.Errorf("missing parameter %v", index) + } + + value := "" + if err := json.Unmarshal(request.Params[index], &value); err != nil { + return "", fmt.Errorf("parameter %v is not a string", index) + } + + return value, nil +} + +func hexUint(value uint64) string { + return fmt.Sprintf("0x%x", value) +} + +func quotedHex(value uint64) json.RawMessage { + return json.RawMessage(fmt.Sprintf("%q", hexUint(value))) +} diff --git a/replay/elproxy_test.go b/replay/elproxy_test.go new file mode 100644 index 00000000..df4b6641 --- /dev/null +++ b/replay/elproxy_test.go @@ -0,0 +1,277 @@ +package replay + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func testELProxy(head uint64) *elProxy { + proxy := newELProxy(logrus.New(), "http://localhost:0") + proxy.setHead(&elBlock{number: head, hash: "0xdeadbeef"}) + + return proxy +} + +func rawParams(t *testing.T, values ...any) []json.RawMessage { + t.Helper() + + params := make([]json.RawMessage, 0, len(values)) + + for _, value := range values { + encoded, err := json.Marshal(value) + require.NoError(t, err) + + params = append(params, encoded) + } + + return params +} + +func TestPinBlockTag(t *testing.T) { + proxy := testELProxy(500) + + tests := []struct { + name string + method string + params []any + wantParam string + wantVisible bool + }{ + { + name: "latest is pinned to the virtual head", + method: "eth_getBlockByNumber", + params: []any{"latest", false}, + wantParam: "0x1f4", + wantVisible: true, + }, + { + name: "finalized is approximated by the virtual head", + method: "eth_getBlockByNumber", + params: []any{"finalized", false}, + wantParam: "0x1f4", + wantVisible: true, + }, + { + name: "a block at the head stays visible", + method: "eth_getBlockByNumber", + params: []any{"0x1f4", false}, + wantParam: "0x1f4", + wantVisible: true, + }, + { + name: "a block beyond the head is hidden", + method: "eth_getBlockByNumber", + params: []any{"0x1f5", false}, + wantVisible: false, + }, + { + name: "earliest is left alone", + method: "eth_getBlockByNumber", + params: []any{"earliest", false}, + wantParam: "earliest", + wantVisible: true, + }, + { + name: "an omitted tag is appended as the head", + method: "eth_getBalance", + params: []any{"0x0000000000000000000000000000000000000001"}, + wantParam: "0x1f4", + wantVisible: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := &rpcRequest{Method: test.method, Params: rawParams(t, test.params...)} + index := blockTagParam[test.method] + + pinned, visible, err := proxy.pinBlockTag(request, index) + require.NoError(t, err) + require.Equal(t, test.wantVisible, visible) + + if !visible { + return + } + + value := "" + require.NoError(t, json.Unmarshal(pinned.Params[index], &value)) + require.Equal(t, test.wantParam, value) + }) + } +} + +func TestBlockNumberFromFilter(t *testing.T) { + filter := map[string]json.RawMessage{ + "fromBlock": json.RawMessage(`"0x10"`), + "toBlock": json.RawMessage(`"latest"`), + } + + from, err := blockNumberFromFilter(filter, "fromBlock", 0) + require.NoError(t, err) + require.Equal(t, uint64(16), from) + + to, err := blockNumberFromFilter(filter, "toBlock", 999) + require.NoError(t, err) + require.Equal(t, uint64(999), to) + + missing, err := blockNumberFromFilter(filter, "blockHash", 42) + require.NoError(t, err) + require.Equal(t, uint64(42), missing) + + _, err = blockNumberFromFilter(map[string]json.RawMessage{ + "fromBlock": json.RawMessage(`"zzz"`), + }, "fromBlock", 0) + require.Error(t, err) +} + +func TestBlockFilterDeliversNewBlocks(t *testing.T) { + proxy := testELProxy(100) + + filterID := proxy.newBlockFilter() + + // nothing has been produced since the filter was created + response := proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, filterID)}) + require.Nil(t, response.Error) + require.JSONEq(t, `[]`, string(response.Result)) + + proxy.setHead(&elBlock{number: 101, hash: "0xaa"}) + proxy.setHead(&elBlock{number: 102, hash: "0xbb"}) + + response = proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, filterID)}) + require.Nil(t, response.Error) + require.JSONEq(t, `["0xaa","0xbb"]`, string(response.Result)) + + // a second poll without new blocks returns nothing again + response = proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, filterID)}) + require.JSONEq(t, `[]`, string(response.Result)) + + // an unknown filter is reported the way a node reports an expired one + response = proxy.handleFilterChanges(&rpcRequest{Params: rawParams(t, "0xdead")}) + require.NotNil(t, response.Error) + require.Contains(t, response.Error.Message, "filter not found") +} + +func TestClampGetLogsToVirtualHead(t *testing.T) { + proxy := testELProxy(500) + + tests := []struct { + name string + filter string + wantFrom string + wantTo string + wantLocal string + }{ + { + name: "latest is clamped to the virtual head", + filter: `{"fromBlock":"0x10","toBlock":"latest"}`, + wantFrom: `"0x10"`, + wantTo: `"0x1f4"`, + }, + { + name: "a range beyond the head is truncated", + filter: `{"fromBlock":"0x10","toBlock":"0x999"}`, + wantFrom: `"0x10"`, + wantTo: `"0x1f4"`, + }, + { + name: "a missing toBlock defaults to the head", + filter: `{"fromBlock":"0x10"}`, + wantFrom: `"0x10"`, + wantTo: `"0x1f4"`, + }, + { + name: "a range entirely beyond the head yields no logs", + filter: `{"fromBlock":"0x500","toBlock":"0x600"}`, + wantLocal: `[]`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := &rpcRequest{ + Method: "eth_getLogs", + Params: []json.RawMessage{json.RawMessage(test.filter)}, + } + + local, forward := proxy.clampGetLogs(request) + + if test.wantLocal != "" { + require.NotNil(t, local) + require.JSONEq(t, test.wantLocal, string(local.Result)) + + return + } + + require.Nil(t, local) + require.NotNil(t, forward) + + clamped := map[string]json.RawMessage{} + require.NoError(t, json.Unmarshal(forward.Params[0], &clamped)) + require.Equal(t, test.wantFrom, string(clamped["fromBlock"])) + require.Equal(t, test.wantTo, string(clamped["toBlock"])) + }) + } +} + +// TestBatchIsForwardedAsOneCall guards the property that makes batching worth anything: +// a batched call must reach the upstream as a single request, not as one per entry. +func TestBatchIsForwardedAsOneCall(t *testing.T) { + upstreamCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamCalls++ + + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + + requests := []rpcRequest{} + require.NoError(t, json.Unmarshal(body, &requests)) + + responses := make([]rpcResponse, 0, len(requests)) + for _, request := range requests { + responses = append(responses, rpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + Result: json.RawMessage(`"ok"`), + }) + } + + writeJSON(w, http.StatusOK, responses) + })) + defer server.Close() + + proxy := newELProxy(logrus.New(), server.URL) + proxy.setHead(&elBlock{number: 500, hash: "0xdead"}) + + batch := []rpcRequest{ + {JSONRPC: "2.0", ID: json.RawMessage(`1`), Method: "eth_getTransactionReceipt", Params: rawParams(t, "0xaa")}, + {JSONRPC: "2.0", ID: json.RawMessage(`2`), Method: "eth_getTransactionReceipt", Params: rawParams(t, "0xbb")}, + {JSONRPC: "2.0", ID: json.RawMessage(`3`), Method: "eth_blockNumber"}, + } + + responses := proxy.handleBatch(context.Background(), batch) + + require.Len(t, responses, 3) + require.Equal(t, 1, upstreamCalls, "the two forwarded entries must share one upstream call") + require.JSONEq(t, `"ok"`, string(responses[0].Result)) + require.JSONEq(t, `"ok"`, string(responses[1].Result)) + require.JSONEq(t, `"0x1f4"`, string(responses[2].Result), "eth_blockNumber is answered locally") +} + +func TestUninstallFilter(t *testing.T) { + proxy := testELProxy(100) + filterID := proxy.newBlockFilter() + + response := proxy.handleUninstallFilter(&rpcRequest{Params: rawParams(t, filterID)}) + require.JSONEq(t, `true`, string(response.Result)) + + response = proxy.handleUninstallFilter(&rpcRequest{Params: rawParams(t, filterID)}) + require.JSONEq(t, `false`, string(response.Result)) +} diff --git a/replay/events.go b/replay/events.go new file mode 100644 index 00000000..3f215d39 --- /dev/null +++ b/replay/events.go @@ -0,0 +1,181 @@ +package replay + +import ( + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// sseEvent is one entry of the beacon node event stream. +type sseEvent struct { + topic string + data []byte +} + +// eventHub is the fake node's /eth/v1/events endpoint: a fan-out of synthesized chain +// events to every connected subscriber, filtered by the topics it asked for. +type eventHub struct { + logger logrus.FieldLogger + + // lossy hubs drop an event for a subscriber whose buffer is full instead of + // waiting for it. Chain events must never be lost, but status snapshots should + // never hold the replay up either. + lossy bool + + mutex sync.Mutex + subscribers map[*eventSubscriber]struct{} +} + +type eventSubscriber struct { + topics map[string]bool + events chan sseEvent + + // done is closed when the subscriber disconnects, so a publisher waiting on a + // slow reader is released instead of blocking forever. + done chan struct{} +} + +func newEventSubscriber(topics map[string]bool) *eventSubscriber { + return &eventSubscriber{ + topics: topics, + events: make(chan sseEvent, 256), + done: make(chan struct{}), + } +} + +func newEventHub(logger logrus.FieldLogger) *eventHub { + return &eventHub{ + logger: logger, + subscribers: make(map[*eventSubscriber]struct{}), + } +} + +// newLossyEventHub returns a hub that never blocks its publisher. +func newLossyEventHub(logger logrus.FieldLogger) *eventHub { + hub := newEventHub(logger) + hub.lossy = true + + return hub +} + +// publish delivers an event to every subscriber that asked for its topic. It waits for +// a subscriber that is behind rather than dropping the event: the replay sets the pace, +// so a slow explorer should slow the replay down, not silently lose a block. This is +// what makes "advance as fast as upstream allows" self-throttle to what the explorer +// can actually index. +func (h *eventHub) publish(event sseEvent) { + h.mutex.Lock() + targets := make([]*eventSubscriber, 0, len(h.subscribers)) + + for sub := range h.subscribers { + if sub.topics[event.topic] { + targets = append(targets, sub) + } + } + h.mutex.Unlock() + + for _, sub := range targets { + if h.lossy { + select { + case sub.events <- event: + default: + } + + continue + } + + select { + case sub.events <- event: + case <-sub.done: + h.logger.Debugf("subscriber disconnected while waiting to take a %v event", event.topic) + } + } +} + +// subscriberCount reports how many event streams are currently connected. +func (h *eventHub) subscriberCount() int { + h.mutex.Lock() + defer h.mutex.Unlock() + + return len(h.subscribers) +} + +func (h *eventHub) add(sub *eventSubscriber) { + h.mutex.Lock() + defer h.mutex.Unlock() + + h.subscribers[sub] = struct{}{} +} + +func (h *eventHub) remove(sub *eventSubscriber) { + h.mutex.Lock() + delete(h.subscribers, sub) + h.mutex.Unlock() + + close(sub.done) +} + +// serveHTTP implements the /eth/v1/events endpoint. Every requested topic is accepted; +// topics the replay cannot synthesize (inclusion lists, fast confirmations) simply +// never fire, which clients already tolerate. +func (h *eventHub) serveHTTP(w http.ResponseWriter, r *http.Request) { + flusher, canFlush := w.(http.Flusher) + if !canFlush { + writeAPIError(w, http.StatusInternalServerError, "streaming not supported") + return + } + + topics := map[string]bool{} + for _, topic := range strings.Split(r.URL.Query().Get("topics"), ",") { + if topic = strings.TrimSpace(topic); topic != "" { + topics[topic] = true + } + } + + if len(topics) == 0 { + if !h.lossy { + writeAPIError(w, http.StatusBadRequest, "no topics requested") + return + } + + // the control stream carries one topic, so asking for it is optional + topics["status"] = true + } + + sub := newEventSubscriber(topics) + + h.add(sub) + defer h.remove(sub) + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flusher.Flush() + + keepalive := time.NewTicker(15 * time.Second) + defer keepalive.Stop() + + for { + select { + case <-r.Context().Done(): + return + case <-keepalive.C: + if _, err := fmt.Fprint(w, ":keepalive\n\n"); err != nil { + return + } + + flusher.Flush() + case event := <-sub.events: + if _, err := fmt.Fprintf(w, "event: %s\ndata: %s\n\n", event.topic, event.data); err != nil { + return + } + + flusher.Flush() + } + } +} diff --git a/replay/events_test.go b/replay/events_test.go new file mode 100644 index 00000000..176de0bc --- /dev/null +++ b/replay/events_test.go @@ -0,0 +1,78 @@ +package replay + +import ( + "bufio" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestEventHubFiltersByTopic(t *testing.T) { + hub := newEventHub(logrus.New()) + + blocks := newEventSubscriber(map[string]bool{"block": true}) + heads := newEventSubscriber(map[string]bool{"head": true}) + + hub.add(blocks) + hub.add(heads) + + require.Equal(t, 2, hub.subscriberCount()) + + hub.publish(sseEvent{topic: "block", data: []byte(`{"slot":"1"}`)}) + + require.Len(t, blocks.events, 1) + require.Empty(t, heads.events, "a subscriber must not receive topics it did not ask for") + + hub.remove(blocks) + require.Equal(t, 1, hub.subscriberCount()) +} + +func TestEventStreamServesSSE(t *testing.T) { + hub := newEventHub(logrus.New()) + + server := httptest.NewServer(http.HandlerFunc(hub.serveHTTP)) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"?topics=head,block", http.NoBody) + require.NoError(t, err) + + rsp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + defer func() { _ = rsp.Body.Close() }() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "text/event-stream", rsp.Header.Get("Content-Type")) + + require.Eventually(t, func() bool { return hub.subscriberCount() == 1 }, 5*time.Second, 5*time.Millisecond) + + hub.publish(sseEvent{topic: "head", data: []byte(`{"slot":"42"}`)}) + + reader := bufio.NewReader(rsp.Body) + + eventLine, err := reader.ReadString('\n') + require.NoError(t, err) + require.Equal(t, "event: head", strings.TrimSpace(eventLine)) + + dataLine, err := reader.ReadString('\n') + require.NoError(t, err) + require.Equal(t, `data: {"slot":"42"}`, strings.TrimSpace(dataLine)) +} + +func TestEventStreamRejectsEmptyTopics(t *testing.T) { + hub := newEventHub(logrus.New()) + + recorder := httptest.NewRecorder() + hub.serveHTTP(recorder, httptest.NewRequest(http.MethodGet, "/eth/v1/events", http.NoBody)) + + require.Equal(t, http.StatusBadRequest, recorder.Code) +} diff --git a/replay/example-config.yaml b/replay/example-config.yaml new file mode 100644 index 00000000..2ed94b15 --- /dev/null +++ b/replay/example-config.yaml @@ -0,0 +1,96 @@ +# Example explorer config for a dora-replay run. Copy it to config-.yaml, point +# the endpoints at your dora-replay listeners, and start the explorer with it. +logging: + outputLevel: "info" + +chain: + displayName: "devnet (replay)" + +server: + host: "0.0.0.0" + port: "8083" + +frontend: + enabled: true + debug: true + pprof: true + minimize: false + + siteName: "Dora the Explorer" + siteSubtitle: "replay" + + ethExplorerLink: "" + + validatorNamesInventory: "https://config.glamsterdam-devnet-8.ethpandaops.io/api/v1/nodes/validator-ranges" + + publicRpcUrl: "" + showSubmitDeposit: true + showSubmitElRequests: true + showValidatorSummary: true + + # the replay steps slot by slot, so cached pages would hide what just changed + disablePageCache: true + + tracoorUrl: "https://tracoor.glamsterdam-devnet-8.ethpandaops.io" + tracoorNetwork: "glamsterdam-devnet-8" + +api: + enabled: true + +# drive every "now" in the explorer off the dora-replay control server instead of the +# real wall clock. the endpoints below must point at the matching dora-replay proxies. +replay: + enabled: true + controlUrl: "http://127.0.0.1:15000" + +beaconapi: + endpoints: + - url: "http://127.0.0.1:15052" + name: "replay-cl" + localCacheSize: 100 + +executionapi: + endpoints: + - url: "http://127.0.0.1:15545" + name: "replay-el" + logBatchSize: 1000 + genesisConfig: "https://config.glamsterdam-devnet-8.ethpandaops.io/el/genesis.json" + +mevIndexer: + relays: [] + refreshInterval: 5m + +indexer: + inMemoryEpochs: 6 + + # the replay starts mid-chain, so there is no history to walk back to: without this + # the synchronizer would try to backfill from epoch 0, and the upstreams no longer + # have the states to do it with. + # + # to repair a hole (e.g. after the explorer was restarted mid-finalization), flip + # this to false and add `resyncFromEpoch: ` for one run: the + # synchronizer then walks from there up to the live finalized epoch and stops. + disableSynchronizer: true + + pubkeyCachePath: "./temp/pubkeys-glamsterdam8-replay.db" + + stateCache: + enabled: true + path: "./temp/statecache-glamsterdam8-replay" + +executionIndexer: + enabled: true + retention: 336h + detailsEnabled: false + tracesEnabled: false + +database: + engine: "sqlite" + sqlite: + file: "tmp-db-glamsterdam8-replay.sqlite" + +blockDb: + engine: "pebble" + pebble: + path: "./temp/blockdb-glamsterdam8-replay.peb" + cacheSize: 100 diff --git a/replay/replay.go b/replay/replay.go new file mode 100644 index 00000000..970ef91b --- /dev/null +++ b/replay/replay.go @@ -0,0 +1,875 @@ +package replay + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "sync" + "time" + + v1 "github.com/ethpandaops/go-eth2-client/api/v1" + "github.com/ethpandaops/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" +) + +const ( + // headSearchDepth bounds how far back the replay looks for the newest block at or + // before the start slot before giving up. + headSearchDepth = 256 + + // finalityFallbackEpochs bounds how far back the replay looks for a state it can + // read finality from when the head state itself has been pruned upstream. + finalityFallbackEpochs = 8 + + // upstreamPollInterval is how often the head of the real chain is re-read, purely + // so the UI can show how far the replay still has to go. + upstreamPollInterval = 30 * time.Second + + // stateLoadLeadSlots is how many slots the replay may still serve while the explorer + // is loading a beacon state. It is the slack that keeps blocks flowing during a + // multi-second read, so the explorer's block indexer is not idled by a wait that + // only its state loader is in. + stateLoadLeadSlots = 4 +) + +// Replay steps a past slot range through a fake beacon/execution node pair, driving a +// virtual clock so an explorer pointed at it sees the range unfold as if it were live. +type Replay struct { + logger logrus.FieldLogger + cfg Config + chain *chainInfo + upstream *upstream + events *eventHub + control *eventHub + clock *clock + el *elProxy + states *stateLoads + + // wake nudges the driver whenever the drive state changed. + wake chan struct{} + + // stateLeadFrom is the slot at which the explorer last started loading a state, or + // 0 when it is not loading one. Driver-owned: only advanceSlot touches it. + stateLeadFrom uint64 + + mutex sync.RWMutex + virtualSlot uint64 + head *blockHeader + finality *finalityCheckpoints + running bool + speed float64 + target uint64 + upstreamSlot uint64 + upstreamAtUTC time.Time + + servers []*http.Server +} + +// New prepares a replay: it reads genesis and the timing specs from upstream, resolves +// the head at the start slot and positions the virtual clock there. +func New(ctx context.Context, logger logrus.FieldLogger, cfg Config) (*Replay, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + + up, err := newUpstream(logger.WithField("module", "upstream"), &cfg) + if err != nil { + return nil, err + } + + chain, err := loadChainInfo(ctx, up) + if err != nil { + return nil, err + } + + up.chain = chain + + logger.WithFields(logrus.Fields{ + "genesis": chain.genesisTime.Format(time.RFC3339), + "slotDuration": chain.slotDuration, + "slotsPerEpoch": chain.slotsPerEpoch, + }).Info("loaded chain info from upstream") + + replay := &Replay{ + logger: logger, + cfg: cfg, + chain: chain, + upstream: up, + events: newEventHub(logger.WithField("module", "events")), + control: newLossyEventHub(logger.WithField("module", "control")), + states: newStateLoads(), + clock: newClock(chain.slotTime(cfg.StartSlot).Add(chain.payloadOffset())), + wake: make(chan struct{}, 1), + virtualSlot: cfg.StartSlot, + } + + if cfg.ExecutionURL != "" { + replay.el = newELProxy(logger.WithField("module", "el"), cfg.ExecutionURL) + } + + if err := replay.initHead(ctx); err != nil { + return nil, err + } + + if replay.el != nil { + if err := replay.el.init(ctx, chain.slotTime(cfg.StartSlot)); err != nil { + return nil, fmt.Errorf("error initializing execution head: %w", err) + } + } + + return replay, nil +} + +// blockOffset is how far into a slot the block and head events are replayed, and +// payloadOffset when the payload becomes available. Both approximate real node timing. +func (c *chainInfo) blockOffset() time.Duration { + return c.slotDuration / 3 +} + +func (c *chainInfo) payloadOffset() time.Duration { + return c.slotDuration * 2 / 3 +} + +// stateGateOffset is how far into a slot the replay checks whether the explorer is +// still loading a beacon state. +func (c *chainInfo) stateGateOffset() time.Duration { + return c.slotDuration / 2 +} + +// initHead walks back from the start slot to the newest block, which becomes the head +// the fake node reports before the first step. +func (r *Replay) initHead(ctx context.Context) error { + slot := r.cfg.StartSlot + + for depth := 0; depth < headSearchDepth; depth++ { + header, err := r.upstream.headerBySlot(ctx, slot) + if err != nil { + return fmt.Errorf("error resolving head at slot %v: %w", slot, err) + } + + if header != nil { + r.mutex.Lock() + r.head = header + r.mutex.Unlock() + + if err := r.refreshFinality(ctx, header); err != nil { + return err + } + + r.logger.WithFields(logrus.Fields{ + "slot": header.Slot, + "root": header.Root, + "finalized": r.finality.FinalizedEpoch, + }).Info("resolved replay head") + + return nil + } + + if slot == 0 { + break + } + + slot-- + } + + return fmt.Errorf("no block found within %v slots before slot %v", headSearchDepth, r.cfg.StartSlot) +} + +// Start brings up the fake nodes and the control server and starts the driver. +func (r *Replay) Start(ctx context.Context) error { + if err := r.listen(ctx, r.cfg.CLListen, r.clHandler(), "consensus"); err != nil { + return err + } + + if r.el != nil { + if err := r.listen(ctx, r.cfg.ELListen, r.el, "execution"); err != nil { + return err + } + } + + if err := r.listen(ctx, r.cfg.ControlListen, r.controlHandler(), "control"); err != nil { + return err + } + + go r.runDriver(ctx) + go r.runUpstreamPoller(ctx) + + if r.cfg.Speed > 0 { + r.Play(r.cfg.Speed) + } + + return nil +} + +func (r *Replay) listen(ctx context.Context, addr string, handler http.Handler, name string) error { + listener, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("could not listen on %v for the %v endpoint: %w", addr, name, err) + } + + server := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 30 * time.Second, + } + + r.servers = append(r.servers, server) + + go func() { + if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { + r.logger.WithError(err).Errorf("%v endpoint stopped", name) + } + }() + + go func() { + <-ctx.Done() + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + r.logger.WithError(err).Debugf("error shutting down %v endpoint", name) + } + }() + + r.logger.Infof("%v endpoint listening on %v", name, addr) + + return nil +} + +// Stop shuts the fake nodes down. +func (r *Replay) Stop() error { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, server := range r.servers { + if err := server.Shutdown(shutdownCtx); err != nil { + return err + } + } + + return nil +} + +// -- drive state ------------------------------------------------------------------ + +// Play runs the replay continuously at the given multiple of real time. +func (r *Replay) Play(speed float64) { + r.mutex.Lock() + r.running = true + r.speed = speed + r.target = 0 + r.mutex.Unlock() + + r.clock.setRate(speed) + r.nudge() + r.notifyStatus() +} + +// Step advances a fixed number of slots as fast as upstream allows, then pauses. +func (r *Replay) Step(slots uint64) { + if slots == 0 { + slots = 1 + } + + r.mutex.Lock() + r.running = true + r.speed = 0 + r.target = r.virtualSlot + slots + r.mutex.Unlock() + + r.clock.setRate(0) + r.nudge() + r.notifyStatus() +} + +// Forward runs to a slot, emitting every slot on the way, then pauses. A speed of 0 +// advances as fast as upstream allows; anything else plays at that multiple of real +// time, which gives the explorer a predictable amount of time per slot to keep up. +func (r *Replay) Forward(slot uint64, speed float64) error { + r.mutex.Lock() + if slot <= r.virtualSlot { + current := r.virtualSlot + r.mutex.Unlock() + + return fmt.Errorf("slot %v is not ahead of the current head slot %v (the replay cannot rewind)", slot, current) + } + + r.running = true + r.speed = speed + r.target = slot + r.mutex.Unlock() + + r.clock.setRate(speed) + r.nudge() + r.notifyStatus() + + return nil +} + +// Pause freezes the replay and the virtual clock where they are. +func (r *Replay) Pause() { + r.mutex.Lock() + r.running = false + r.mutex.Unlock() + + r.clock.setRate(0) + r.nudge() + r.notifyStatus() +} + +// Resume continues what the replay was doing before it was paused, keeping both the +// speed and any target it was running towards. A target that has already been reached +// is dropped, so resuming after a completed step runs on rather than pausing again. +func (r *Replay) Resume() { + r.mutex.Lock() + r.running = true + if r.target != 0 && r.target <= r.virtualSlot { + r.target = 0 + } + speed := r.speed + r.mutex.Unlock() + + r.clock.setRate(speed) + r.nudge() + r.notifyStatus() +} + +// SetSpeed changes the playback rate without starting or stopping the replay. +func (r *Replay) SetSpeed(speed float64) { + if speed < 0 { + speed = 0 + } + + r.mutex.Lock() + r.speed = speed + running := r.running + r.mutex.Unlock() + + if running { + r.clock.setRate(speed) + } + + r.nudge() + r.notifyStatus() +} + +func (r *Replay) nudge() { + select { + case r.wake <- struct{}{}: + default: + } +} + +func (r *Replay) isRunning() bool { + r.mutex.RLock() + defer r.mutex.RUnlock() + + return r.running +} + +// Status is a snapshot of the replay for the console, the control endpoint and the +// explorer's own replay UI. +type Status struct { + VirtualSlot uint64 `json:"virtual_slot"` + VirtualEpoch uint64 `json:"virtual_epoch"` + VirtualTime time.Time `json:"virtual_time"` + + HeadSlot uint64 `json:"head_slot"` + HeadRoot string `json:"head_root"` + FinalizedEpoch uint64 `json:"finalized_epoch"` + JustifiedEpoch uint64 `json:"justified_epoch"` + ExecutionBlock uint64 `json:"execution_block"` + + Running bool `json:"running"` + Speed float64 `json:"speed"` + Rate float64 `json:"rate"` + StartSlot uint64 `json:"start_slot"` + TargetSlot uint64 `json:"target_slot"` + + // StateLoads is how many beacon states the explorer is pulling right now, and + // Holding says the replay's clock is frozen waiting for them. + StateLoads int `json:"state_loads"` + Holding bool `json:"holding"` + + // UpstreamSlot is the head of the real chain, so the UI can show how much of it is + // still ahead of the replay. + UpstreamSlot uint64 `json:"upstream_slot"` + UpstreamEpoch uint64 `json:"upstream_epoch"` + UpstreamSeen time.Time `json:"upstream_seen"` + + GenesisTime time.Time `json:"genesis_time"` + SlotDurationMs uint64 `json:"slot_duration_ms"` + SlotsPerEpoch uint64 `json:"slots_per_epoch"` + + Subscribers int `json:"event_subscribers"` + Upstream string `json:"upstream"` + Tracoor bool `json:"tracoor"` +} + +// Status returns a snapshot of where the replay currently stands. +func (r *Replay) Status() Status { + r.mutex.RLock() + defer r.mutex.RUnlock() + + virtualTime, rate := r.clock.state() + + status := Status{ + VirtualSlot: r.virtualSlot, + VirtualEpoch: r.chain.epochOf(r.virtualSlot), + VirtualTime: virtualTime.UTC(), + Running: r.running, + Speed: r.speed, + Rate: rate, + StartSlot: r.cfg.StartSlot, + TargetSlot: r.target, + UpstreamSlot: r.upstreamSlot, + UpstreamEpoch: r.chain.epochOf(r.upstreamSlot), + UpstreamSeen: r.upstreamAtUTC, + GenesisTime: r.chain.genesisTime, + SlotDurationMs: uint64(r.chain.slotDuration / time.Millisecond), + SlotsPerEpoch: r.chain.slotsPerEpoch, + StateLoads: r.states.count(), + Holding: r.clock.isHeld(), + Subscribers: r.events.subscriberCount(), + Upstream: redactURL(r.cfg.UpstreamURL), + Tracoor: r.upstream.tracoor != nil, + } + + if r.head != nil { + status.HeadSlot = r.head.Slot + status.HeadRoot = r.head.Root + } + + if r.finality != nil { + status.FinalizedEpoch = r.finality.FinalizedEpoch + status.JustifiedEpoch = r.finality.JustifiedEpoch + } + + if r.el != nil { + status.ExecutionBlock = r.el.head() + } + + return status +} + +// -- driver ----------------------------------------------------------------------- + +func (r *Replay) runDriver(ctx context.Context) { + for { + if !r.isRunning() { + if err := r.awaitResume(ctx); err != nil { + return + } + + continue + } + + r.mutex.RLock() + next := r.virtualSlot + 1 + target := r.target + r.mutex.RUnlock() + + if target != 0 && next > target { + r.logger.Infof("reached slot %v, pausing", target) + r.Pause() + + continue + } + + if err := r.advanceSlot(ctx, next); err != nil { + if ctx.Err() != nil { + return + } + + r.logger.WithError(err).Errorf("failed advancing to slot %v, pausing", next) + r.Pause() + } + } +} + +// runUpstreamPoller keeps track of where the real chain currently is, which is what the +// replay is progressing through. It is only informational, so a failed read is ignored. +func (r *Replay) runUpstreamPoller(ctx context.Context) { + ticker := time.NewTicker(upstreamPollInterval) + defer ticker.Stop() + + for { + if header, err := r.upstream.headerByRoot(ctx, "head"); err == nil && header != nil { + r.mutex.Lock() + changed := r.upstreamSlot != header.Slot + r.upstreamSlot = header.Slot + r.upstreamAtUTC = time.Now().UTC() + r.mutex.Unlock() + + if changed { + r.notifyStatus() + } + } + + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (r *Replay) awaitResume(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.wake: + if r.isRunning() { + return nil + } + } + } +} + +// advanceSlot moves the replay to the given slot in three phases, mirroring how a real +// node experiences a slot: the boundary, the block, and the payload becoming available. +func (r *Replay) advanceSlot(ctx context.Context, slot uint64) error { + boundary := r.chain.slotTime(slot) + + if err := r.waitForVirtual(ctx, boundary); err != nil { + return err + } + + r.mutex.Lock() + r.virtualSlot = slot + r.mutex.Unlock() + + header, err := r.upstream.headerBySlot(ctx, slot) + if err != nil { + return fmt.Errorf("error fetching header: %w", err) + } + + if err := r.waitForVirtual(ctx, boundary.Add(r.chain.blockOffset())); err != nil { + return err + } + + if header != nil { + if err := r.emitBlock(ctx, header); err != nil { + return err + } + } + + // halfway through the slot, wait for any beacon state the explorer is loading. The + // clock is frozen while waiting, so a slow state read costs real time but no + // virtual time: without this the replay would run on and the explorer would be + // several slots behind the moment the state finally arrived. + if err := r.waitForVirtual(ctx, boundary.Add(r.chain.stateGateOffset())); err != nil { + return err + } + + if err := r.awaitStateLoads(ctx, slot); err != nil { + return err + } + + if err := r.waitForVirtual(ctx, boundary.Add(r.chain.payloadOffset())); err != nil { + return err + } + + if err := r.advanceExecution(ctx, slot, boundary, header); err != nil { + return err + } + + r.notifyStatus() + + if !r.isPlaying() { + if err := r.settle(ctx); err != nil { + return err + } + } + + return nil +} + +// awaitStateLoads keeps the replay from running away while the explorer is loading a +// beacon state, without stalling it outright. +// +// The explorer loads states on a different goroutine than it indexes blocks on, so a +// state read of several seconds does not stop it from processing blocks — it only stops +// it from finishing that epoch's stats. Holding the whole replay for the read therefore +// idles the block indexer for nothing, and emits no block or head events at all while it +// lasts. So the replay is allowed to run stateLoadLeadSlots further while a read is in +// flight, and only holds once it would get further ahead than that. +// +// Once it does hold, the clock is frozen, so the wait costs real time and no virtual +// time. It gives up after StateHoldTimeout so a stuck read cannot wedge the replay. +func (r *Replay) awaitStateLoads(ctx context.Context, slot uint64) error { + idle := r.states.idleChan() + + select { + case <-idle: + r.stateLeadFrom = 0 + + return nil + default: + } + + // first slot of this busy period: remember where the explorer started falling behind + if r.stateLeadFrom == 0 { + r.stateLeadFrom = slot + } + + if slot-r.stateLeadFrom < stateLoadLeadSlots { + // let the replay run on; the explorer can index these blocks meanwhile + return nil + } + + r.clock.hold() + r.notifyStatus() + + defer func() { + r.clock.release() + r.stateLeadFrom = 0 + r.notifyStatus() + }() + + started := time.Now() + pending := r.states.count() + + select { + case <-ctx.Done(): + return ctx.Err() + + case <-idle: + r.logger.Debugf("held %v for %v state load(s)", time.Since(started).Round(time.Millisecond), pending) + + case <-time.After(r.cfg.StateHoldTimeout): + r.logger.Warnf("%v state load(s) did not finish within %v, continuing", pending, r.cfg.StateHoldTimeout) + } + + return nil +} + +func (r *Replay) isPlaying() bool { + r.mutex.RLock() + defer r.mutex.RUnlock() + + return r.running && r.speed > 0 +} + +func (r *Replay) settle(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(r.cfg.StepSettle): + return nil + } +} + +// waitForVirtual blocks until the virtual clock reaches a point in time. While playing +// that means sleeping the corresponding amount of real time; while stepping the clock +// simply jumps there, and while paused it waits for the replay to be resumed. +func (r *Replay) waitForVirtual(ctx context.Context, target time.Time) error { + for { + if !r.isRunning() { + if err := r.awaitResume(ctx); err != nil { + return err + } + + continue + } + + if !r.isPlaying() { + if r.clock.now().Before(target) { + r.clock.set(target) + } + + return nil + } + + delay := r.clock.realDelayUntil(target) + if delay <= 0 { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + case <-r.wake: + } + } +} + +// emitBlock publishes the events a real node emits for a new block, after making the +// block the head so that anything the explorer requests in response already resolves. +func (r *Replay) emitBlock(ctx context.Context, header *blockHeader) error { + if r.cfg.EmitBids && r.chain.bidsActiveAt(header.Slot) { + bid, err := r.upstream.payloadBid(ctx, header.Root) + if err != nil { + r.logger.WithError(err).Debugf("could not read payload bid of block %v", header.Root) + } else if bid != nil { + r.events.publish(sseEvent{topic: "execution_payload_bid", data: bid}) + } + } + + r.mutex.Lock() + previousEpoch := uint64(0) + if r.head != nil { + previousEpoch = r.chain.epochOf(r.head.Slot) + } + r.head = header + r.mutex.Unlock() + + blockEvent := &v1.BlockEvent{ + Slot: phase0.Slot(header.Slot), + Block: parseRoot(header.Root), + } + r.publishJSON("block", blockEvent) + + headEvent := &v1.HeadEvent{ + Slot: phase0.Slot(header.Slot), + Block: parseRoot(header.Root), + State: parseRoot(header.StateRoot), + EpochTransition: header.Slot%r.chain.slotsPerEpoch == 0, + } + r.publishJSON("head", headEvent) + + // finality only ever moves at an epoch boundary, so it is re-read once per epoch + // rather than for every block + if r.chain.epochOf(header.Slot) != previousEpoch { + if err := r.refreshFinality(ctx, header); err != nil { + r.logger.WithError(err).Warnf("could not refresh finality at slot %v", header.Slot) + } + } + + return nil +} + +// refreshFinality reads the finality the chain knew at a head block and emits a +// finalized_checkpoint event when it moved. +func (r *Replay) refreshFinality(ctx context.Context, header *blockHeader) error { + finality, err := r.readFinality(ctx, header) + if err != nil { + return err + } + + r.mutex.Lock() + previous := r.finality + r.finality = finality + r.mutex.Unlock() + + // the initial read only establishes the baseline, it is not a new checkpoint + if previous == nil || previous.FinalizedEpoch == finality.FinalizedEpoch { + return nil + } + + finalizedHeader, err := r.upstream.headerByRoot(ctx, finality.FinalizedRoot) + if err != nil || finalizedHeader == nil { + r.logger.Debugf("could not resolve finalized header %v", finality.FinalizedRoot) + return nil + } + + event := &v1.FinalizedCheckpointEvent{ + Block: parseRoot(finality.FinalizedRoot), + State: parseRoot(finalizedHeader.StateRoot), + Epoch: phase0.Epoch(finality.FinalizedEpoch), + } + r.publishJSON("finalized_checkpoint", event) + + r.logger.Infof("finalized checkpoint moved to epoch %v", finality.FinalizedEpoch) + + return nil +} + +// readFinality asks the upstream what the chain had finalized at a head block. Reading +// finality forces the node to load that state, and nodes keep only a shallow window of +// them, so a pruned head state falls back to the epoch boundaries below it: finality +// only moves at an epoch boundary, so the epoch's first block answers for the whole +// epoch. +func (r *Replay) readFinality(ctx context.Context, header *blockHeader) (*finalityCheckpoints, error) { + finality, err := r.upstream.finality(ctx, header.StateRoot) + if err == nil { + return finality, nil + } + + if err != errNotFound { + return nil, err + } + + epoch := r.chain.epochOf(header.Slot) + + for attempt := 0; attempt < finalityFallbackEpochs && epoch > 0; attempt++ { + boundary, headerErr := r.upstream.headerBySlot(ctx, epoch*r.chain.slotsPerEpoch) + epoch-- + + if headerErr != nil || boundary == nil { + continue + } + + finality, err = r.upstream.finality(ctx, boundary.StateRoot) + if err == nil { + r.logger.Debugf("read finality from epoch boundary slot %v (head state was pruned)", boundary.Slot) + + return finality, nil + } + + if err != errNotFound { + return nil, err + } + } + + return nil, fmt.Errorf("upstream has no state to read finality from at slot %v; use an archive node or a node that keeps historical states", header.Slot) +} + +// advanceExecution moves the virtual execution head to the last block produced at or +// before this slot. A new execution block whose timestamp is exactly this slot's time +// means the payload for this slot was revealed, which is what the availability event +// reports post-Gloas. +func (r *Replay) advanceExecution(ctx context.Context, slot uint64, boundary time.Time, header *blockHeader) error { + if r.el == nil { + return nil + } + + added, err := r.el.advanceTo(ctx, boundary) + if err != nil { + return fmt.Errorf("error advancing execution head: %w", err) + } + + if header == nil { + return nil + } + + for _, block := range added { + if block.timestamp.Equal(boundary) { + event := &v1.ExecutionPayloadAvailableEvent{ + BlockRoot: parseRoot(header.Root), + Slot: phase0.Slot(slot), + } + r.publishJSON("execution_payload_available", event) + + break + } + } + + return nil +} + +// redactURL strips credentials from an endpoint so they do not end up on the console +// or in the status endpoint. +func redactURL(endpoint string) string { + parsed, err := url.Parse(endpoint) + if err != nil { + return endpoint + } + + return parsed.Redacted() +} + +func (r *Replay) publishJSON(topic string, event any) { + data, err := json.Marshal(event) + if err != nil { + r.logger.WithError(err).Errorf("could not encode %v event", topic) + return + } + + r.events.publish(sseEvent{topic: topic, data: data}) +} diff --git a/replay/replay_test.go b/replay/replay_test.go new file mode 100644 index 00000000..9297f362 --- /dev/null +++ b/replay/replay_test.go @@ -0,0 +1,285 @@ +package replay + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +const ( + testGenesisUnix = 1700000000 + testSlotsPerEpoch = 32 + testSlotSeconds = 12 +) + +// fakeBeacon is a minimal beacon node serving a synthetic chain, used to drive the +// replay end to end without a real upstream. +type fakeBeacon struct { + // emptySlots are slots that carry no block. + emptySlots map[uint64]bool + + // finalizedEpoch is what every finality read reports. + finalizedEpoch uint64 +} + +func (f *fakeBeacon) rootOf(slot uint64) string { + return fmt.Sprintf("0x%064x", slot) +} + +func (f *fakeBeacon) stateRootOf(slot uint64) string { + return fmt.Sprintf("0x%064x", slot+1_000_000) +} + +func (f *fakeBeacon) start() *httptest.Server { + mux := http.NewServeMux() + + mux.HandleFunc("/eth/v1/beacon/genesis", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{"genesis_time": strconv.Itoa(testGenesisUnix)}, + }) + }) + + mux.HandleFunc("/eth/v1/config/spec", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]string{ + "SECONDS_PER_SLOT": strconv.Itoa(testSlotSeconds), + "SLOTS_PER_EPOCH": strconv.Itoa(testSlotsPerEpoch), + }, + }) + }) + + mux.HandleFunc("/eth/v1/beacon/headers/", func(w http.ResponseWriter, r *http.Request) { + id := strings.TrimPrefix(r.URL.Path, "/eth/v1/beacon/headers/") + + slot, err := strconv.ParseUint(id, 10, 64) + if err != nil { + // resolve by root, which the synthetic chain encodes as the slot number + parsed, parseErr := strconv.ParseUint(strings.TrimLeft(strings.TrimPrefix(id, "0x"), "0"), 16, 64) + if parseErr != nil { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + slot = parsed + } + + if f.emptySlots[slot] { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "root": f.rootOf(slot), + "header": map[string]any{ + "message": map[string]any{ + "slot": strconv.FormatUint(slot, 10), + "parent_root": f.rootOf(slot - 1), + "state_root": f.stateRootOf(slot), + }, + }, + }, + }) + }) + + mux.HandleFunc("/eth/v1/beacon/states/", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "current_justified": map[string]any{ + "epoch": strconv.FormatUint(f.finalizedEpoch+1, 10), + "root": f.rootOf(1), + }, + "finalized": map[string]any{ + "epoch": strconv.FormatUint(f.finalizedEpoch, 10), + "root": f.rootOf(2), + }, + }, + }) + }) + + mux.HandleFunc("/eth/v2/beacon/blocks/", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{ + "message": map[string]any{ + "body": map[string]any{ + "signed_execution_payload_bid": map[string]any{"message": map[string]any{"slot": "1"}}, + }, + }, + }, + }) + }) + + return httptest.NewServer(mux) +} + +func newTestReplay(t *testing.T, beacon *fakeBeacon, startSlot uint64) (*Replay, *httptest.Server) { + t.Helper() + + server := beacon.start() + t.Cleanup(server.Close) + + cfg := DefaultConfig() + cfg.UpstreamURL = server.URL + cfg.StartSlot = startSlot + cfg.EmitBids = false + cfg.StepSettle = time.Millisecond + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + replay, err := New(context.Background(), logger, cfg) + require.NoError(t, err) + + return replay, server +} + +// collectEvents subscribes to the hub the way an event stream client would. +func collectEvents(replay *Replay, topics ...string) *eventSubscriber { + filter := make(map[string]bool, len(topics)) + for _, topic := range topics { + filter[topic] = true + } + + sub := newEventSubscriber(filter) + replay.events.add(sub) + + return sub +} + +func TestNewResolvesHeadAtStartSlot(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{100: true, 99: true}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + + require.Equal(t, uint64(100), replay.virtualSlot) + require.NotNil(t, replay.head) + require.Equal(t, uint64(98), replay.head.Slot, "head must be the newest block at or before the start slot") + require.Equal(t, uint64(2), replay.finality.FinalizedEpoch) +} + +func TestStepEmitsBlockAndHead(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + sub := collectEvents(replay, "block", "head") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go replay.runDriver(ctx) + replay.Step(1) + + blockEvent := awaitEvent(t, sub, "block") + require.Equal(t, "101", jsonField(t, blockEvent.data, "slot")) + require.Equal(t, beacon.rootOf(101), jsonField(t, blockEvent.data, "block")) + + headEvent := awaitEvent(t, sub, "head") + require.Equal(t, "101", jsonField(t, headEvent.data, "slot")) + require.Equal(t, beacon.stateRootOf(101), jsonField(t, headEvent.data, "state")) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause after the step") + require.Equal(t, uint64(101), replay.Status().VirtualSlot) + require.Equal(t, uint64(101), replay.Status().HeadSlot) +} + +func TestEmptySlotAdvancesWithoutEvents(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{101: true}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + sub := collectEvents(replay, "block", "head") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go replay.runDriver(ctx) + replay.Step(1) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause after the step") + + require.Equal(t, uint64(101), replay.Status().VirtualSlot) + require.Equal(t, uint64(100), replay.Status().HeadSlot, "an empty slot must not move the head") + require.Empty(t, sub.events, "an empty slot must not emit events") +} + +func TestForwardRunsToTargetSlot(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + go replay.runDriver(ctx) + require.NoError(t, replay.Forward(110, 0)) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause at the target") + require.Equal(t, uint64(110), replay.Status().VirtualSlot) +} + +func TestVirtualClockTracksReplayedSlot(t *testing.T) { + beacon := &fakeBeacon{emptySlots: map[uint64]bool{}, finalizedEpoch: 2} + + replay, _ := newTestReplay(t, beacon, 100) + + // before stepping, the clock sits inside the start slot + require.Equal(t, uint64(100), slotOfVirtualTime(replay)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go replay.runDriver(ctx) + replay.Step(3) + + requireEventually(t, func() bool { return !replay.isRunning() }, "replay should pause after the step") + require.Equal(t, uint64(103), slotOfVirtualTime(replay)) +} + +func slotOfVirtualTime(replay *Replay) uint64 { + elapsed := replay.clock.now().Sub(replay.chain.genesisTime) + + return uint64(elapsed / replay.chain.slotDuration) +} + +func awaitEvent(t *testing.T, sub *eventSubscriber, topic string) sseEvent { + t.Helper() + + deadline := time.After(10 * time.Second) + + for { + select { + case event := <-sub.events: + if event.topic == topic { + return event + } + case <-deadline: + t.Fatalf("timed out waiting for a %v event", topic) + } + } +} + +func jsonField(t *testing.T, data []byte, field string) string { + t.Helper() + + parsed := map[string]any{} + require.NoError(t, json.Unmarshal(data, &parsed)) + + value, ok := parsed[field].(string) + require.Truef(t, ok, "field %v is missing or not a string in %s", field, data) + + return value +} + +func requireEventually(t *testing.T, condition func() bool, message string) { + t.Helper() + + require.Eventually(t, condition, 10*time.Second, 5*time.Millisecond, message) +} diff --git a/replay/stateloads.go b/replay/stateloads.go new file mode 100644 index 00000000..49070061 --- /dev/null +++ b/replay/stateloads.go @@ -0,0 +1,71 @@ +package replay + +import "sync" + +// stateLoads counts the beacon states the explorer is currently pulling through the +// proxy. A full state is tens of megabytes and takes seconds to fetch, decompress and +// decode, during which the explorer cannot process anything else — so the replay uses +// this to hold its clock rather than running ahead of what the explorer has seen. +type stateLoads struct { + mutex sync.Mutex + active int + total uint64 + + // idle is closed while nothing is loading, and replaced with an open channel as + // soon as a load starts, so a waiter can select on it. + idle chan struct{} +} + +func newStateLoads() *stateLoads { + loads := &stateLoads{idle: make(chan struct{})} + close(loads.idle) + + return loads +} + +// begin records the start of a state load and returns the function that ends it. +func (s *stateLoads) begin() func() { + s.mutex.Lock() + + if s.active == 0 { + s.idle = make(chan struct{}) + } + + s.active++ + s.total++ + s.mutex.Unlock() + + ended := false + + return func() { + if ended { + return + } + + ended = true + + s.mutex.Lock() + defer s.mutex.Unlock() + + s.active-- + if s.active == 0 { + close(s.idle) + } + } +} + +// idleChan returns a channel that is closed once no state is being loaded. It is +// already closed when nothing is loading right now. +func (s *stateLoads) idleChan() <-chan struct{} { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.idle +} + +func (s *stateLoads) count() int { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.active +} diff --git a/replay/stateloads_test.go b/replay/stateloads_test.go new file mode 100644 index 00000000..565fb569 --- /dev/null +++ b/replay/stateloads_test.go @@ -0,0 +1,279 @@ +package replay + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestStateLoadsTracksActiveReads(t *testing.T) { + loads := newStateLoads() + + requireClosed(t, loads.idleChan(), "nothing is loading yet") + require.Zero(t, loads.count()) + + endFirst := loads.begin() + endSecond := loads.begin() + + require.Equal(t, 2, loads.count()) + requireOpen(t, loads.idleChan(), "two reads are in flight") + + endFirst() + require.Equal(t, 1, loads.count()) + requireOpen(t, loads.idleChan(), "one read is still in flight") + + endSecond() + require.Zero(t, loads.count()) + requireClosed(t, loads.idleChan(), "the last read finished") + + // ending twice must not double-count + endSecond() + require.Zero(t, loads.count()) +} + +func TestStateLoadsBecomeBusyAgain(t *testing.T) { + loads := newStateLoads() + + end := loads.begin() + end() + + requireClosed(t, loads.idleChan(), "idle after the first read") + + loads.begin() + requireOpen(t, loads.idleChan(), "a later read must make it busy again") +} + +// TestClockDoesNotMoveWhileHeld is the whole point of the state gate: waiting for the +// explorer must cost real time but no virtual time, or the replay runs ahead of what +// the explorer has actually seen. +func TestClockDoesNotMoveWhileHeld(t *testing.T) { + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newClock(anchor) + clock.setRate(100) + + time.Sleep(20 * time.Millisecond) + + clock.hold() + frozen := clock.now() + + _, rate := clock.state() + require.Zero(t, rate, "the explorer must see a stopped clock while the replay is holding") + + time.Sleep(30 * time.Millisecond) + require.Equal(t, frozen, clock.now(), "no virtual time may pass while held") + + clock.release() + + _, rate = clock.state() + require.Equal(t, 100.0, rate, "releasing must restore the rate it was running at") + + time.Sleep(20 * time.Millisecond) + require.True(t, clock.now().After(frozen), "the clock must resume after the hold") +} + +func TestClockHoldsNest(t *testing.T) { + clock := newClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + clock.setRate(10) + + clock.hold() + clock.hold() + clock.release() + + require.True(t, clock.isHeld(), "the clock stays held until every hold is released") + + clock.release() + require.False(t, clock.isHeld()) + + // releasing more often than holding must not unbalance the counter + clock.release() + require.False(t, clock.isHeld()) +} + +func TestAwaitStateLoadsHoldsUntilTheReadFinishes(t *testing.T) { + replay := testConsoleReplay(100) + replay.clock.setRate(50) + + end := replay.states.begin() + + released := make(chan struct{}) + + // the gate only holds once the replay is more than stateLoadLeadSlots ahead of the + // slot at which the read started + replay.stateLeadFrom = 100 + + go func() { + defer close(released) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100+stateLoadLeadSlots)) + }() + + require.Eventually(t, func() bool { return replay.clock.isHeld() }, 5*time.Second, 5*time.Millisecond, + "the gate must freeze the clock while a state is loading") + + frozen := replay.clock.now() + + // the gate must keep holding, not return + time.Sleep(50 * time.Millisecond) + + select { + case <-released: + t.Fatal("the gate returned while a state was still loading") + default: + } + + require.True(t, replay.clock.isHeld()) + require.True(t, replay.Status().Holding) + require.Equal(t, 1, replay.Status().StateLoads) + require.Equal(t, frozen, replay.clock.now(), "the 50ms wait must have cost no virtual time") + + end() + + select { + case <-released: + case <-time.After(5 * time.Second): + t.Fatal("the gate did not release when the state load finished") + } + + require.False(t, replay.clock.isHeld()) + require.False(t, replay.Status().Holding) +} + +func TestAwaitStateLoadsReturnsImmediatelyWhenIdle(t *testing.T) { + replay := testConsoleReplay(100) + + done := make(chan struct{}) + + go func() { + defer close(done) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100)) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the gate must not wait when no state is loading") + } + + require.False(t, replay.clock.isHeld()) +} + +func TestAwaitStateLoadsGivesUpAfterTheTimeout(t *testing.T) { + replay := testConsoleReplay(100) + replay.cfg.StateHoldTimeout = 50 * time.Millisecond + + replay.states.begin() // never finishes + replay.stateLeadFrom = 100 + + done := make(chan struct{}) + + go func() { + defer close(done) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100+stateLoadLeadSlots)) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("a stuck state load must not wedge the replay for good") + } + + require.False(t, replay.clock.isHeld(), "the hold must be lifted when the gate gives up") +} + +func TestAwaitStateLoadsStopsOnShutdown(t *testing.T) { + replay := testConsoleReplay(100) + replay.states.begin() // never finishes + replay.stateLeadFrom = 100 + + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + + go func() { done <- replay.awaitStateLoads(ctx, 100+stateLoadLeadSlots) }() + + cancel() + + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("the gate must give up when the replay shuts down") + } +} + +func requireClosed(t *testing.T, channel <-chan struct{}, message string) { + t.Helper() + + select { + case <-channel: + default: + t.Fatalf("expected an idle (closed) channel: %v", message) + } +} + +func requireOpen(t *testing.T, channel <-chan struct{}, message string) { + t.Helper() + + select { + case <-channel: + t.Fatalf("expected a busy (open) channel: %v", message) + default: + } +} + +// TestStateLoadGateLetsTheReplayRunOnBriefly is the behaviour that keeps blocks flowing +// during a multi-second state read: the explorer indexes blocks on a different goroutine +// than it loads states on, so the replay serves a few more slots before it holds. +func TestStateLoadGateLetsTheReplayRunOnBriefly(t *testing.T) { + replay := testConsoleReplay(100) + replay.clock.setRate(10) + + replay.states.begin() // never finishes + + for slot := uint64(100); slot < 100+stateLoadLeadSlots; slot++ { + done := make(chan struct{}) + + go func() { + defer close(done) + require.NoError(t, replay.awaitStateLoads(context.Background(), slot)) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("the gate must not hold at slot %v, only %v slots into the read", slot, slot-100) + } + + require.False(t, replay.clock.isHeld(), "the clock must keep running within the lead window") + } + + // beyond the lead window it holds + held := make(chan struct{}) + + go func() { + defer close(held) + require.NoError(t, replay.awaitStateLoads(context.Background(), 100+stateLoadLeadSlots)) + }() + + require.Eventually(t, func() bool { return replay.clock.isHeld() }, 5*time.Second, 5*time.Millisecond, + "the gate must hold once the replay would get further ahead than the lead window") +} + +// TestStateLoadGateResetsBetweenReads guards that the lead window is per read, not +// cumulative: a finished read must not leave the next one starting mid-window. +func TestStateLoadGateResetsBetweenReads(t *testing.T) { + replay := testConsoleReplay(100) + + end := replay.states.begin() + require.NoError(t, replay.awaitStateLoads(context.Background(), 100)) + require.Equal(t, uint64(100), replay.stateLeadFrom) + + end() + require.NoError(t, replay.awaitStateLoads(context.Background(), 101)) + require.Zero(t, replay.stateLeadFrom, "an idle tracker must clear the lead window") + + replay.states.begin() + require.NoError(t, replay.awaitStateLoads(context.Background(), 200)) + require.Equal(t, uint64(200), replay.stateLeadFrom, "the next read starts its own window") +} diff --git a/replay/tracoor.go b/replay/tracoor.go new file mode 100644 index 00000000..e46c79f8 --- /dev/null +++ b/replay/tracoor.go @@ -0,0 +1,169 @@ +package replay + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/klauspost/compress/zstd" +) + +// zstdMagic identifies a zstd frame. Tracoor stores its artifacts compressed and the +// object store hands them back with Content-Encoding: zstd, which the Go HTTP client +// does not unwrap on its own. +var zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} + +// tracoorArtifact is a stored artifact and the slot it belongs to. The slot is what +// lets the replay label the artifact with the right fork. +type tracoorArtifact struct { + body []byte + slot uint64 +} + +// tracoorClient resolves artifacts from a tracoor instance. Tracoor indexes by root +// rather than by slot, so it can only answer for blocks and states whose root is +// already known from a header. +type tracoorClient struct { + base string + network string + client *http.Client + decoder *zstd.Decoder +} + +func newTracoorClient(baseURL, network string) (*tracoorClient, error) { + decoder, err := zstd.NewReader(nil) + if err != nil { + return nil, fmt.Errorf("could not create zstd decoder: %w", err) + } + + return &tracoorClient{ + base: strings.TrimSuffix(baseURL, "/"), + network: network, + client: newPooledClient(5 * time.Minute), + decoder: decoder, + }, nil +} + +func (t *tracoorClient) fetchBeaconBlock(ctx context.Context, blockRoot string) (*tracoorArtifact, error) { + return t.fetch(ctx, "list-beacon-block", "block_root", blockRoot, "beacon_blocks", "beacon_block") +} + +func (t *tracoorClient) fetchBeaconState(ctx context.Context, stateRoot string) (*tracoorArtifact, error) { + return t.fetch(ctx, "list-beacon-state", "state_root", stateRoot, "beacon_states", "beacon_state") +} + +func (t *tracoorClient) fetch(ctx context.Context, endpoint, rootField, root, listField, artifactType string) (*tracoorArtifact, error) { + id, slot, err := t.lookup(ctx, endpoint, rootField, root, listField) + if err != nil { + return nil, err + } + + body, err := t.download(ctx, artifactType, id) + if err != nil { + return nil, err + } + + return &tracoorArtifact{body: body, slot: slot}, nil +} + +// lookup asks tracoor which stored artifacts match a root and returns the newest one. +func (t *tracoorClient) lookup(ctx context.Context, endpoint, rootField, root, listField string) (string, uint64, error) { + request := map[string]any{ + "network": t.network, + rootField: root, + "pagination": map[string]any{ + "limit": 1, + "offset": 0, + "order_by": "fetched_at DESC", + }, + } + + body, err := json.Marshal(request) + if err != nil { + return "", 0, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.base+"/v1/api/"+endpoint, bytes.NewReader(body)) + if err != nil { + return "", 0, err + } + + req.Header.Set("Content-Type", "application/json") + + rsp, err := t.client.Do(req) + if err != nil { + return "", 0, err + } + defer func() { _ = rsp.Body.Close() }() + + if rsp.StatusCode != http.StatusOK { + data, _ := io.ReadAll(rsp.Body) + return "", 0, fmt.Errorf("tracoor %v returned %v: %s", endpoint, rsp.StatusCode, truncate(data, 200)) + } + + // the response shape differs per endpoint only in the name of the list field + parsed := map[string][]struct { + ID string `json:"id"` + Slot string `json:"slot"` + }{} + + if err := json.NewDecoder(rsp.Body).Decode(&parsed); err != nil { + return "", 0, fmt.Errorf("error parsing tracoor response: %w", err) + } + + items := parsed[listField] + if len(items) == 0 || items[0].ID == "" { + return "", 0, errNotFound + } + + slot, err := strconv.ParseUint(items[0].Slot, 10, 64) + if err != nil { + return "", 0, fmt.Errorf("invalid slot %q in tracoor response: %w", items[0].Slot, err) + } + + return items[0].ID, slot, nil +} + +func (t *tracoorClient) download(ctx context.Context, artifactType, id string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, t.base+"/download/"+artifactType+"/"+id, http.NoBody) + if err != nil { + return nil, err + } + + rsp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = rsp.Body.Close() }() + + if rsp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("tracoor download of %v/%v returned %v", artifactType, id, rsp.StatusCode) + } + + body, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + + return t.decompress(body) +} + +// decompress unwraps a zstd frame, leaving anything else untouched. +func (t *tracoorClient) decompress(body []byte) ([]byte, error) { + if !bytes.HasPrefix(body, zstdMagic) { + return body, nil + } + + decoded, err := t.decoder.DecodeAll(body, nil) + if err != nil { + return nil, fmt.Errorf("error decompressing tracoor artifact: %w", err) + } + + return decoded, nil +} diff --git a/replay/upstream.go b/replay/upstream.go new file mode 100644 index 00000000..e09e3ad1 --- /dev/null +++ b/replay/upstream.go @@ -0,0 +1,499 @@ +package replay + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// errNotFound is returned when the upstream has no artifact for a request. It maps to +// a 404 on the proxy, which is a normal answer for empty slots. +var errNotFound = fmt.Errorf("not found") + +// upstreamError carries a non-2xx upstream answer so the proxy can hand the same status +// and body back to its own client. Clients read these codes: a 400 for "this block has +// no payload envelope" means something quite different from a gateway failure. +type upstreamError struct { + status int + body []byte + header http.Header +} + +func (e *upstreamError) Error() string { + return fmt.Sprintf("upstream returned %v: %s", e.status, truncate(e.body, 200)) +} + +// upstream is the read side of the replay: a beacon node that answers by slot or root, +// with an optional tracoor fallback for artifacts the node has already pruned. +type upstream struct { + logger logrus.FieldLogger + base *url.URL + client *http.Client + tracoor *tracoorClient + cache *artifactCache + + // chain is set once the timing specs are known; it labels tracoor artifacts with + // the fork they belong to. + chain *chainInfo +} + +// artifact is a raw upstream response, kept encoded so it can be forwarded verbatim. +// The headers matter as much as the body: SSZ responses carry their fork in +// Eth-Consensus-Version, without which the explorer cannot decode them. +type artifact struct { + body []byte + header http.Header +} + +// newPooledClient returns an HTTP client that keeps connections alive across the many +// small requests a replay makes. The default transport keeps only two idle connections +// per host, which for a TLS upstream means a fresh handshake on nearly every request — +// on a remote devnet that alone dominates the time it takes to step a slot. +func newPooledClient(timeout time.Duration) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConns = 128 + transport.MaxIdleConnsPerHost = 64 + transport.MaxConnsPerHost = 64 + transport.IdleConnTimeout = 10 * time.Minute + + return &http.Client{Transport: transport, Timeout: timeout} +} + +func newUpstream(logger logrus.FieldLogger, cfg *Config) (*upstream, error) { + base, err := url.Parse(strings.TrimSuffix(cfg.UpstreamURL, "/")) + if err != nil { + return nil, fmt.Errorf("invalid upstream url: %w", err) + } + + up := &upstream{ + logger: logger, + base: base, + client: newPooledClient(10 * time.Minute), + } + + if cfg.TracoorURL != "" { + tracoor, err := newTracoorClient(cfg.TracoorURL, cfg.TracoorNetwork) + if err != nil { + return nil, err + } + + up.tracoor = tracoor + } + + if cfg.CacheDir != "" { + cache, err := newArtifactCache(logger, cfg.CacheDir) + if err != nil { + return nil, err + } + + up.cache = cache + } + + return up, nil +} + +// get fetches a path from the beacon upstream, serving it from the artifact cache when +// the path is immutable and already recorded. +func (u *upstream) get(ctx context.Context, path string, accept string) (*artifact, error) { + cacheKey := "" + if u.cache != nil && isImmutablePath(path) { + cacheKey = u.cache.key(path, accept) + if art := u.cache.load(cacheKey); art != nil { + return art, nil + } + } + + art, err := u.fetch(ctx, path, accept) + if err != nil { + return nil, err + } + + if cacheKey != "" { + u.cache.store(cacheKey, art) + } + + return art, nil +} + +func (u *upstream) fetch(ctx context.Context, path string, accept string) (*artifact, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.base.String()+path, http.NoBody) + if err != nil { + return nil, err + } + + if accept != "" { + req.Header.Set("Accept", accept) + } + + rsp, err := u.client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = rsp.Body.Close() }() + + body, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + + if rsp.StatusCode == http.StatusNotFound { + return nil, errNotFound + } + + if rsp.StatusCode != http.StatusOK { + return nil, &upstreamError{ + status: rsp.StatusCode, + body: body, + header: forwardableHeaders(rsp.Header), + } + } + + return &artifact{body: body, header: forwardableHeaders(rsp.Header)}, nil +} + +// hopByHopHeaders are the response headers that describe this particular connection +// rather than the artifact, so they must not be copied to the proxied response. +var hopByHopHeaders = map[string]bool{ + "Connection": true, + "Content-Encoding": true, + "Content-Length": true, + "Keep-Alive": true, + "Proxy-Authenticate": true, + "Proxy-Connection": true, + "Te": true, + "Trailer": true, + "Transfer-Encoding": true, + "Upgrade": true, + "Date": true, + "Server": true, +} + +func forwardableHeaders(src http.Header) http.Header { + dst := make(http.Header, len(src)) + + for key, values := range src { + if hopByHopHeaders[http.CanonicalHeaderKey(key)] { + continue + } + + dst[http.CanonicalHeaderKey(key)] = values + } + + return dst +} + +func (u *upstream) getJSON(ctx context.Context, path string, out any) error { + art, err := u.get(ctx, path, "application/json") + if err != nil { + return err + } + + if err := json.Unmarshal(art.body, out); err != nil { + return fmt.Errorf("error parsing %v: %w", path, err) + } + + return nil +} + +// blockHeader is the part of a beacon block header the replay needs to track the head +// and to rewrite `head` aliases into concrete roots. +type blockHeader struct { + Slot uint64 + Root string + StateRoot string + ParentRoot string + BlockNumber uint64 +} + +type headerResponse struct { + Data struct { + Root string `json:"root"` + Header struct { + Message struct { + Slot string `json:"slot"` + ParentRoot string `json:"parent_root"` + StateRoot string `json:"state_root"` + } `json:"message"` + } `json:"header"` + } `json:"data"` +} + +func (r *headerResponse) toHeader() (*blockHeader, error) { + slot, err := strconv.ParseUint(r.Data.Header.Message.Slot, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid slot in header response: %w", err) + } + + return &blockHeader{ + Slot: slot, + Root: r.Data.Root, + StateRoot: r.Data.Header.Message.StateRoot, + ParentRoot: r.Data.Header.Message.ParentRoot, + }, nil +} + +// headerBySlot returns the block header at a slot, or nil when the slot is empty. +func (u *upstream) headerBySlot(ctx context.Context, slot uint64) (*blockHeader, error) { + return u.header(ctx, strconv.FormatUint(slot, 10)) +} + +// headerByRoot returns the block header for a block root, or nil when it is unknown. +func (u *upstream) headerByRoot(ctx context.Context, root string) (*blockHeader, error) { + return u.header(ctx, root) +} + +func (u *upstream) header(ctx context.Context, blockID string) (*blockHeader, error) { + rsp := headerResponse{} + + err := u.getJSON(ctx, "/eth/v1/beacon/headers/"+blockID, &rsp) + if err != nil { + if err == errNotFound { + return nil, nil + } + + return nil, err + } + + return rsp.toHeader() +} + +// finalityCheckpoints is the finality the chain knew at a given block root. Raw keeps +// the upstream response so the replay can serve it back verbatim, including the +// previous justified checkpoint it does not track itself. +type finalityCheckpoints struct { + JustifiedEpoch uint64 + JustifiedRoot string + FinalizedEpoch uint64 + FinalizedRoot string + Raw json.RawMessage +} + +func (u *upstream) finality(ctx context.Context, stateID string) (*finalityCheckpoints, error) { + rsp := struct { + Data struct { + CurrentJustified struct { + Epoch string `json:"epoch"` + Root string `json:"root"` + } `json:"current_justified"` + Finalized struct { + Epoch string `json:"epoch"` + Root string `json:"root"` + } `json:"finalized"` + } `json:"data"` + }{} + + art, err := u.get(ctx, "/eth/v1/beacon/states/"+stateID+"/finality_checkpoints", "application/json") + if err != nil { + return nil, err + } + + raw := json.RawMessage(art.body) + + if err := json.Unmarshal(raw, &rsp); err != nil { + return nil, fmt.Errorf("error parsing finality checkpoints: %w", err) + } + + justifiedEpoch, err := strconv.ParseUint(rsp.Data.CurrentJustified.Epoch, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid justified epoch: %w", err) + } + + finalizedEpoch, err := strconv.ParseUint(rsp.Data.Finalized.Epoch, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid finalized epoch: %w", err) + } + + return &finalityCheckpoints{ + JustifiedEpoch: justifiedEpoch, + JustifiedRoot: rsp.Data.CurrentJustified.Root, + FinalizedEpoch: finalizedEpoch, + FinalizedRoot: rsp.Data.Finalized.Root, + Raw: raw, + }, nil +} + +// payloadBid returns the winning execution payload bid carried by a Gloas block, as +// raw JSON ready to be replayed on the event stream. It returns nil for blocks from +// forks that have no bid. +func (u *upstream) payloadBid(ctx context.Context, root string) (json.RawMessage, error) { + rsp := struct { + Data struct { + Message struct { + Body struct { + SignedExecutionPayloadBid json.RawMessage `json:"signed_execution_payload_bid"` + } `json:"body"` + } `json:"message"` + } `json:"data"` + }{} + + if err := u.getJSON(ctx, "/eth/v2/beacon/blocks/"+root, &rsp); err != nil { + return nil, err + } + + bid := rsp.Data.Message.Body.SignedExecutionPayloadBid + if len(bid) == 0 || string(bid) == "null" { + return nil, nil + } + + return bid, nil +} + +func isStatePath(path string) bool { + return strings.Contains(path, "/debug/beacon/states/") +} + +// serve answers a proxied GET for a client of the fake node. Beacon states are read +// from tracoor first when it is configured: it keeps every state, while beacon nodes +// prune all but the most recent ones, and serving a 17 MB state per epoch out of the +// archive would hammer the devnet. +func (u *upstream) serve(w http.ResponseWriter, r *http.Request, path string) { + accept := r.Header.Get("Accept") + + // states are read from tracoor first, blocks are not: the node still has every + // block of the replayed range and answers in one round trip, while a tracoor read + // costs a lookup plus a download + if isStatePath(path) { + if art := u.tryTracoor(r.Context(), path, accept); art != nil { + writeArtifact(w, art, http.StatusOK) + return + } + } + + art, err := u.get(r.Context(), path, accept) + + switch { + case err == errNotFound: + if art = u.tryTracoor(r.Context(), path, "application/octet-stream"); art == nil { + writeAPIError(w, http.StatusNotFound, "not found") + return + } + + case err != nil: + var upErr *upstreamError + if errors.As(err, &upErr) { + // hand the upstream's own answer back unchanged; the client knows what a + // 400 or a 503 from a beacon node means + u.logger.Debugf("upstream %v answered %v", path, upErr.status) + writeArtifact(w, &artifact{body: upErr.body, header: upErr.header}, upErr.status) + + return + } + + u.logger.WithError(err).Warnf("upstream request failed: %v", path) + writeAPIError(w, http.StatusBadGateway, err.Error()) + + return + } + + writeArtifact(w, art, http.StatusOK) +} + +// tryTracoor resolves a root-addressed block or state from tracoor, returning nil when +// tracoor is not configured, cannot serve this path, or the client cannot take SSZ. +func (u *upstream) tryTracoor(ctx context.Context, path, accept string) *artifact { + if u.tracoor == nil || !acceptsSSZ(accept) { + return nil + } + + segments := strings.Split(strings.Trim(path, "/"), "/") + root := segments[len(segments)-1] + + if !strings.HasPrefix(root, "0x") { + return nil + } + + cacheKey := "" + if u.cache != nil { + cacheKey = u.cache.key(path, "tracoor") + if art := u.cache.load(cacheKey); art != nil { + return art + } + } + + var ( + fetched *tracoorArtifact + err error + ) + + switch { + case strings.Contains(path, "/debug/beacon/states/"): + fetched, err = u.tracoor.fetchBeaconState(ctx, root) + case strings.Contains(path, "/beacon/blocks/"): + fetched, err = u.tracoor.fetchBeaconBlock(ctx, root) + default: + return nil + } + + if err != nil { + if err != errNotFound { + u.logger.WithError(err).Debugf("tracoor lookup failed for %v", path) + } + + return nil + } + + art := &artifact{ + body: fetched.body, + header: http.Header{ + "Content-Type": []string{"application/octet-stream"}, + "Eth-Consensus-Version": []string{u.forkAt(fetched.slot)}, + }, + } + + if cacheKey != "" { + u.cache.store(cacheKey, art) + } + + u.logger.Debugf("served %v from tracoor (slot %v, %v bytes)", path, fetched.slot, len(art.body)) + + return art +} + +func (u *upstream) forkAt(slot uint64) string { + if u.chain == nil { + return "" + } + + return u.chain.forkAt(slot) +} + +// acceptsSSZ reports whether a client will take an SSZ-encoded response. Tracoor only +// stores SSZ, so a JSON-only client has to be served from the beacon node. +func acceptsSSZ(accept string) bool { + return accept == "" || strings.Contains(accept, "application/octet-stream") || strings.Contains(accept, "*/*") +} + +func writeArtifact(w http.ResponseWriter, art *artifact, status int) { + for key, values := range art.header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + if w.Header().Get("Content-Type") == "" { + w.Header().Set("Content-Type", "application/octet-stream") + } + + w.WriteHeader(status) + + if _, err := w.Write(art.body); err != nil { + return + } +} + +func truncate(data []byte, max int) string { + if len(data) <= max { + return string(data) + } + + return string(data[:max]) + "..." +} diff --git a/static/js/page-index.js b/static/js/page-index.js index 31ed78da..76f18175 100644 --- a/static/js/page-index.js +++ b/static/js/page-index.js @@ -6,7 +6,7 @@ initCountdownTooltips(); }); - var refreshInterval = 15000; + var defaultRefreshInterval = 15000; var lastRefresh = new Date().getTime(); var loopTimer = null; var isRefreshing = false; @@ -97,10 +97,22 @@ }, }; + // refreshInterval is read on every loop rather than captured once, so an external + // driver (dora-replay, which advances slots far faster than real time) can retune + // it at runtime by setting window.doraIndexRefreshInterval. + function refreshInterval() { + var override = window.doraIndexRefreshInterval; + if (typeof override === "number" && isFinite(override) && override > 0) { + return override; + } + + return defaultRefreshInterval; + } + function scheduleLoop() { if(loopTimer) return; - var refreshTimeout = refreshInterval - ((new Date().getTime() - lastRefresh)); + var refreshTimeout = refreshInterval() - ((new Date().getTime() - lastRefresh)); if(refreshTimeout < 0) refreshTimeout = 0; else if(refreshTimeout > 1000) @@ -110,14 +122,13 @@ function refreshLoop() { loopTimer = null; - var refreshTimeout = refreshInterval - ((new Date().getTime() - lastRefresh)); + var refreshTimeout = refreshInterval() - ((new Date().getTime() - lastRefresh)); if(refreshTimeout < 0) refreshTimeout = 0; document.getElementById("update_timer").innerText = "Next update in " + Math.ceil(refreshTimeout / 1000) + "s"; if(refreshTimeout <= 0) { lastRefresh = new Date().getTime(); - refreshTimeout = refreshInterval; refresh(); } } diff --git a/templates/_layout/layout.html b/templates/_layout/layout.html index 24b47fed..091fd2fc 100644 --- a/templates/_layout/layout.html +++ b/templates/_layout/layout.html @@ -60,6 +60,10 @@ + {{ if .ReplayControlUrl }} + + + {{ end }} {{ template "js" .Data }} diff --git a/types/config.go b/types/config.go index 1c88da07..e2e39953 100644 --- a/types/config.go +++ b/types/config.go @@ -214,6 +214,16 @@ type Config struct { AllowedMethods []string `yaml:"allowedMethods" envconfig:"RPC_PROXY_ALLOWED_METHODS"` } `yaml:"rpcProxy"` + // Replay drives the explorer off the virtual clock of a dora-replay control + // server instead of the real wall clock, so a past slot range can be stepped + // through as if it were happening live. Only meaningful when the beacon and + // execution endpoints point at the matching dora-replay proxies. + Replay struct { + Enabled bool `yaml:"enabled" envconfig:"REPLAY_ENABLED"` + ControlUrl string `yaml:"controlUrl" envconfig:"REPLAY_CONTROL_URL"` + PollInterval time.Duration `yaml:"pollInterval" envconfig:"REPLAY_POLL_INTERVAL"` + } `yaml:"replay"` + // EnsResolver optionally resolves execution addresses to their primary ENS name. // Names are resolved on the local network (the chain this explorer indexes, via // the main execution pool) and on every configured remote network (each with its diff --git a/types/models.go b/types/models.go index 66247143..001b2823 100644 --- a/types/models.go +++ b/types/models.go @@ -34,6 +34,12 @@ type PageData struct { ApiEnabled bool ExecutionIndexerEnabled bool EnsSearchEnabled bool + + // ReplayControlUrl is the dora-replay control server this explorer is driven by. + // It is only set in replay mode, and it is the whole of the explorer's knowledge + // about the replay UI: the layout side-loads that UI from the replay process and + // hands it this address to talk to. + ReplayControlUrl string } type MainMenuItem struct {