diff --git a/op-node/metrics/metrics.go b/op-node/metrics/metrics.go index e718656d7a9..905a8bb4f4f 100644 --- a/op-node/metrics/metrics.go +++ b/op-node/metrics/metrics.go @@ -35,6 +35,8 @@ type Metricer interface { SetSequencerState(active bool) RecordPipelineReset() RecordFollowSourceRequest(result string) + RecordFollowSourceReorg(action string) + RecordSuperAuthorityReorgSignal(reason string) RecordSequencingError() RecordPublishingError() RecordDerivationError() @@ -84,8 +86,10 @@ type Metrics struct { L1SourceCache *metrics.CacheMetrics L2SourceCache *metrics.CacheMetrics - L2FollowSourceCache *metrics.CacheMetrics - FollowSourceRequests *prometheus.CounterVec + L2FollowSourceCache *metrics.CacheMetrics + FollowSourceRequests *prometheus.CounterVec + FollowSourceReorgs *prometheus.CounterVec + SuperAuthorityReorgSignals *prometheus.CounterVec DerivationIdle prometheus.Gauge @@ -198,6 +202,8 @@ func NewMetrics(procName string, labels prometheus.Labels) *Metrics { Name: "follow_source_requests_total", Help: "Count of follow source requests by result", }, []string{"result"}), + FollowSourceReorgs: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "follow_source_reorgs_total", Help: "Count of follow source reorg decisions by action"}, []string{"action"}), + SuperAuthorityReorgSignals: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "super_authority_reorg_signals_total", Help: "Count of super authority reorg signals by reason"}, []string{"reason"}), DerivationIdle: factory.NewGauge(prometheus.GaugeOpts{ Namespace: ns, @@ -515,6 +521,14 @@ func (m *Metrics) RecordFollowSourceRequest(result string) { m.FollowSourceRequests.WithLabelValues(result).Inc() } +func (m *Metrics) RecordFollowSourceReorg(action string) { + m.FollowSourceReorgs.WithLabelValues(action).Inc() +} + +func (m *Metrics) RecordSuperAuthorityReorgSignal(reason string) { + m.SuperAuthorityReorgSignals.WithLabelValues(reason).Inc() +} + func (m *Metrics) RecordSequencerReset() { m.SequencerResets.Record() } @@ -670,6 +684,12 @@ func (n *noopMetricer) RecordPipelineReset() { func (n *noopMetricer) RecordFollowSourceRequest(result string) { } +func (n *noopMetricer) RecordFollowSourceReorg(action string) { +} + +func (n *noopMetricer) RecordSuperAuthorityReorgSignal(reason string) { +} + func (n *noopMetricer) RecordSequencingError() { } diff --git a/op-node/rollup/engine/engine_controller.go b/op-node/rollup/engine/engine_controller.go index ea09d49eaf9..14c2dcef23b 100644 --- a/op-node/rollup/engine/engine_controller.go +++ b/op-node/rollup/engine/engine_controller.go @@ -251,6 +251,7 @@ func (e *EngineController) resolveVerifiedAsSafe(block eth.BlockID) eth.L2BlockR } br, err := e.engine.L2BlockRefByHash(e.ctx, block.Hash) if err != nil { + e.metrics.RecordSuperAuthorityReorgSignal("unknown_to_engine") e.log.Warn("super authority safe head unknown to engine (reorg signal)", "super_authority_safe", block, "err", err) return e.crossSafeFallback("el-unknown") @@ -260,6 +261,7 @@ func (e *EngineController) resolveVerifiedAsSafe(block eth.BlockID) eth.L2BlockR "super_authority_safe", br, "err", err) return e.crossSafeFallback("canonicality-lookup-failed") } else if !canonical { + e.metrics.RecordSuperAuthorityReorgSignal("non_canonical") e.log.Warn("super authority safe head non-canonical (reorg signal)", "super_authority_safe", br, "canonical", canonicalRef) return e.crossSafeFallback("non-canonical") @@ -1602,6 +1604,12 @@ func (e *EngineController) FollowSource(eSafeBlockRef, eLocalSafeRef, eFinalized // External local safe is found locally but differs: the follower diverged from upstream // and must reorg onto it. + e.log.Warn("Follow Source: local safe diverged from upstream", + "external_local_safe", eLocalSafeRef, + "local_safe", e.localSafeHead, + "local_unsafe", e.unsafeHead, + "external_safe", eSafeBlockRef, + "finalized", eFinalizedRef) if e.originSelectorResetter != nil { // This follower is also a sequencer (the origin-selector resetter is wired only when // sequencing is enabled). A soft unsafe-head update would be clobbered by the next @@ -1610,12 +1618,24 @@ func (e *EngineController) FollowSource(eSafeBlockRef, eLocalSafeRef, eFinalized // cancels the in-flight build and FCUs onto the upstream chain in one shot (head==safe); // the sequencer then rebuilds from there. logger.Warn("Follow Source: Reorg onto upstream chain") + e.metrics.RecordFollowSourceReorg("force_reset") e.forceReset(e.ctx, eLocalSafeRef, eLocalSafeRef, eLocalSafeRef, eSafeBlockRef, eFinalizedRef, false) + e.log.Info("Follow Source: reorg onto upstream chain completed", + "action", "force_reset", + "local_safe", e.localSafeHead, + "local_unsafe", e.unsafeHead, + "cross_safe", e.SafeL2Head()) return } // Pure verifier (no local block production): a soft update suffices and may trigger or // retarget EL sync. + e.metrics.RecordFollowSourceReorg("soft_update") logger.Warn("Follow Source: Reorg. May Trigger EL sync") followExternalRefs(true) + e.log.Info("Follow Source: upstream refs applied", + "action", "soft_update", + "local_safe", e.localSafeHead, + "local_unsafe", e.unsafeHead, + "cross_safe", e.SafeL2Head()) }