feat(observability): enable metrics via config + operator monitoring setup

crates/execution/evm/src/reth_env/config.rs

@@ -29,6 +29,8 @@ use tracing::warn;
 #[derive(Debug, Clone, Default, Parser)]
 pub struct RethMetricArgs {
     /// Enable Prometheus metrics for reth execution-layer components.
+    ///
+    /// Overrides `reth_metrics_address` in parameters.yaml when passed.
     #[arg(long = "reth-metrics", value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Reth Metrics"
     )]
     pub prometheus: Option<SocketAddr>,

crates/infrastructure/config/src/node.rs

@@ -2,14 +2,14 @@
 
 use crate::{ConfigFmt, ConfigTrait, NodeInfo, RaylsDirs};
 use rayls_infrastructure_types::{
-    get_available_tcp_port, get_available_udp_port, test_genesis, Address, BlsPublicKey,
-    BlsSignature, Genesis, Multiaddr, NetworkPublicKey, RaylsNetwork,
+    get_available_udp_port, test_genesis, Address, BlsPublicKey,
+    BlsSignature, Genesis, NetworkPublicKey, RaylsNetwork,
     ETHEREUM_BLOCK_GAS_LIMIT_56BITS, MAINNET_COMMITTEE, MAINNET_GENESIS, MAINNET_PARAMETERS,
     MIN_RAYLS_PROTOCOL_BASE_FEE, TESTNET_COMMITTEE, TESTNET_GENESIS, TESTNET_PARAMETERS,
 };
 use reth_chainspec::ChainSpec;
 use serde::{Deserialize, Serialize};
-use std::{fs::File, io::Write, time::Duration};
+use std::{fs::File, io::Write, net::SocketAddr, time::Duration};
 use tracing::info;
 
 /// The filename to use when reading/writing the validator's BlsKey.
@@ -261,6 +261,15 @@ pub struct Parameters {
     /// Controls the maximum gas per block and batch.
     #[serde(default = "Parameters::default_gas_limit")]
     pub gas_limit: u64,
+    /// Address for the consensus Prometheus metrics endpoint (the Narwhal/consensus metrics
+    /// suite). `None` (the default) leaves it off. The `--metrics` CLI flag overrides this when
+    /// passed, so operators can enable metrics from `parameters.yaml` without a flag.
+    #[serde(default)]
+    pub metrics_address: Option<SocketAddr>,
+    /// Address for the reth execution-layer Prometheus metrics endpoint. `None` (the default)
+    /// leaves it off. The `--reth-metrics` CLI flag overrides this when passed.
+    #[serde(default)]
+    pub reth_metrics_address: Option<SocketAddr>,
 }
 
 impl Parameters {
@@ -335,29 +344,6 @@ impl Default for NetworkAdminServerParameters {
     }
 }
 
-/// Prometheus metrics multiaddr.
-#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
-pub struct PrometheusMetricsParameters {
-    /// Socket address the server should be listening to.
-    pub socket_addr: Multiaddr,
-}
-
-impl Default for PrometheusMetricsParameters {
-    fn default() -> Self {
-        let host = "127.0.0.1";
-        Self {
-            socket_addr: format!(
-                "/ip4/{}/tcp/{}/http",
-                host,
-                get_available_tcp_port(host)
-                    .expect("os has available TCP port for default prometheus metrics")
-            )
-            .parse()
-            .expect("default prometheus metrics to parse available socket addr on localhost"),
-        }
-    }
-}
-
 impl Default for Parameters {
     fn default() -> Self {
         Self {
@@ -376,6 +362,8 @@ impl Default for Parameters {
             network: RaylsNetwork::default(),
             min_base_fee: Parameters::default_min_base_fee(),
             gas_limit: Parameters::default_gas_limit(),
+            metrics_address: None,
+            reth_metrics_address: None,
         }
     }
 }
@@ -395,5 +383,48 @@ impl Parameters {
         info!(network = %self.network, "Rayls network hardfork profile");
         info!("Minimum base fee set to {} wei", self.min_base_fee);
         info!("Block gas limit set to {}", self.gas_limit);
+        if let Some(addr) = self.metrics_address {
+            info!(%addr, "Consensus Prometheus metrics endpoint");
+        }
+        if let Some(addr) = self.reth_metrics_address {
+            info!(%addr, "Reth execution-layer Prometheus metrics endpoint");
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parameters_without_metrics_address_default_to_off() {
+        // Backward compatibility: an existing parameters.yaml with no `metrics_address` must
+        // still parse, leaving consensus metrics disabled.
+        let params: Parameters =
+            serde_yaml::from_str("gc_depth: 50\n").expect("parses without metrics_address");
+        assert_eq!(params.metrics_address, None);
+        assert_eq!(params.gc_depth, 50);
+    }
+
+    #[test]
+    fn parameters_parse_metrics_address_when_present() {
+        let params: Parameters = serde_yaml::from_str("metrics_address: \"127.0.0.1:9184\"\n")
+            .expect("parses with metrics_address");
+        assert_eq!(params.metrics_address, Some("127.0.0.1:9184".parse().unwrap()));
+    }
+
+    #[test]
+    fn default_parameters_have_metrics_off() {
+        assert_eq!(Parameters::default().metrics_address, None);
+        assert_eq!(Parameters::default().reth_metrics_address, None);
+    }
+
+    #[test]
+    fn reth_metrics_address_parses_and_defaults_off() {
+        let off: Parameters = serde_yaml::from_str("gc_depth: 50\n").unwrap();
+        assert_eq!(off.reth_metrics_address, None);
+        let on: Parameters =
+            serde_yaml::from_str("reth_metrics_address: \"0.0.0.0:9001\"\n").unwrap();
+        assert_eq!(on.reth_metrics_address, Some("0.0.0.0:9001".parse().unwrap()));
     }
 }

crates/infrastructure/network-cli/src/node.rs

@@ -74,9 +74,10 @@ pub struct NodeCommand<Ext: clap::Args + fmt::Debug = NoArgs> {
     #[arg(long, value_name = "NAMED_RL_NETWORK", verbatim_doc_comment)]
     pub chain: Option<NamedChain>,
 
-    /// Enable Prometheus consensus metrics.
+    /// Enable Prometheus consensus metrics, served at the given interface and port.
     ///
-    /// The metrics will be served at the given interface and port.
+    /// Overrides `metrics_address` in parameters.yaml when passed. If neither is set,
+    /// consensus metrics stay off.
     #[arg(long, value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Consensus Metrics")]
     pub metrics: Option<SocketAddr>,
 
@@ -265,6 +266,22 @@ impl<Ext: clap::Args + fmt::Debug> NodeCommand<Ext> {
             consensus_db,
         } = self;
 
+        // Both metrics endpoints can also be enabled from parameters.yaml — `metrics_address` for
+        // the consensus/Narwhal suite and `reth_metrics_address` for the execution layer. The
+        // `--metrics` / `--reth-metrics` CLI flags override the config values when passed.
+        let metrics = metrics.or(rayls_infrastructure_config.parameters.metrics_address);
+        if let Some(addr) = metrics {
+            info!(target: "cli", %addr, "consensus Prometheus metrics enabled");
+        }
+        let mut reth = reth;
+        reth.reth_metrics.prometheus = reth
+            .reth_metrics
+            .prometheus
+            .or(rayls_infrastructure_config.parameters.reth_metrics_address);
+        if let Some(addr) = reth.reth_metrics.prometheus {
+            info!(target: "cli", %addr, "reth execution-layer Prometheus metrics enabled");
+        }
+
         debug!(target: "cli", "node command genesis: {:#?}", rayls_infrastructure_config.genesis());
 
         // set up reth node config for engine components

crates/middleware/orchestrator/src/types/health.rs

@@ -23,7 +23,7 @@ use tracing::info;
 /// No connection limits or rate limiting are implemented.
 /// Connections are immediately closed after sending response.
 ///
-/// To enable on node startup, use `rayls-network node --enable-healthcheck`.
+/// To enable on node startup, use `rayls-network node --healthcheck <PORT>`.
 /// See `rayls-network-cli::node` for more info.
 #[derive(Debug)]
 pub(crate) struct HealthcheckServer;

etc/monitoring/README.md

@@ -0,0 +1,66 @@
+# Rayls node monitoring
+
+A ready-to-run Prometheus + Grafana + node_exporter setup for observing a Rayls node.
+
+Background and the reuse-vs-build strategy are in the Network Observability research
+(`axyl-private#381`, under the observability initiative `#429`).
+
+## What a node exposes
+
+A Rayls node has **two** Prometheus endpoints (separate registries), plus host metrics via
+node_exporter:
+
+| Endpoint | Enable with | Exposes |
+|---|---|---|
+| **Consensus / Narwhal** | `metrics_address` in parameters.yaml, or `--metrics <socket>` | `ConsensusMetrics`, `PrimaryMetrics`, `WorkerMetrics`, `NetworkMetrics`, `ExecutorMetrics` — round progress, commit latency, certificate throughput, leader election, DAG depth, epoch, peer connectivity, storage gauges |
+| **Reth execution** | `reth_metrics_address` in parameters.yaml, or `--reth-metrics <socket>` | reth execution-layer metrics — block processing, txpool, DB/static-file, process |
+| **Host** | run node_exporter | RSS / CPU / disk / network |
+
+Both endpoints are **off by default**. The CLI flags override the parameters.yaml values when
+passed.
+
+## 1. Enable the endpoints
+
+Add to the node's `parameters.yaml`:
+
+```yaml
+metrics_address: "0.0.0.0:9184"        # consensus / Narwhal suite
+reth_metrics_address: "0.0.0.0:9001"   # reth execution layer
+```
+
+Restart the node. It logs the active endpoints at startup (`… metrics enabled`).
+
+## 2. Point Prometheus at your node
+
+Edit [`prometheus.yml`](./prometheus.yml) so the `rayls-consensus` / `rayls-execution` targets
+match the addresses above. The defaults assume the node runs on the docker host
+(`host.docker.internal`).
+
+## 3. Run the stack
+
+```sh
+docker compose -f etc/monitoring/docker-compose.yml up -d
+```
+
+- Prometheus → http://localhost:9090 (check **Status → Targets**: all three jobs `UP`)
+- Grafana → http://localhost:3000 (`admin` / `admin`)
+
+The Prometheus datasource is auto-provisioned in Grafana.
+
+## 4. Dashboards
+
+- **Execution layer:** in Grafana, *Dashboards → Import → 20638* — the official
+  [reth dashboard](https://grafana.com/grafana/dashboards/20638-reth/). It works as-is against the
+  `rayls-execution` metrics.
+- **Consensus / validator health:** a Rayls-specific dashboard (per-validator liveness, committee/
+  epoch state, round lag) is the next initiative deliverable — see
+  [#428](https://github.com/raylsnetwork/axyl-private/issues/428). Until then, the consensus
+  metrics are queryable directly in Prometheus (e.g. `current_round`, `last_committed_round`,
+  `consensus_dag_rounds`, `leader_election`, `connected_peers_count`).
+
+## Notes
+
+- **Two targets per node, on purpose.** The consensus and execution metrics live in separate
+  registries, so they are scraped as two jobs. The `layer` label (`consensus` / `execution` /
+  `host`) distinguishes them.
+- This stack is a **local/operator convenience**, not a hardened production deployment.

etc/monitoring/docker-compose.yml

@@ -0,0 +1,61 @@
+# One-command Prometheus + Grafana + node_exporter stack for monitoring a Rayls node.
+#
+# Quick start (see README.md for the full walkthrough):
+#   1. Enable the node's metrics endpoints in its parameters.yaml:
+#        metrics_address: "0.0.0.0:9184"        # consensus / Narwhal suite
+#        reth_metrics_address: "0.0.0.0:9001"   # reth execution layer
+#      (or pass --metrics / --reth-metrics on the CLI). Restart the node.
+#   2. Adjust the targets in prometheus.yml if the node isn't on the docker host.
+#   3. `docker compose -f etc/monitoring/docker-compose.yml up -d`
+#   4. Open Grafana at http://localhost:3000 (admin / admin) and import the reth dashboard
+#      (Grafana.com dashboard ID 20638) against the "Prometheus" datasource.
+#
+# This is a local/operator convenience, not a production monitoring stack.
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: rayls-prometheus
+    restart: unless-stopped
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    ports:
+      - "9090:9090"
+    extra_hosts:
+      # lets the scrape targets reach a node running on the docker host
+      - "host.docker.internal:host-gateway"
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: rayls-grafana
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_AUTH_ANONYMOUS_ENABLED=false
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafana-data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: rayls-node-exporter
+    restart: unless-stopped
+    command:
+      - --path.rootfs=/host
+    pid: host
+    volumes:
+      - /:/host:ro,rslave
+    ports:
+      - "9100:9100"
+
+volumes:
+  prometheus-data:
+  grafana-data:

etc/monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,11 @@
+# Grafana datasource provisioning — auto-adds the Prometheus datasource on first start,
+# so imported dashboards (e.g. the reth dashboard, Grafana.com ID 20638) resolve immediately.
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

etc/monitoring/prometheus.yml

@@ -0,0 +1,39 @@
+# Prometheus scrape config for a Rayls node.
+#
+# Rayls exposes metrics on TWO separate endpoints, in two separate registries (see
+# doc/design/network-observability-research.md):
+#   - consensus / Narwhal suite  — enabled via `metrics_address` in parameters.yaml (or --metrics)
+#   - reth execution layer       — enabled via `reth_metrics_address` in parameters.yaml (or --reth-metrics)
+# Because they are distinct registries, scrape BOTH as separate targets per node; the `layer`
+# label lets dashboards/queries tell them apart. Host metrics come from node_exporter.
+#
+# Point the `targets` below at each node's configured endpoints (defaults shown are the ports
+# used by etc/monitoring/docker-compose.yml against a node running on the docker host).
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  # Consensus / Narwhal metrics — ConsensusMetrics, PrimaryMetrics, WorkerMetrics, NetworkMetrics,
+  # ExecutorMetrics, plus storage gauges. Set `metrics_address` in the node's parameters.yaml.
+  - job_name: rayls-consensus
+    static_configs:
+      - targets: ["host.docker.internal:9184"]
+        labels:
+          layer: consensus
+
+  # Reth execution-layer metrics — block processing, txpool, DB/static-file, process.
+  # Set `reth_metrics_address` in the node's parameters.yaml.
+  - job_name: rayls-execution
+    static_configs:
+      - targets: ["host.docker.internal:9001"]
+        labels:
+          layer: execution
+
+  # Host RSS / CPU / disk / network. Run node_exporter alongside the node.
+  - job_name: node-exporter
+    static_configs:
+      - targets: ["node-exporter:9100"]
+        labels:
+          layer: host