diff --git a/crates/execution/evm/src/reth_env/config.rs b/crates/execution/evm/src/reth_env/config.rs index b3d711e0..41081910 100644 --- a/crates/execution/evm/src/reth_env/config.rs +++ b/crates/execution/evm/src/reth_env/config.rs @@ -29,6 +29,8 @@ use tracing::warn; #[derive(Debug, Clone, Default, Parser)] pub struct RethMetricArgs { /// Enable Prometheus metrics for reth execution-layer components. + /// + /// Overrides `reth_metrics_address` in parameters.yaml when passed. #[arg(long = "reth-metrics", value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Reth Metrics" )] pub prometheus: Option, diff --git a/crates/infrastructure/config/src/node.rs b/crates/infrastructure/config/src/node.rs index b42a0a3f..8d75eb41 100644 --- a/crates/infrastructure/config/src/node.rs +++ b/crates/infrastructure/config/src/node.rs @@ -2,14 +2,14 @@ use crate::{ConfigFmt, ConfigTrait, NodeInfo, RaylsDirs}; use rayls_infrastructure_types::{ - get_available_tcp_port, get_available_udp_port, test_genesis, Address, BlsPublicKey, - BlsSignature, Genesis, Multiaddr, NetworkPublicKey, RaylsNetwork, + get_available_udp_port, test_genesis, Address, BlsPublicKey, + BlsSignature, Genesis, NetworkPublicKey, RaylsNetwork, ETHEREUM_BLOCK_GAS_LIMIT_56BITS, MAINNET_COMMITTEE, MAINNET_GENESIS, MAINNET_PARAMETERS, MIN_RAYLS_PROTOCOL_BASE_FEE, TESTNET_COMMITTEE, TESTNET_GENESIS, TESTNET_PARAMETERS, }; use reth_chainspec::ChainSpec; use serde::{Deserialize, Serialize}; -use std::{fs::File, io::Write, time::Duration}; +use std::{fs::File, io::Write, net::SocketAddr, time::Duration}; use tracing::info; /// The filename to use when reading/writing the validator's BlsKey. @@ -261,6 +261,15 @@ pub struct Parameters { /// Controls the maximum gas per block and batch. #[serde(default = "Parameters::default_gas_limit")] pub gas_limit: u64, + /// Address for the consensus Prometheus metrics endpoint (the Narwhal/consensus metrics + /// suite). `None` (the default) leaves it off. The `--metrics` CLI flag overrides this when + /// passed, so operators can enable metrics from `parameters.yaml` without a flag. + #[serde(default)] + pub metrics_address: Option, + /// Address for the reth execution-layer Prometheus metrics endpoint. `None` (the default) + /// leaves it off. The `--reth-metrics` CLI flag overrides this when passed. + #[serde(default)] + pub reth_metrics_address: Option, } impl Parameters { @@ -335,29 +344,6 @@ impl Default for NetworkAdminServerParameters { } } -/// Prometheus metrics multiaddr. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub struct PrometheusMetricsParameters { - /// Socket address the server should be listening to. - pub socket_addr: Multiaddr, -} - -impl Default for PrometheusMetricsParameters { - fn default() -> Self { - let host = "127.0.0.1"; - Self { - socket_addr: format!( - "/ip4/{}/tcp/{}/http", - host, - get_available_tcp_port(host) - .expect("os has available TCP port for default prometheus metrics") - ) - .parse() - .expect("default prometheus metrics to parse available socket addr on localhost"), - } - } -} - impl Default for Parameters { fn default() -> Self { Self { @@ -376,6 +362,8 @@ impl Default for Parameters { network: RaylsNetwork::default(), min_base_fee: Parameters::default_min_base_fee(), gas_limit: Parameters::default_gas_limit(), + metrics_address: None, + reth_metrics_address: None, } } } @@ -395,5 +383,48 @@ impl Parameters { info!(network = %self.network, "Rayls network hardfork profile"); info!("Minimum base fee set to {} wei", self.min_base_fee); info!("Block gas limit set to {}", self.gas_limit); + if let Some(addr) = self.metrics_address { + info!(%addr, "Consensus Prometheus metrics endpoint"); + } + if let Some(addr) = self.reth_metrics_address { + info!(%addr, "Reth execution-layer Prometheus metrics endpoint"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parameters_without_metrics_address_default_to_off() { + // Backward compatibility: an existing parameters.yaml with no `metrics_address` must + // still parse, leaving consensus metrics disabled. + let params: Parameters = + serde_yaml::from_str("gc_depth: 50\n").expect("parses without metrics_address"); + assert_eq!(params.metrics_address, None); + assert_eq!(params.gc_depth, 50); + } + + #[test] + fn parameters_parse_metrics_address_when_present() { + let params: Parameters = serde_yaml::from_str("metrics_address: \"127.0.0.1:9184\"\n") + .expect("parses with metrics_address"); + assert_eq!(params.metrics_address, Some("127.0.0.1:9184".parse().unwrap())); + } + + #[test] + fn default_parameters_have_metrics_off() { + assert_eq!(Parameters::default().metrics_address, None); + assert_eq!(Parameters::default().reth_metrics_address, None); + } + + #[test] + fn reth_metrics_address_parses_and_defaults_off() { + let off: Parameters = serde_yaml::from_str("gc_depth: 50\n").unwrap(); + assert_eq!(off.reth_metrics_address, None); + let on: Parameters = + serde_yaml::from_str("reth_metrics_address: \"0.0.0.0:9001\"\n").unwrap(); + assert_eq!(on.reth_metrics_address, Some("0.0.0.0:9001".parse().unwrap())); } } diff --git a/crates/infrastructure/network-cli/src/node.rs b/crates/infrastructure/network-cli/src/node.rs index ccc1f7a0..28e989aa 100644 --- a/crates/infrastructure/network-cli/src/node.rs +++ b/crates/infrastructure/network-cli/src/node.rs @@ -74,9 +74,10 @@ pub struct NodeCommand { #[arg(long, value_name = "NAMED_RL_NETWORK", verbatim_doc_comment)] pub chain: Option, - /// Enable Prometheus consensus metrics. + /// Enable Prometheus consensus metrics, served at the given interface and port. /// - /// The metrics will be served at the given interface and port. + /// Overrides `metrics_address` in parameters.yaml when passed. If neither is set, + /// consensus metrics stay off. #[arg(long, value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Consensus Metrics")] pub metrics: Option, @@ -265,6 +266,22 @@ impl NodeCommand { consensus_db, } = self; + // Both metrics endpoints can also be enabled from parameters.yaml — `metrics_address` for + // the consensus/Narwhal suite and `reth_metrics_address` for the execution layer. The + // `--metrics` / `--reth-metrics` CLI flags override the config values when passed. + let metrics = metrics.or(rayls_infrastructure_config.parameters.metrics_address); + if let Some(addr) = metrics { + info!(target: "cli", %addr, "consensus Prometheus metrics enabled"); + } + let mut reth = reth; + reth.reth_metrics.prometheus = reth + .reth_metrics + .prometheus + .or(rayls_infrastructure_config.parameters.reth_metrics_address); + if let Some(addr) = reth.reth_metrics.prometheus { + info!(target: "cli", %addr, "reth execution-layer Prometheus metrics enabled"); + } + debug!(target: "cli", "node command genesis: {:#?}", rayls_infrastructure_config.genesis()); // set up reth node config for engine components diff --git a/crates/middleware/orchestrator/src/types/health.rs b/crates/middleware/orchestrator/src/types/health.rs index 54c517c2..66c88a07 100644 --- a/crates/middleware/orchestrator/src/types/health.rs +++ b/crates/middleware/orchestrator/src/types/health.rs @@ -23,7 +23,7 @@ use tracing::info; /// No connection limits or rate limiting are implemented. /// Connections are immediately closed after sending response. /// -/// To enable on node startup, use `rayls-network node --enable-healthcheck`. +/// To enable on node startup, use `rayls-network node --healthcheck `. /// See `rayls-network-cli::node` for more info. #[derive(Debug)] pub(crate) struct HealthcheckServer; diff --git a/etc/monitoring/README.md b/etc/monitoring/README.md new file mode 100644 index 00000000..1aef623e --- /dev/null +++ b/etc/monitoring/README.md @@ -0,0 +1,66 @@ +# Rayls node monitoring + +A ready-to-run Prometheus + Grafana + node_exporter setup for observing a Rayls node. + +Background and the reuse-vs-build strategy are in the Network Observability research +(`axyl-private#381`, under the observability initiative `#429`). + +## What a node exposes + +A Rayls node has **two** Prometheus endpoints (separate registries), plus host metrics via +node_exporter: + +| Endpoint | Enable with | Exposes | +|---|---|---| +| **Consensus / Narwhal** | `metrics_address` in parameters.yaml, or `--metrics ` | `ConsensusMetrics`, `PrimaryMetrics`, `WorkerMetrics`, `NetworkMetrics`, `ExecutorMetrics` — round progress, commit latency, certificate throughput, leader election, DAG depth, epoch, peer connectivity, storage gauges | +| **Reth execution** | `reth_metrics_address` in parameters.yaml, or `--reth-metrics ` | reth execution-layer metrics — block processing, txpool, DB/static-file, process | +| **Host** | run node_exporter | RSS / CPU / disk / network | + +Both endpoints are **off by default**. The CLI flags override the parameters.yaml values when +passed. + +## 1. Enable the endpoints + +Add to the node's `parameters.yaml`: + +```yaml +metrics_address: "0.0.0.0:9184" # consensus / Narwhal suite +reth_metrics_address: "0.0.0.0:9001" # reth execution layer +``` + +Restart the node. It logs the active endpoints at startup (`… metrics enabled`). + +## 2. Point Prometheus at your node + +Edit [`prometheus.yml`](./prometheus.yml) so the `rayls-consensus` / `rayls-execution` targets +match the addresses above. The defaults assume the node runs on the docker host +(`host.docker.internal`). + +## 3. Run the stack + +```sh +docker compose -f etc/monitoring/docker-compose.yml up -d +``` + +- Prometheus → http://localhost:9090 (check **Status → Targets**: all three jobs `UP`) +- Grafana → http://localhost:3000 (`admin` / `admin`) + +The Prometheus datasource is auto-provisioned in Grafana. + +## 4. Dashboards + +- **Execution layer:** in Grafana, *Dashboards → Import → 20638* — the official + [reth dashboard](https://grafana.com/grafana/dashboards/20638-reth/). It works as-is against the + `rayls-execution` metrics. +- **Consensus / validator health:** a Rayls-specific dashboard (per-validator liveness, committee/ + epoch state, round lag) is the next initiative deliverable — see + [#428](https://github.com/raylsnetwork/axyl-private/issues/428). Until then, the consensus + metrics are queryable directly in Prometheus (e.g. `current_round`, `last_committed_round`, + `consensus_dag_rounds`, `leader_election`, `connected_peers_count`). + +## Notes + +- **Two targets per node, on purpose.** The consensus and execution metrics live in separate + registries, so they are scraped as two jobs. The `layer` label (`consensus` / `execution` / + `host`) distinguishes them. +- This stack is a **local/operator convenience**, not a hardened production deployment. diff --git a/etc/monitoring/docker-compose.yml b/etc/monitoring/docker-compose.yml new file mode 100644 index 00000000..9c092668 --- /dev/null +++ b/etc/monitoring/docker-compose.yml @@ -0,0 +1,61 @@ +# One-command Prometheus + Grafana + node_exporter stack for monitoring a Rayls node. +# +# Quick start (see README.md for the full walkthrough): +# 1. Enable the node's metrics endpoints in its parameters.yaml: +# metrics_address: "0.0.0.0:9184" # consensus / Narwhal suite +# reth_metrics_address: "0.0.0.0:9001" # reth execution layer +# (or pass --metrics / --reth-metrics on the CLI). Restart the node. +# 2. Adjust the targets in prometheus.yml if the node isn't on the docker host. +# 3. `docker compose -f etc/monitoring/docker-compose.yml up -d` +# 4. Open Grafana at http://localhost:3000 (admin / admin) and import the reth dashboard +# (Grafana.com dashboard ID 20638) against the "Prometheus" datasource. +# +# This is a local/operator convenience, not a production monitoring stack. + +services: + prometheus: + image: prom/prometheus:latest + container_name: rayls-prometheus + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + extra_hosts: + # lets the scrape targets reach a node running on the docker host + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:latest + container_name: rayls-grafana + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=false + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + depends_on: + - prometheus + + node-exporter: + image: prom/node-exporter:latest + container_name: rayls-node-exporter + restart: unless-stopped + command: + - --path.rootfs=/host + pid: host + volumes: + - /:/host:ro,rslave + ports: + - "9100:9100" + +volumes: + prometheus-data: + grafana-data: diff --git a/etc/monitoring/grafana/provisioning/datasources/prometheus.yml b/etc/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..5d48d41d --- /dev/null +++ b/etc/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,11 @@ +# Grafana datasource provisioning — auto-adds the Prometheus datasource on first start, +# so imported dashboards (e.g. the reth dashboard, Grafana.com ID 20638) resolve immediately. +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/etc/monitoring/prometheus.yml b/etc/monitoring/prometheus.yml new file mode 100644 index 00000000..2cbdf7bb --- /dev/null +++ b/etc/monitoring/prometheus.yml @@ -0,0 +1,39 @@ +# Prometheus scrape config for a Rayls node. +# +# Rayls exposes metrics on TWO separate endpoints, in two separate registries (see +# doc/design/network-observability-research.md): +# - consensus / Narwhal suite — enabled via `metrics_address` in parameters.yaml (or --metrics) +# - reth execution layer — enabled via `reth_metrics_address` in parameters.yaml (or --reth-metrics) +# Because they are distinct registries, scrape BOTH as separate targets per node; the `layer` +# label lets dashboards/queries tell them apart. Host metrics come from node_exporter. +# +# Point the `targets` below at each node's configured endpoints (defaults shown are the ports +# used by etc/monitoring/docker-compose.yml against a node running on the docker host). + +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + # Consensus / Narwhal metrics — ConsensusMetrics, PrimaryMetrics, WorkerMetrics, NetworkMetrics, + # ExecutorMetrics, plus storage gauges. Set `metrics_address` in the node's parameters.yaml. + - job_name: rayls-consensus + static_configs: + - targets: ["host.docker.internal:9184"] + labels: + layer: consensus + + # Reth execution-layer metrics — block processing, txpool, DB/static-file, process. + # Set `reth_metrics_address` in the node's parameters.yaml. + - job_name: rayls-execution + static_configs: + - targets: ["host.docker.internal:9001"] + labels: + layer: execution + + # Host RSS / CPU / disk / network. Run node_exporter alongside the node. + - job_name: node-exporter + static_configs: + - targets: ["node-exporter:9100"] + labels: + layer: host