From 3b7ff47125d7119a8e3f7b50fe92a70312c55f72 Mon Sep 17 00:00:00 2001 From: Dan Bryan Date: Sun, 23 Aug 2026 18:24:46 -0400 Subject: [PATCH] Add optional Prometheus metrics endpoint Serves /metrics from a metrics.listen config block: connection state and dial results, requests by type and result including double-sign refusals, sign latency, the persisted sign state, raw backend latency and errors, and build/key metadata, plus the standard Go and process collectors. Also defaults the file backend algorithm to ed25519 as documented. --- .golangci.yml | 1 + README.md | 11 + cmd/kms/main.go | 19 + config/config.go | 8 + config/default.yaml | 7 + docs/grafana-dashboard.json | 1485 +++++++++++++++++++++++++++++++++++ docs/metrics.md | 89 +++ go.mod | 2 +- go.sum | 2 + internal/app/build.go | 10 + internal/manager/manager.go | 115 +++ internal/metrics/metrics.go | 172 ++++ internal/signer/address.go | 17 + 13 files changed, 1937 insertions(+), 1 deletion(-) create mode 100644 docs/grafana-dashboard.json create mode 100644 docs/metrics.md create mode 100644 internal/metrics/metrics.go create mode 100644 internal/signer/address.go diff --git a/.golangci.yml b/.golangci.yml index d3a0de6..7d56011 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,6 +29,7 @@ linters: - github.com/libp2p/go-libp2p - github.com/miekg/pkcs11 - github.com/aws/aws-sdk-go-v2 + - github.com/prometheus/client_golang - github.com/spf13/cobra - github.com/stretchr/testify - github.com/decred/dcrd/dcrec/secp256k1 diff --git a/README.md b/README.md index 548c8cc..a2efa8a 100644 --- a/README.md +++ b/README.md @@ -440,6 +440,17 @@ CometBFT are Ed25519, so no extra setup is needed. --- +## Metrics + +kms can serve Prometheus metrics (an optional `metrics:` config block; the +standard Go and process collectors plus signer metrics: connection state, +requests and results including double-sign refusals, sign latency, the +persisted double-sign state, backend latency and errors, and build/key +metadata). See [docs/metrics.md](docs/metrics.md) for the full reference and +starter alerts. + +--- + ## Testing ```sh diff --git a/cmd/kms/main.go b/cmd/kms/main.go index 44d1b79..55dc4b6 100644 --- a/cmd/kms/main.go +++ b/cmd/kms/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "net/http" "os" "os/signal" "path/filepath" @@ -15,6 +16,7 @@ import ( "github.com/cosmos/kms/config" "github.com/cosmos/kms/internal/app" "github.com/cosmos/kms/internal/identity" + "github.com/cosmos/kms/internal/metrics" "github.com/cosmos/kms/internal/signer" "github.com/cosmos/kms/internal/version" ) @@ -148,6 +150,23 @@ func startCmd() *cobra.Command { defer srv.Close() } + if cfg.Metrics != nil { + if cfg.Metrics.Listen == "" { + return fmt.Errorf("config: metrics block requires listen") + } + ms, merr := metrics.NewServer(cfg.Metrics.Listen) + if merr != nil { + return fmt.Errorf("metrics listen %q: %w", cfg.Metrics.Listen, merr) + } + go func() { + if serr := ms.Serve(); serr != nil && serr != http.ErrServerClosed { + logger.Error("metrics server failed", "err", serr) + } + }() + defer ms.Close() + logger.Info("serving metrics", "listen", cfg.Metrics.Listen, "path", "/metrics") + } + logger.Info("kms started; press Ctrl-C to stop") sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) diff --git a/config/config.go b/config/config.go index bb299ac..f76d79a 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,14 @@ type Config struct { Validators []Validator `yaml:"validators"` Keys []Key `yaml:"keys"` GRPC *GRPCConfig `yaml:"grpc"` + + Metrics *MetricsConfig `yaml:"metrics"` +} + +// MetricsConfig enables the optional Prometheus endpoint. The listener +// carries no authentication; restrict access by network policy. +type MetricsConfig struct { + Listen string `yaml:"listen"` // host:port for GET /metrics } // Chain declares a chain and its double-sign state file. diff --git a/config/default.yaml b/config/default.yaml index fd66a7f..4995660 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -61,3 +61,10 @@ keys: # algorithm: ed25519 # optional; default "ed25519" for awskms # key_id: alias/attestor # KMS id, ARN, or alias/ # region: us-east-1 # optional; AWS default chain otherwise + +# Optional Prometheus endpoint. Serves GET /metrics with the kms signer +# metrics plus the standard Go and process collectors. The listener carries +# NO authentication; restrict access with network controls. +# +# metrics: +# listen: 0.0.0.0:8545 diff --git a/docs/grafana-dashboard.json b/docs/grafana-dashboard.json new file mode 100644 index 0000000..a79e61e --- /dev/null +++ b/docs/grafana-dashboard.json @@ -0,0 +1,1485 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus-compatible datasource scraping the kms /metrics endpoint", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "uid": "kms-signer", + "title": "KMS Remote Signer", + "description": "Is the remote signer up, signing, and fast. Green healthy, yellow warning-backed, red critical-backed, purple informational.", + "tags": [ + "kms", + "validators" + ], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "annotations": { + "list": [] + }, + "editable": true, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "type": "row", + "id": 1, + "title": "Signer: up and signing?", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "Connected", + "description": "The dial-out privval connection to the validator. DOWN means the validator cannot sign: SignerDisconnected (critical, 1m).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(kms_validator_connected{job=\"kms\"})", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red", + "index": 0 + }, + "1": { + "text": "CONNECTED", + "color": "green", + "index": 1 + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "showPercentChange": false, + "percentChangeColorMode": "standard", + "text": {} + }, + "id": 2, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + } + }, + { + "type": "stat", + "title": "Last precommit", + "description": "Seconds since the last precommit was signed. Blocks are ~5 s apart; over 60 s means signing has stopped or the chain has: SignerNotSigning (critical, 2m).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "time() - max(kms_last_signed_timestamp_seconds{job=\"kms\", type=\"precommit\"})", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "showPercentChange": false, + "percentChangeColorMode": "standard", + "text": {} + }, + "id": 3, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + } + }, + { + "type": "stat", + "title": "Refused (1h)", + "description": "Requests the double-sign guard declined (height/round/step regression or conflicting data). Never expected in normal operation: SignerRefusedRequest (critical, immediate).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(kms_requests_total{job=\"kms\", result=\"refused\"}[1h])) or vector(0)", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "showPercentChange": false, + "percentChangeColorMode": "standard", + "text": {} + }, + "id": 4, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + } + }, + { + "type": "stat", + "title": "Backend errors (1h)", + "description": "Sign failures from the key custodian (file, pkcs11, awskms): BackendErrors (warning, 2m).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(kms_backend_errors_total[1h])) or vector(0)", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "showPercentChange": false, + "percentChangeColorMode": "standard", + "text": {} + }, + "id": 5, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + } + }, + { + "type": "stat", + "title": "Sign p99 (5m)", + "description": "Whole-request signing latency. Normal is tens of microseconds on the file backend; sustained 100 ms+ costs rounds: SignLatencyHigh (warning, 15m).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(kms_sign_duration_seconds_bucket{job=\"kms\"}[5m])))", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.1 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "showPercentChange": false, + "percentChangeColorMode": "standard", + "text": {} + }, + "id": 6, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + } + }, + { + "type": "stat", + "title": "Connects (1h)", + "description": "Successful dials in the last hour. More than one usually means the validator restarted or the network path flapped. Informational.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase(kms_validator_dials_total{job=\"kms\", result=\"ok\"}[1h])) or vector(0)", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "#B877D9" + }, + "mappings": [], + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "showPercentChange": false, + "percentChangeColorMode": "standard", + "text": {} + }, + "id": 7, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + } + }, + { + "type": "row", + "id": 8, + "title": "Identity", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "table", + "title": "Signing key", + "description": "The key each chain signs with. The address must match the validator's on-chain consensus address; anything else means the wrong key is loaded.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kms_key_info{job=\"kms\"}", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": false, + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "includeByName": { + "chain_id": true, + "backend": true, + "algorithm": true, + "address": true + }, + "indexByName": { + "chain_id": 0, + "backend": 1, + "algorithm": 2, + "address": 3 + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": false, + "minWidth": 90 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + }, + "noValue": "-" + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [] + }, + "id": 9, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 6 + } + }, + { + "type": "table", + "title": "Build", + "description": "Running signer build.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kms_build_info{job=\"kms\"}", + "refId": "A", + "editorMode": "code", + "legendFormat": "__auto", + "range": false, + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "includeByName": { + "version": true, + "go_version": true + }, + "indexByName": { + "version": 0, + "go_version": 1 + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": false, + "minWidth": 90 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + }, + "noValue": "-" + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [] + }, + "id": 10, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + } + }, + { + "type": "row", + "id": 11, + "title": "Latency and throughput", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Sign latency", + "description": "Per-type p99 and overall p50 signing latency. The yellow line is the SignLatencyHigh threshold (100 ms).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le, type) (rate(kms_sign_duration_seconds_bucket{job=\"kms\"}[5m])))", + "refId": "A", + "editorMode": "code", + "legendFormat": "{{type}} p99", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(kms_sign_duration_seconds_bucket{job=\"kms\"}[5m])))", + "refId": "B", + "editorMode": "code", + "legendFormat": "p50", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#EAB839", + "value": 0.1 + } + ] + }, + "noValue": "0", + "min": 0 + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 12, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + } + }, + { + "type": "timeseries", + "title": "Requests served", + "description": "Successful privval requests by type. Prevote and precommit track block cadence, proposal only when this validator proposes, pubkey once per height, ping every few seconds. Informational.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (type) (rate(kms_requests_total{job=\"kms\", result=\"ok\"}[5m]))", + "refId": "A", + "editorMode": "code", + "legendFormat": "{{type}}", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "noValue": "0", + "min": 0 + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 13, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + } + }, + { + "type": "row", + "id": 14, + "title": "Progress and problems", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Signed height", + "description": "Height of the last signature per message type. All three climb together on a healthy validator; a type falling behind means failed rounds. Informational; staleness is covered by the Last precommit stat.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (type) (kms_last_signed_height{job=\"kms\"})", + "refId": "A", + "editorMode": "code", + "legendFormat": "{{type}}", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "noValue": "0", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 15, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + } + }, + { + "type": "timeseries", + "title": "Refusals and errors", + "description": "Everything that did not sign cleanly. refused is the double-sign guard (critical-backed); error and backend errors are warning-backed.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (result) (rate(kms_requests_total{job=\"kms\", result!=\"ok\"}[5m]))", + "refId": "A", + "editorMode": "code", + "legendFormat": "{{result}}", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(kms_backend_errors_total[5m])) or vector(0)", + "refId": "B", + "editorMode": "code", + "legendFormat": "backend errors", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "noValue": "0", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "refused" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#EAB839" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "backend errors" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#EAB839" + } + } + ] + } + ] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 16, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + } + }, + { + "type": "row", + "id": 19, + "title": "Process", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Memory", + "description": "Signer process memory. A slow climb over days is a leak. Informational.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "process_resident_memory_bytes{job=\"kms\"}", + "refId": "A", + "editorMode": "code", + "legendFormat": "RSS", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_memstats_heap_inuse_bytes{job=\"kms\"}", + "refId": "B", + "editorMode": "code", + "legendFormat": "heap in use", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "noValue": "0", + "min": 0 + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 20, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 42 + } + }, + { + "type": "timeseries", + "title": "CPU", + "description": "Signer CPU. Signing is cheap; sustained load points at reconnect churn or something else on the pod. Informational.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(process_cpu_seconds_total{job=\"kms\"}[5m])", + "refId": "A", + "editorMode": "code", + "legendFormat": "cpu", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "noValue": "0", + "min": 0 + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 21, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 42 + } + }, + { + "type": "timeseries", + "title": "Goroutines", + "description": "Goroutine count; flat in steady state, one bump per connection. Unbounded growth is a leak. Informational.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_goroutines{job=\"kms\"}", + "refId": "A", + "editorMode": "code", + "legendFormat": "goroutines", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "barAlignment": 0, + "lineWidth": 2, + "fillOpacity": 18, + "gradientMode": "opacity", + "spanNulls": true, + "insertNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "noValue": "0", + "min": 0 + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "hideZeros": false + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "id": 22, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 42 + } + } + ] +} diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..fe24072 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,89 @@ +# Metrics + +kms can serve Prometheus metrics over HTTP. Collection is always on; the +endpoint is opt-in: + +```yaml +metrics: + listen: 0.0.0.0:8545 # GET /metrics; omit the block to disable +``` + +The listener carries no authentication, the same trade as the gRPC signer +service: restrict access with network controls. + +Naming follows the Prometheus conventions: counters end in `_total`, +durations are `_seconds` histograms in base units, timestamps are +`_timestamp_seconds` gauges (alert on `time()` minus the value), and +constant metadata series end in `_info` with a value of 1. All series are +prefixed `kms_`, and every signer series carries `chain_id` because one +process can sign for several chains. + +## Connection + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `kms_validator_connected` | gauge | chain_id, addr | 1 while the dial-out privval connection is established | +| `kms_validator_connected_since_timestamp_seconds` | gauge | chain_id, addr | Unix time the current connection was established | +| `kms_validator_dials_total` | counter | chain_id, addr, result=`ok`\|`error` | Dial attempts; a burst of `ok` means the validator is restarting | + +## Signing + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `kms_requests_total` | counter | chain_id, type=`proposal`\|`prevote`\|`precommit`\|`pubkey`\|`ping`, result=`ok`\|`refused`\|`error` | Every privval request served. `refused` is the double-sign guard declining a height/round/step regression or conflicting data | +| `kms_sign_duration_seconds` | histogram | chain_id, type | Whole-request latency: double-sign checks, backend signing, state persistence | +| `kms_last_signed_height` / `_round` | gauge | chain_id, type | Most recent successful signature per message type | +| `kms_last_signed_timestamp_seconds` | gauge | chain_id, type | Unix time of that signature | + +## Double-sign protection state + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `kms_sign_state_height` / `_round` / `_step` | gauge | chain_id | The persisted high-water mark the signer refuses to cross (step: 1 proposal, 2 prevote, 3 precommit) | + +## Key backend + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `kms_backend_sign_duration_seconds` | histogram | backend, algorithm | Raw `Sign` latency of the custodian (file, pkcs11, awskms). The pkcs11 and awskms backends put an HSM or a network API in the hot path, so this is where their health shows | +| `kms_backend_errors_total` | counter | backend, algorithm | Errors returned by the custodian's `Sign` | + +## Metadata + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `kms_build_info` | gauge (1) | version, go_version | Running build | +| `kms_key_info` | gauge (1) | chain_id, backend, algorithm, address | Configured key per chain; `address` is the public consensus address, so dashboards can assert the signer holds the expected key | + +The standard `go_*` and `process_*` collectors (heap, GC, goroutines, file +descriptors, RSS) are served from the same registry. + +## Dashboard + +A ready-to-import Grafana dashboard covering these metrics ships at +[`docs/grafana-dashboard.json`](grafana-dashboard.json). Importing it prompts +for a Prometheus datasource; thresholds on the board match the starter alerts +below. + +## Starter alerts + +```yaml +- alert: SignerDisconnected + expr: kms_validator_connected == 0 + for: 1m +- alert: SignerNotSigning + expr: time() - max by (chain_id) (kms_last_signed_timestamp_seconds) > 60 + for: 2m +- alert: SignerRefusedRequest + expr: increase(kms_requests_total{result="refused"}[5m]) > 0 +- alert: BackendErrors + expr: increase(kms_backend_errors_total[5m]) > 0 +- alert: SignLatencyHigh + expr: histogram_quantile(0.99, sum by (le) (rate(kms_sign_duration_seconds_bucket[5m]))) > 0.1 + for: 15m +``` + +`SignerDisconnected` only works while the process is alive to report 0; pair +it with scrape-absence alerting (`up == 0` / `absent(up{job=...})`) for the +process-death case. + diff --git a/go.mod b/go.mod index b9b8387..c148788 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/libp2p/go-libp2p v0.47.0 github.com/miekg/pkcs11 v1.1.2 github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a + github.com/prometheus/client_golang v1.24.1 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.54.0 @@ -126,7 +127,6 @@ require ( github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.24.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect diff --git a/go.sum b/go.sum index 3873183..812f8b6 100644 --- a/go.sum +++ b/go.sum @@ -185,6 +185,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= diff --git a/internal/app/build.go b/internal/app/build.go index e302002..914eb8e 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -16,6 +16,7 @@ import ( gensignerservice "github.com/cosmos/kms/gen/signerservice" "github.com/cosmos/kms/internal/identity" "github.com/cosmos/kms/internal/manager" + "github.com/cosmos/kms/internal/metrics" "github.com/cosmos/kms/internal/signer" "github.com/cosmos/kms/internal/signerservice" "github.com/cosmos/kms/internal/transport" @@ -53,6 +54,12 @@ func Build(c *config.Config, allowFresh []string, logger log.Logger) (mgr *manag if berr != nil { return nil, cleanup, berr } + s = metrics.WrapSigner(string(k.Backend), string(s.Scheme()), s) + if addr, aerr := signer.ConsensusAddress(s); aerr == nil { + for _, id := range k.ChainIDs { + metrics.KeyInfo.WithLabelValues(id, string(k.Backend), string(s.Scheme()), addr).Set(1) + } + } closers = append(closers, s) for _, id := range k.ChainIDs { if _, dup := keySigners[id]; dup { @@ -130,6 +137,9 @@ func Build(c *config.Config, allowFresh []string, logger log.Logger) (mgr *manag func newPrivvalSigner(k config.Key) (signing.Signer, error) { switch k.Backend { case config.BackendFile: + if k.Algorithm == "" { + k.Algorithm = config.AlgoED25519 + } s, err := file.Open(file.Config{ Algorithm: k.Algorithm, KeyFile: k.KeyFile, diff --git a/internal/manager/manager.go b/internal/manager/manager.go index b752f3a..b5789dc 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -2,6 +2,7 @@ package manager import ( "net" + "strings" "sync" "time" @@ -10,7 +11,10 @@ import ( "github.com/cometbft/cometbft/libs/protoio" "github.com/cometbft/cometbft/privval" privvalproto "github.com/cometbft/cometbft/proto/tendermint/privval" + cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" + "github.com/cometbft/cometbft/types" + "github.com/cosmos/kms/internal/metrics" ) const ( @@ -79,10 +83,18 @@ func (m *Manager) run(c ValidatorConn) { defer m.wg.Done() logger := m.logger.With("chain", c.ChainID, "addr", c.Addr) + + // Materialize the per-connection series up front so a validator that is + // unreachable from startup reports connected == 0 rather than no data. + metrics.ValidatorConnected.WithLabelValues(c.ChainID, c.Addr).Set(0) + metrics.ValidatorDials.WithLabelValues(c.ChainID, c.Addr, "ok").Add(0) + metrics.ValidatorDials.WithLabelValues(c.ChainID, c.Addr, "error").Add(0) + base := c.Dialer if base == nil { base = privval.DialTCPFn(c.Addr, defaultDialTimeout, c.IdentityKey) } + base = instrumentDialer(base, c.ChainID, c.Addr) dialer := backoffDialer(base, m.stop, logger, defaultBackoffInitial, defaultBackoffMax) for { @@ -97,8 +109,11 @@ func (m *Manager) run(c ValidatorConn) { return // errDialerStopped: shutting down } logger.Info("kms: connected") + metrics.ValidatorConnected.WithLabelValues(c.ChainID, c.Addr).Set(1) + metrics.ValidatorConnectedSince.WithLabelValues(c.ChainID, c.Addr).Set(float64(time.Now().Unix())) m.handleConnection(conn, c, logger) + metrics.ValidatorConnected.WithLabelValues(c.ChainID, c.Addr).Set(0) if !c.Reconnect { logger.Info("kms: reconnect disabled; connection closed") @@ -155,8 +170,24 @@ func (m *Manager) serveConn(conn net.Conn, chainID string, signer types.PrivVali return } + start := time.Now() resp, err := privval.DefaultValidationRequestHandler(signer, req, chainID) kind, kv, ping := describeRequest(req) + mkind := requestKind(req) + metrics.Requests.WithLabelValues(chainID, mkind, classifyResult(err)).Inc() + switch mkind { + case kindProposal, kindPrevote, kindPrecommit: + metrics.SignDuration.WithLabelValues(chainID, mkind).Observe(time.Since(start).Seconds()) + if err == nil { + h, r := requestHeightRound(req) + metrics.LastSignedHeight.WithLabelValues(chainID, mkind).Set(float64(h)) + metrics.LastSignedRound.WithLabelValues(chainID, mkind).Set(float64(r)) + metrics.LastSignedTimestamp.WithLabelValues(chainID, mkind).Set(float64(time.Now().Unix())) + metrics.SignStateHeight.WithLabelValues(chainID).Set(float64(h)) + metrics.SignStateRound.WithLabelValues(chainID).Set(float64(r)) + metrics.SignStateStep.WithLabelValues(chainID).Set(signStep(mkind)) + } + } switch { case err != nil: // resp already carries an embedded RemoteSignerError; log loudly and still send it. @@ -203,3 +234,87 @@ func describeRequest(req privvalproto.Message) (kind string, kv []any, ping bool return "unknown", nil, false } } + +// Metric label values for privval request kinds. +const ( + kindProposal = "proposal" + kindPrevote = "prevote" + kindPrecommit = "precommit" + kindPubkey = "pubkey" + kindPing = "ping" +) + +// instrumentDialer wraps a SocketDialer so every dial attempt is counted. +func instrumentDialer(base privval.SocketDialer, chainID, addr string) privval.SocketDialer { + return func() (net.Conn, error) { + conn, err := base() + if err != nil { + metrics.ValidatorDials.WithLabelValues(chainID, addr, "error").Inc() + return nil, err + } + metrics.ValidatorDials.WithLabelValues(chainID, addr, "ok").Inc() + return conn, nil + } +} + +// requestKind maps a privval request onto the metrics type label. +func requestKind(req privvalproto.Message) string { + switch r := req.Sum.(type) { + case *privvalproto.Message_SignVoteRequest: + if v := r.SignVoteRequest.GetVote(); v != nil && v.Type == cmtproto.PrecommitType { + return kindPrecommit + } + return kindPrevote + case *privvalproto.Message_SignProposalRequest: + return kindProposal + case *privvalproto.Message_PubKeyRequest: + return kindPubkey + case *privvalproto.Message_PingRequest: + return kindPing + default: + return "unknown" + } +} + +// classifyResult maps a request-handler error onto the metrics result label. +// FilePV double-sign refusals surface as height/round/step regression or +// conflicting-data errors; everything else is an error. +func classifyResult(err error) string { + if err == nil { + return "ok" + } + msg := err.Error() + if strings.Contains(msg, "regression") || strings.Contains(msg, "conflicting data") { + return "refused" + } + return "error" +} + +// requestHeightRound extracts the height and round from a signing request. +func requestHeightRound(req privvalproto.Message) (int64, int32) { + switch r := req.Sum.(type) { + case *privvalproto.Message_SignVoteRequest: + if v := r.SignVoteRequest.GetVote(); v != nil { + return v.Height, v.Round + } + case *privvalproto.Message_SignProposalRequest: + if p := r.SignProposalRequest.GetProposal(); p != nil { + return p.Height, p.Round + } + } + return 0, 0 +} + +// signStep maps a signed message type onto the FilePV step it advances the +// double-sign floor to (1 proposal, 2 prevote, 3 precommit). +func signStep(kind string) float64 { + switch kind { + case kindProposal: + return 1 + case kindPrevote: + return 2 + case kindPrecommit: + return 3 + } + return 0 +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..86e13d6 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,172 @@ +// Package metrics defines the kms Prometheus registry, collectors, and the +// optional HTTP endpoint that serves them. +// +// Collection is always on; serving is opt-in via the `metrics` config block. +// Naming follows the Prometheus conventions: counters end in _total, durations +// are _seconds histograms in base units, timestamps are _timestamp_seconds +// gauges, and one-value metadata series end in _info. +package metrics + +import ( + "context" + "net" + "net/http" + "runtime" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/cosmos/kms/internal/version" + "github.com/cosmos/kms/signing" +) + +// Registry holds every kms collector plus the standard Go runtime and process +// collectors. +var Registry = prometheus.NewRegistry() + +func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec { + c := prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help}, labels) + Registry.MustRegister(c) + return c +} + +func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec { + g := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels) + Registry.MustRegister(g) + return g +} + +func newHistogramVec(name, help string, buckets []float64, labels ...string) *prometheus.HistogramVec { + h := prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: name, Help: help, Buckets: buckets}, labels) + Registry.MustRegister(h) + return h +} + +// signBuckets spans sub-millisecond in-memory signing through multi-second +// HSM or cloud-API latency. +var signBuckets = []float64{.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5} + +var ( + // BuildInfo is the standard build-information series with a constant value of 1. + BuildInfo = newGaugeVec("kms_build_info", + "Build information for the running kms binary; always 1.", + "version", "go_version") + + // KeyInfo describes each configured consensus key with a constant value of 1. + // The address label is the public consensus address derived from the key. + KeyInfo = newGaugeVec("kms_key_info", + "Configured signing key per chain; always 1. Labels carry the custodian backend, algorithm, and public consensus address.", + "chain_id", "backend", "algorithm", "address") + + // ValidatorConnected reports whether the outbound privval connection is up. + ValidatorConnected = newGaugeVec("kms_validator_connected", + "Whether the privval connection to this validator endpoint is currently established (1) or not (0).", + "chain_id", "addr") + + // ValidatorConnectedSince is the Unix time the current connection was established. + ValidatorConnectedSince = newGaugeVec("kms_validator_connected_since_timestamp_seconds", + "Unix time at which the current privval connection was established.", + "chain_id", "addr") + + // ValidatorDials counts dial attempts by result. + ValidatorDials = newCounterVec("kms_validator_dials_total", + "Privval dial attempts by result. A burst of ok results usually means the validator is restarting.", + "chain_id", "addr", "result") + + // Requests counts every privval request served, by type and result. + // result is ok, refused (the double-sign guard declined to sign), or error. + Requests = newCounterVec("kms_requests_total", + "Privval requests served, by request type and result. result=refused means the double-sign protection declined the request.", + "chain_id", "type", "result") + + // SignDuration measures whole-request signing latency as seen by the validator. + SignDuration = newHistogramVec("kms_sign_duration_seconds", + "Time to serve a signing request, including double-sign checks, backend signing, and state persistence.", + signBuckets, "chain_id", "type") + + // LastSignedHeight/Round/Timestamp record the most recent successful signature. + LastSignedHeight = newGaugeVec("kms_last_signed_height", + "Height of the most recent successfully signed message of this type.", + "chain_id", "type") + LastSignedRound = newGaugeVec("kms_last_signed_round", + "Round of the most recent successfully signed message of this type.", + "chain_id", "type") + LastSignedTimestamp = newGaugeVec("kms_last_signed_timestamp_seconds", + "Unix time of the most recent successful signature of this type. Alert on time() minus this value.", + "chain_id", "type") + + // SignState mirrors the persisted double-sign high-water mark. + SignStateHeight = newGaugeVec("kms_sign_state_height", + "Height of the persisted double-sign protection state; the signer refuses to sign at or below it.", + "chain_id") + SignStateRound = newGaugeVec("kms_sign_state_round", + "Round of the persisted double-sign protection state.", + "chain_id") + SignStateStep = newGaugeVec("kms_sign_state_step", + "Step of the persisted double-sign protection state (1 proposal, 2 prevote, 3 precommit).", + "chain_id") + + // BackendSignDuration and BackendErrors observe the raw key-custodian call. + BackendSignDuration = newHistogramVec("kms_backend_sign_duration_seconds", + "Raw Sign call latency of the key backend, excluding privval handling.", + signBuckets, "backend", "algorithm") + BackendErrors = newCounterVec("kms_backend_errors_total", + "Errors returned by the key backend's Sign call.", + "backend", "algorithm") +) + +func init() { + Registry.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + BuildInfo.WithLabelValues(version.String(), runtime.Version()).Set(1) +} + +// Server serves the registry over HTTP at /metrics. +type Server struct { + srv *http.Server + ln net.Listener +} + +// NewServer binds the listener. The endpoint carries no authentication; +// restrict access by network policy, exactly as with the gRPC signer service. +func NewServer(listen string) (*Server, error) { + ln, err := net.Listen("tcp", listen) + if err != nil { + return nil, err + } + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{})) + return &Server{srv: &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second}, ln: ln}, nil +} + +// Serve blocks serving the endpoint until Close. +func (s *Server) Serve() error { return s.srv.Serve(s.ln) } + +// Close shuts the server down. +func (s *Server) Close() { _ = s.srv.Close() } + +// wrappedSigner instruments a signing.Signer's Sign call. +type wrappedSigner struct { + signing.Signer + backend string + algorithm string +} + +// WrapSigner returns s with Sign latency and error instrumentation attached. +func WrapSigner(backend, algorithm string, s signing.Signer) signing.Signer { + return &wrappedSigner{Signer: s, backend: backend, algorithm: algorithm} +} + +func (w *wrappedSigner) Sign(ctx context.Context, payload []byte) ([]byte, error) { + start := time.Now() + sig, err := w.Signer.Sign(ctx, payload) + BackendSignDuration.WithLabelValues(w.backend, w.algorithm).Observe(time.Since(start).Seconds()) + if err != nil { + BackendErrors.WithLabelValues(w.backend, w.algorithm).Inc() + } + return sig, err +} diff --git a/internal/signer/address.go b/internal/signer/address.go new file mode 100644 index 0000000..6208ec8 --- /dev/null +++ b/internal/signer/address.go @@ -0,0 +1,17 @@ +package signer + +import ( + "context" + + "github.com/cosmos/kms/signing" +) + +// ConsensusAddress derives the public consensus address for a signer's key, +// in CometBFT's canonical uppercase-hex encoding. +func ConsensusAddress(s signing.Signer) (string, error) { + pk, err := newSignerPrivKey(context.Background(), s) + if err != nil { + return "", err + } + return pk.PubKey().Address().String(), nil +}