diff --git a/docs/DASHBOARDS.md b/docs/DASHBOARDS.md index e023c0a0..75f02cbd 100644 --- a/docs/DASHBOARDS.md +++ b/docs/DASHBOARDS.md @@ -18,3 +18,18 @@ Import [jenkins-kibana-dashboards.ndjson](../src/main/kibana/jenkins-kibana-dash ![Jenkins overview](./images/kibana_jenkins_overview_dashboard.png) ![Jenkins Provisioning](./images/kibana_jenkins_provisioning_dashboard.png) + +## Grafana + +Specific dashboard to be imported in your Grafana instance once the plugin and OpenTelemetry Collector have been configured with Prometheus. + +There are different ways to import a Grafana dashboard: + +* Through the UI +* Through dashboard provisioning + +### Jenkins Overview Grafana Dashboard + +Import [jenkins-overview.json](../src/main/grafana/jenkins-overview.json). + +For full setup instructions including OTel Collector and Prometheus configuration, see the [Grafana Dashboard Setup Guide](grafana-dashboard-setup.md). diff --git a/docs/grafana-dashboard-setup.md b/docs/grafana-dashboard-setup.md new file mode 100644 index 00000000..8db01488 --- /dev/null +++ b/docs/grafana-dashboard-setup.md @@ -0,0 +1,217 @@ +# Grafana dashboard setup + +This guide explains how to visualise Jenkins metrics in Grafana using the +Prometheus + OpenTelemetry Collector stack. For Elastic Kibana dashboards, +see [DASHBOARDS.md](DASHBOARDS.md). + +## Overview + +``` +Jenkins controller + │ OTLP/gRPC (port 4317) + ▼ +OpenTelemetry Collector + │ Prometheus exporter (port 8889) + ▼ +Prometheus ◄──── scrapes /metrics + │ + ▼ +Grafana ◄──── queries Prometheus datasource +``` + +## Prerequisites + +- Jenkins OpenTelemetry plugin installed and configured with an OTLP endpoint +- OpenTelemetry Collector running and reachable from Jenkins +- Prometheus running and reachable from the Collector +- Grafana running with a Prometheus datasource configured + +--- + +## Step 1 - Configure the OpenTelemetry Collector + +The Collector must have a `metrics` pipeline with a Prometheus exporter. +The `resource_to_telemetry_conversion` option is required - without it, +OpenTelemetry resource attributes (like `service.name`) are not converted +to Prometheus labels, so panels cannot filter by Jenkins instance. + +```yaml +# otel-collector-config.yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true # converts resource attributes ⇒ Prometheus labels + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] + traces: # keep if you also export traces to a backend + receivers: [otlp] + processors: [batch] + exporters: [] # add your trace backend exporter here +``` + +> ℹ️ Prometheus only supports the metrics signal. Traces and logs require +> a separate backend (Jaeger, Elastic, Loki, etc.). + +--- + +## Step 2 - Configure Prometheus to scrape the Collector + +Add a scrape job to your `prometheus.yml`: + +```yaml +scrape_configs: + - job_name: jenkins-otel + static_configs: + - targets: ['otel-collector:8889'] # host:port of the Collector + scrape_interval: 15s +``` + +Verify metrics are arriving after a build: + +```bash +curl http://otel-collector:8889/metrics | grep ci_pipeline +``` + +You should see metrics like `ci_pipeline_run_duration_milliseconds_count`. + +--- + +## Step 3 - Import the Grafana dashboard + +A ready-to-import Grafana dashboard JSON is provided at +[`src/main/grafana/jenkins-overview.json`](../src/main/grafana/jenkins-overview.json). + +### Import via the Grafana UI + +1. Open Grafana and go to **Dashboards ⇒ Import**. +2. Click **Upload JSON file** and select `src/main/grafana/jenkins-overview.json`. +3. Select your Prometheus datasource from the dropdown. +4. Click **Import**. + +### Import via provisioning (recommended for production) + +Copy the JSON file to your Grafana provisioning directory: + +```bash +cp src/main/grafana/jenkins-overview.json \ + /etc/grafana/provisioning/dashboards/jenkins-overview.json +``` + +Add a provisioning config if you don't already have one: + +```yaml +# /etc/grafana/provisioning/dashboards/jenkins.yaml +apiVersion: 1 +providers: + - name: Jenkins + type: file + options: + path: /etc/grafana/provisioning/dashboards +``` + +Restart Grafana. The dashboard appears automatically under **Dashboards**. + +--- + +## Step 4 - Link Grafana back to Jenkins build pages (optional) + +You can configure the plugin so that each Jenkins build page shows a +**"View in Grafana"** link. Use the `customObservabilityBackend` setting. + +### Via the Jenkins UI + +1. Go to **Manage Jenkins ⇒ Configure System ⇒ OpenTelemetry**. +2. Under **Visualization**, click **Add Observability Backend** ⇒ + **Custom Observability Backend**. +3. Set **Metrics visualization URL template** to: + ``` + http://your-grafana-host:3000/d/jenkins-overview/jenkins-overview?orgId=1&from=${startTime}&to=${endTime} + ``` +4. Click **Save**. + +### Via JCasC + +```yaml +unclassified: + openTelemetry: + authentication: "noAuthentication" + endpoint: "otel-collector:4317" + exportOtelConfigurationAsEnvironmentVariables: true + observabilityBackends: + - customObservabilityBackend: + name: "Grafana" + metricsVisualizationUrlTemplate: > + http://your-grafana-host:3000/d/jenkins-overview/jenkins-overview + ?orgId=1&from=${startTime}&to=${endTime} +``` + +--- + +## Dashboard panels + +The `jenkins-overview.json` dashboard includes the following panels: + +| Panel | Metric | Description | +|---|---|---| +| Build rate | `ci_pipeline_run_duration_milliseconds_count` | Builds per minute | +| Build duration (p50 / p95) | `ci_pipeline_run_duration_milliseconds` | Pipeline duration percentiles | +| Build success rate | `ci_pipeline_run_duration_milliseconds_count` filtered by `ci_pipeline_result="SUCCESS"` | Ratio of successful builds | +| Queue time (p50 / p95) | `ci_pipeline_run_queue_duration_milliseconds` | Time spent waiting in queue | +| Active executors | `jenkins_executor_count` | Executors in use vs available | +| Failed builds | `ci_pipeline_run_duration_milliseconds_count` filtered by `ci_pipeline_result="FAILURE"` | Count of failed builds | +| JVM heap usage | `runtime_jvm_memory_usage` | Jenkins controller JVM memory | +| HTTP request duration | `http_server_request_duration_seconds` | Jenkins controller HTTP latency | + +--- + +## Troubleshooting + +**No data in Grafana panels** + +- Confirm Prometheus is scraping the Collector: open `http://prometheus:9090/targets` + and check the `jenkins-otel` job is `UP`. +- Confirm `resource_to_telemetry_conversion.enabled: true` is set in the + Collector config - without this, label filters in the dashboard panels + will match nothing. +- Run at least one Jenkins build after configuring the plugin - metrics are + only emitted when builds occur. + +**`ci_pipeline_run_duration_milliseconds` missing** + +This metric requires an allow-list to be set. By default it matches nothing. +Add to **Configuration Properties** in the plugin: + +``` +otel.instrumentation.jenkins.run.metric.duration.allow_list=.* +``` + +Or restrict to specific jobs: + +``` +otel.instrumentation.jenkins.run.metric.duration.allow_list=my-team/.* +``` + +See [monitoring-metrics.md](monitoring-metrics.md) for the full list of +configuration parameters that control metric cardinality. + +**Dashboard shows data for wrong Jenkins instance** + +If you run multiple Jenkins controllers, add `service.name` as a Grafana +variable to filter by instance. Each controller's metrics are labelled with +the `service_name` Prometheus label (derived from the OTel `service.name` +resource attribute). \ No newline at end of file diff --git a/src/main/grafana/jenkins-overview.json b/src/main/grafana/jenkins-overview.json new file mode 100644 index 00000000..d5daec64 --- /dev/null +++ b/src/main/grafana/jenkins-overview.json @@ -0,0 +1,368 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus datasource scraping the OpenTelemetry Collector", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, + { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, + { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, + { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" } + ], + "annotations": { "list": [] }, + "description": "Jenkins health and CI pipeline metrics from the Jenkins OpenTelemetry plugin via Prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 20, + "title": "Pipeline health", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 80 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Builds (last 1h)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "increase(ci_pipeline_run_duration_milliseconds_count{service_name=~\"$service_name\"}[1h])", + "legendFormat": "builds", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 80 }, + { "color": "green", "value": 95 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 2, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Success rate (last 1h)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "100 * increase(ci_pipeline_run_duration_milliseconds_count{ci_pipeline_result=\"SUCCESS\",service_name=~\"$service_name\"}[1h]) / clamp_min(increase(ci_pipeline_run_duration_milliseconds_count{service_name=~\"$service_name\"}[1h]), 1)", + "legendFormat": "success %", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60000 }, + { "color": "red", "value": 300000 } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 3, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Median build duration (p50)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.50, sum(rate(ci_pipeline_run_duration_milliseconds_bucket{service_name=~\"$service_name\"}[30m])) by (le))", + "legendFormat": "p50", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 120000 }, + { "color": "red", "value": 600000 } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 4, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "p95 build duration", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.95, sum(rate(ci_pipeline_run_duration_milliseconds_bucket{service_name=~\"$service_name\"}[30m])) by (le))", + "legendFormat": "p95", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "unit": "ms", "custom": { "lineWidth": 1, "fillOpacity": 10 } }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "id": 5, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Build duration over time (p50 / p95)", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.50, sum(rate(ci_pipeline_run_duration_milliseconds_bucket{service_name=~\"$service_name\"}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.95, sum(rate(ci_pipeline_run_duration_milliseconds_bucket{service_name=~\"$service_name\"}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "B" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "unit": "short", "custom": { "lineWidth": 1, "fillOpacity": 10 } }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, + "id": 6, + "options": { "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Build results over time", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "increase(ci_pipeline_run_duration_milliseconds_count{ci_pipeline_result=\"SUCCESS\",service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "SUCCESS", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "increase(ci_pipeline_run_duration_milliseconds_count{ci_pipeline_result=\"FAILURE\",service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "FAILURE", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "increase(ci_pipeline_run_duration_milliseconds_count{ci_pipeline_result=\"ABORTED\",service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "ABORTED", + "refId": "C" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "id": 21, + "title": "Queue and executors", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "unit": "ms", "custom": { "lineWidth": 1, "fillOpacity": 10 } }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 7, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Queue wait time (p50 / p95)", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.50, sum(rate(ci_pipeline_run_queue_duration_milliseconds_bucket{service_name=~\"$service_name\"}[$__rate_interval])) by (le))", + "legendFormat": "p50 queue time", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.95, sum(rate(ci_pipeline_run_queue_duration_milliseconds_bucket{service_name=~\"$service_name\"}[$__rate_interval])) by (le))", + "legendFormat": "p95 queue time", + "refId": "B" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "unit": "short", "custom": { "lineWidth": 1, "fillOpacity": 10 }, "min": 0 }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "id": 8, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Executors — busy vs available", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(jenkins_executor_count{status=\"busy\",service_name=~\"$service_name\"}) by (service_name)", + "legendFormat": "busy", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(jenkins_executor_count{status=\"idle\",service_name=~\"$service_name\"}) by (service_name)", + "legendFormat": "idle", + "refId": "B" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "id": 22, + "title": "Jenkins controller JVM", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "unit": "bytes", "custom": { "lineWidth": 1, "fillOpacity": 10 }, "min": 0 }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "id": 9, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "JVM heap memory", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(runtime_jvm_memory_usage{type=\"heap\",service_name=~\"$service_name\"}) by (service_name)", + "legendFormat": "heap used", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(runtime_jvm_memory_limit{type=\"heap\",service_name=~\"$service_name\"}) by (service_name)", + "legendFormat": "heap limit", + "refId": "B" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "unit": "s", "custom": { "lineWidth": 1, "fillOpacity": 10 }, "min": 0 }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, + "id": 10, + "options": { "legend": { "calcs": ["mean", "p95"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "HTTP request duration (controller)", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket{service_name=~\"$service_name\"}[$__rate_interval])) by (le, http_route))", + "legendFormat": "p50 {{http_route}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket{service_name=~\"$service_name\"}[$__rate_interval])) by (le, http_route))", + "legendFormat": "p95 {{http_route}}", + "refId": "B" + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["jenkins", "opentelemetry", "ci"], + "templating": { + "list": [ + { + "current": {}, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "definition": "label_values(ci_pipeline_run_duration_milliseconds_count, service_name)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "service_name", + "options": [], + "query": { + "query": "label_values(ci_pipeline_run_duration_milliseconds_count, service_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query", + "label": "Jenkins instance" + } + ] + }, + "time": { "from": "now-3h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Jenkins overview (OpenTelemetry)", + "uid": "jenkins-otel-overview", + "version": 1 +}