From 12e8557e1e5604d2c321f61c77e8817f2756de0e Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Mon, 21 Apr 2025 23:00:28 +0200 Subject: [PATCH 01/11] Add OTel cicd.* metrics --- .../job/MonitoringRunListener.java | 54 ++++++- .../opentelemetry/semconv/CicdMetrics.java | 135 ++++++++++++++++++ 2 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index a0bb44432..a2a9745ac 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -27,12 +27,14 @@ import io.jenkins.plugins.opentelemetry.job.opentelemetry.OtelContextAwareAbstractRunListener; import io.jenkins.plugins.opentelemetry.job.runhandler.RunHandler; import io.jenkins.plugins.opentelemetry.queue.RemoteSpanAction; +import io.jenkins.plugins.opentelemetry.semconv.CicdMetrics; import io.jenkins.plugins.opentelemetry.semconv.ConfigurationKey; import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; import io.jenkins.plugins.opentelemetry.semconv.JenkinsMetrics; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; @@ -48,6 +50,7 @@ import io.opentelemetry.context.propagation.TextMapGetter; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.semconv.ExceptionAttributes; +import io.opentelemetry.semconv.incubating.CicdIncubatingAttributes; import jenkins.YesNoMaybe; import jenkins.model.Jenkins; import org.jenkinsci.plugins.workflow.job.WorkflowRun; @@ -72,14 +75,13 @@ import java.util.stream.Collectors; import static com.google.common.base.Verify.verifyNotNull; -import static java.util.Arrays.asList; -import static java.util.Collections.unmodifiableList; /** * TODO support reconfiguration */ @Extension(dynamicLoadable = YesNoMaybe.YES, optional = true) public class MonitoringRunListener extends OtelContextAwareAbstractRunListener implements OpenTelemetryLifecycleListener { + static final String PIPELINE_NAME_OTHER = "#other#"; static final Pattern MATCH_ANYTHING = Pattern.compile(".*"); static final Pattern MATCH_NOTHING = Pattern.compile("$^"); @@ -87,13 +89,16 @@ public class MonitoringRunListener extends OtelContextAwareAbstractRunListener i // TODO support configurability of these histogram buckets. Note that the conversion from a string to a list of // doubles will require boilerplate so we are interested in getting user feedback before implementing this. static final List DURATION_SECONDS_BUCKETS = - unmodifiableList( - asList(1D, 2D, 4D, 8D, 16D, 32D, 64D, 128D, 256D, 512D, 1024D, 2048D, 4096D, 8192D)); + List.of(1D, 2D, 4D, 8D, 16D, 32D, 64D, 128D, 256D, 512D, 1024D, 2048D, 4096D, 8192D); protected static final Logger LOGGER = Logger.getLogger(MonitoringRunListener.class.getName()); private AtomicInteger activeRunGauge; private List causeHandlers; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private DoubleHistogram runDurationHistogram; private LongCounter runLaunchedCounter; private LongCounter runStartedCounter; @@ -107,6 +112,9 @@ public class MonitoringRunListener extends OtelContextAwareAbstractRunListener i @VisibleForTesting Pattern runDurationHistogramDenyList; + private DoubleHistogram cicdPipelineRunDurationHistogram; + private LongUpDownCounter cicdPipelineRunActiveCounter; + @PostConstruct public void postConstruct() { LOGGER.log(Level.FINE, () -> "Start monitoring Jenkins build executions..."); @@ -135,6 +143,9 @@ public void postConstruct() { runDurationHistogramAllowList = MATCH_ANYTHING; // allow all runDurationHistogramDenyList = MATCH_NOTHING; // deny nothing + cicdPipelineRunDurationHistogram = CicdMetrics.newCiCdPipelineRunDurationHistogram(meter); + cicdPipelineRunActiveCounter = CicdMetrics.newCiCdPipelineRunActiveCounter(meter); + meter.gaugeBuilder(JenkinsMetrics.CI_PIPELINE_RUN_ACTIVE) .ofLongs() .setDescription("Gauge of active jobs") @@ -213,6 +224,10 @@ public void _onInitialize(@NonNull Run run) { LOGGER.log(Level.FINE, () -> run.getFullDisplayName() + " - onInitialize"); activeRunGauge.incrementAndGet(); + cicdPipelineRunActiveCounter.add(1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.PENDING + )); RunHandler runHandler = getRunHandlers().stream().filter(rh -> rh.canCreateSpanBuilder(run)).findFirst() .orElseThrow((Supplier) () -> new IllegalStateException("No RunHandler found for run " + run.getClass() + " - " + run)); @@ -354,6 +369,14 @@ public String get(@Nullable Map carrier, String key) { @Override public void _onStarted(@NonNull Run run, @NonNull TaskListener listener) { + cicdPipelineRunActiveCounter.add(-1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.PENDING + )); + cicdPipelineRunActiveCounter.add(1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.EXECUTING + )); try (Scope parentScope = endPipelinePhaseSpan(run)) { Span runSpan = getTracer().spanBuilder(ExtendedJenkinsAttributes.JENKINS_JOB_SPAN_PHASE_RUN_NAME).setParent(Context.current()).startSpan(); LOGGER.log(Level.FINE, () -> run.getFullDisplayName() + " - begin " + OtelUtils.toDebugString(runSpan)); @@ -366,6 +389,15 @@ public void _onStarted(@NonNull Run run, @NonNull TaskListener listener) { @Override public void _onCompleted(@NonNull Run run, @NonNull TaskListener listener) { + cicdPipelineRunActiveCounter.add(-1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.EXECUTING + )); + cicdPipelineRunActiveCounter.add(1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.FINALIZING + )); + try (Scope ignoredParentScope = endPipelinePhaseSpan(run)) { Span finalizeSpan = getTracer().spanBuilder(ExtendedJenkinsAttributes.JENKINS_JOB_SPAN_PHASE_FINALIZE_NAME).setParent(Context.current()).startSpan(); LOGGER.log(Level.FINE, () -> run.getFullDisplayName() + " - begin " + OtelUtils.toDebugString(finalizeSpan)); @@ -454,13 +486,25 @@ public void _onFinalized(@NonNull Run run) { runDurationHistogramAllowList.matcher(jobFullName).matches() && !runDurationHistogramDenyList.matcher(jobFullName).matches() ? - jobFullName : "#other#"; + jobFullName : PIPELINE_NAME_OTHER; runDurationHistogram.record( TimeUnit.SECONDS.convert(run.getDuration(), TimeUnit.MILLISECONDS), Attributes.of( ExtendedJenkinsAttributes.CI_PIPELINE_ID, pipelineId, ExtendedJenkinsAttributes.CI_PIPELINE_RUN_RESULT, result.toString()) ); + + // FIXME CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE & ErrorAttributes.ERROR_TYPE + cicdPipelineRunDurationHistogram.record( + TimeUnit.SECONDS.convert(run.getDuration(), TimeUnit.MILLISECONDS), + Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, pipelineId, + CicdIncubatingAttributes.CICD_PIPELINE_RESULT, CicdMetrics.fromJenkinsResultToOtelCicdPipelineResult(result))); + + cicdPipelineRunActiveCounter.add(-1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.FINALIZING + )); } finally { activeRunGauge.decrementAndGet(); } diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java new file mode 100644 index 000000000..09a1795cf --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java @@ -0,0 +1,135 @@ +package io.jenkins.plugins.opentelemetry.semconv; + +import hudson.model.Result; +import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogramBuilder; +import io.opentelemetry.api.incubator.metrics.ExtendedDoubleUpDownCounterBuilder; +import io.opentelemetry.api.incubator.metrics.ExtendedLongCounterBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.semconv.ErrorAttributes; +import io.opentelemetry.semconv.incubating.CicdIncubatingAttributes; + +import java.util.List; + +public class CicdMetrics { + // FIXME WHAT ARE THE SPECS FOR THESE BUCKETS? + // TODO support configurability of these histogram buckets. Note that the conversion from a string to a list of + // doubles will require boilerplate so we are interested in getting user feedback before implementing this. + static final List DURATION_SECONDS_BUCKETS = + List.of(1D, 2D, 4D, 8D, 16D, 32D, 64D, 128D, 256D, 512D, 1024D, 2048D, 4096D, 8192D); + + public static DoubleHistogram newCiCdPipelineRunDurationHistogram(Meter meter) { + DoubleHistogramBuilder cicdPipelineRunDurationHistogramBuilder = + meter + .histogramBuilder("cicd.pipeline.run.duration") + .setUnit("s") + .setDescription("Duration of a pipeline run grouped by pipeline, state and result.") + .setExplicitBucketBoundariesAdvice(DURATION_SECONDS_BUCKETS); + if (cicdPipelineRunDurationHistogramBuilder instanceof ExtendedDoubleHistogramBuilder extendedDoubleHistogramBuilder) { + extendedDoubleHistogramBuilder + .setAttributesAdvice( + List.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, + CicdIncubatingAttributes.CICD_PIPELINE_RESULT, + ErrorAttributes.ERROR_TYPE)); + } + + return cicdPipelineRunDurationHistogramBuilder.build(); + } + + public static LongUpDownCounter newCiCdPipelineRunActiveCounter(Meter meter) { + LongUpDownCounterBuilder cicdPipelineRunActiveCounterBuilder = + meter + .upDownCounterBuilder("cicd.pipeline.run.active") + .setUnit("{run}") + .setDescription("Number of active pipeline runs grouped by pipeline and state."); + if (cicdPipelineRunActiveCounterBuilder instanceof ExtendedDoubleUpDownCounterBuilder pipelineRunActiveCounterBuilder) { + pipelineRunActiveCounterBuilder + .setAttributesAdvice( + List.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE)); + } + + return cicdPipelineRunActiveCounterBuilder.build(); + } + + public static LongUpDownCounter newCiCdWorkerCount(Meter meter) { + LongUpDownCounterBuilder cicdWorkerCountBuilder = + meter + .upDownCounterBuilder("cicd.worker.count") + .setUnit("{worker}") + .setDescription("The number of workers on the CICD system by state."); + if (cicdWorkerCountBuilder instanceof ExtendedDoubleUpDownCounterBuilder cicdWorkerCountExtendedBuilder) { + cicdWorkerCountExtendedBuilder + .setAttributesAdvice( + List.of( + CicdIncubatingAttributes.CICD_WORKER_STATE)); + } + + return cicdWorkerCountBuilder.build(); + } + + public static LongCounter newCiCdPipelineRunErrors(Meter meter) { + LongCounterBuilder cicdPipelineRunErrorsBuilder = + meter + .counterBuilder("cicd.pipeline.run.errors") + .setUnit("{error}") + .setDescription("Number of errors in a pipeline run grouped by pipeline and error type."); + if (cicdPipelineRunErrorsBuilder instanceof ExtendedLongCounterBuilder cicdPipelineRunErrorsExtendedBuilder) { + cicdPipelineRunErrorsExtendedBuilder + .setAttributesAdvice( + List.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, + ErrorAttributes.ERROR_TYPE)); + } + + return cicdPipelineRunErrorsBuilder.build(); + } + + public static LongCounter newCiCdSystemErrors(Meter meter) { + LongCounterBuilder cicdSystemErrorsBuilder = + meter + .counterBuilder("cicd.system.errors") + .setUnit("{error}") + .setDescription("Number of errors in the CICD system grouped by component and error type."); + if (cicdSystemErrorsBuilder instanceof ExtendedLongCounterBuilder cicdSystemErrorsExtendedBuilder) { + cicdSystemErrorsExtendedBuilder + .setAttributesAdvice( + List.of( + CicdIncubatingAttributes.CICD_SYSTEM_COMPONENT, + ErrorAttributes.ERROR_TYPE)); + } + return cicdSystemErrorsBuilder.build(); + } + + /** + * Convert a Jenkins {@link Result} to a an OpenTelemetry + * {@link CicdIncubatingAttributes#CICD_PIPELINE_RESULT} according to the + * OpenTelemetry + * CICD Semantic Conventions + */ + public static String fromJenkinsResultToOtelCicdPipelineResult(Result result) { + if (result == null) { + return "#null#"; + } + if (result.equals(Result.ABORTED)) { + return CicdIncubatingAttributes.CicdPipelineResultIncubatingValues.CANCELLATION; + } else if (result.equals(Result.FAILURE)) { + return CicdIncubatingAttributes.CicdPipelineResultIncubatingValues.FAILURE; + } else if (result.equals(Result.NOT_BUILT)) { + return "not_built"; + } else if (result.equals(Result.SUCCESS)) { + return CicdIncubatingAttributes.CicdPipelineResultIncubatingValues.SUCCESS; + } else if (result.equals(Result.UNSTABLE)) { + return "unstable"; + } + return result.toString().toLowerCase(); + } +} From f98eef0811b509d458868241d971daaae33c88b8 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Tue, 22 Apr 2025 08:22:00 +0200 Subject: [PATCH 02/11] Add OTel cicd.* metrics and attributes --- .../job/MonitoringRunListener.java | 4 +++ .../semconv/ExtendedJenkinsAttributes.java | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index a2a9745ac..bba4e282e 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -93,6 +93,10 @@ public class MonitoringRunListener extends OtelContextAwareAbstractRunListener i protected static final Logger LOGGER = Logger.getLogger(MonitoringRunListener.class.getName()); + /** + * @deprecated use {@link #cicdPipelineRunActiveCounter} + */ + @Deprecated private AtomicInteger activeRunGauge; private List causeHandlers; /** diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java index de2d4f472..226f0075c 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java @@ -24,8 +24,11 @@ public final class ExtendedJenkinsAttributes extends JenkinsAttributes { public static final AttributeKey CI_PIPELINE_TYPE = AttributeKey.stringKey("ci.pipeline.type"); public static final AttributeKey CI_PIPELINE_MULTIBRANCH_TYPE = AttributeKey.stringKey("ci.pipeline.multibranch.type"); /** + * FIXME should we use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_NAME} for the jenkins job name or display name? * @see Job#getFullName() + * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_NAME} */ + @Deprecated public static final AttributeKey CI_PIPELINE_ID = AttributeKey.stringKey("ci.pipeline.id"); public static final AttributeKey CI_PIPELINE_NAME = AttributeKey.stringKey("ci.pipeline.name"); public static final AttributeKey CI_PIPELINE_TEMPLATE_ID = AttributeKey.stringKey("ci.pipeline.template.id"); @@ -43,21 +46,40 @@ public final class ExtendedJenkinsAttributes extends JenkinsAttributes { public static final AttributeKey CI_PIPELINE_RUN_COMPLETED = AttributeKey.booleanKey("ci.pipeline.run.completed"); public static final AttributeKey CI_PIPELINE_RUN_DURATION_MILLIS = AttributeKey.longKey("ci.pipeline.run.durationMillis"); public static final AttributeKey CI_PIPELINE_RUN_DESCRIPTION = AttributeKey.stringKey("ci.pipeline.run.description"); + /** + * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_RUN_ID} + */ + @Deprecated public static final AttributeKey CI_PIPELINE_RUN_NUMBER = AttributeKey.longKey("ci.pipeline.run.number"); public static final AttributeKey> CI_PIPELINE_RUN_PARAMETER_IS_SENSITIVE = AttributeKey.booleanArrayKey("ci.pipeline.parameter.sensitive"); public static final AttributeKey> CI_PIPELINE_RUN_PARAMETER_NAME = AttributeKey.stringArrayKey("ci.pipeline.parameter.name"); public static final AttributeKey> CI_PIPELINE_RUN_PARAMETER_VALUE = AttributeKey.stringArrayKey("ci.pipeline.parameter.value"); /** * @see Run#getResult() + * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_RESULT} */ + @Deprecated public static final AttributeKey CI_PIPELINE_RUN_RESULT = AttributeKey.stringKey("ci.pipeline.run.result"); + /** + * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_RUN_URL_FULL} + */ + @Deprecated public static final AttributeKey CI_PIPELINE_RUN_URL = AttributeKey.stringKey("ci.pipeline.run.url"); public static final AttributeKey CI_PIPELINE_RUN_USER = AttributeKey.stringKey("ci.pipeline.run.user"); public static final AttributeKey> CI_PIPELINE_RUN_AXIS_NAMES = AttributeKey.stringArrayKey("ci.pipeline.axis.names"); public static final AttributeKey> CI_PIPELINE_RUN_AXIS_VALUES = AttributeKey.stringArrayKey("ci.pipeline.axis.values"); + /** + * FIXME introduce {@link io.opentelemetry.semconv.incubating.VcsIncubatingAttributes#VCS_REPOSITORY_URL_FULL} and {@link io.opentelemetry.semconv.incubating.VcsIncubatingAttributes#VCS_OWNER_NAME} + * @deprecated use {@link io.opentelemetry.semconv.incubating.VcsIncubatingAttributes#VCS_REPOSITORY_NAME} + */ + @Deprecated public static final AttributeKey GIT_REPOSITORY = AttributeKey.stringKey("git.repository"); + /** + * @deprecated use {@link io.opentelemetry.semconv.incubating.VcsIncubatingAttributes#VCS_REF_BASE_NAME} and {@link io.opentelemetry.semconv.incubating.VcsIncubatingAttributes#VCS_REF_BASE_TYPE} + */ + @Deprecated public static final AttributeKey GIT_BRANCH = AttributeKey.stringKey("git.branch"); public static final AttributeKey GIT_USERNAME = AttributeKey.stringKey("git.username"); public static final AttributeKey GIT_CLONE_DEPTH = AttributeKey.longKey("git.clone.depth"); @@ -65,11 +87,15 @@ public final class ExtendedJenkinsAttributes extends JenkinsAttributes { /** * @see StepDescriptor#getDisplayName() + * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_TASK_NAME} */ + @Deprecated public static final AttributeKey JENKINS_STEP_NAME = AttributeKey.stringKey("jenkins.pipeline.step.name"); /** * @see StepDescriptor#getFunctionName() + * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_TASK_TYPE} */ + @Deprecated public static final AttributeKey JENKINS_STEP_TYPE = AttributeKey.stringKey("jenkins.pipeline.step.type"); /** * @see org.jenkinsci.plugins.workflow.graph.FlowNode#getId() From 16363ae0177cfbbb8df182fe0c74b215097cacd5 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Wed, 23 Apr 2025 22:49:21 +0200 Subject: [PATCH 03/11] Add OTel cicd.* metrics and attributes --- .../JenkinsExecutorMonitoringInitializer.java | 13 +++++++- .../job/MonitoringRunListener.java | 30 +++++++++++++++++-- .../opentelemetry/semconv/CicdMetrics.java | 12 +++++--- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java b/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java index 9c2712301..d27a3b741 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java @@ -13,10 +13,12 @@ import hudson.model.Node; import io.jenkins.plugins.opentelemetry.JenkinsControllerOpenTelemetry; import io.jenkins.plugins.opentelemetry.api.OpenTelemetryLifecycleListener; +import io.jenkins.plugins.opentelemetry.semconv.CicdMetrics; import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import io.opentelemetry.semconv.incubating.CicdIncubatingAttributes; import jenkins.YesNoMaybe; import jenkins.model.Jenkins; @@ -53,6 +55,7 @@ public void postConstruct() { final ObservableLongMeasurement totalExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_TOTAL).setUnit("${executors}").setDescription("Total executors").ofLongs().buildObserver(); final ObservableLongMeasurement nodes = meter.gaugeBuilder(JENKINS_NODE).setUnit("${nodes}").setDescription("Nodes").ofLongs().buildObserver(); final ObservableLongMeasurement executors = meter.gaugeBuilder(JENKINS_EXECUTOR).setUnit("${executors}").setDescription("Per label executors").ofLongs().buildObserver(); + ObservableLongMeasurement cicdWorkers = CicdMetrics.newCiCdWorkerCounter(meter); // TODO the metrics below should be deprecated in favor of // * `jenkins.executor` metric with the `status` and `label`attributes @@ -78,6 +81,7 @@ public void postConstruct() { // TOTAL EXECUTORS AtomicInteger totalExecutorsIdle = new AtomicInteger(); AtomicInteger totalExecutorsBusy = new AtomicInteger(); + AtomicInteger totalExecutorsOffline = new AtomicInteger(); AtomicInteger nodeOnline = new AtomicInteger(); AtomicInteger nodeOffline = new AtomicInteger(); @@ -105,12 +109,19 @@ public void postConstruct() { } }); } else { + totalExecutorsOffline.addAndGet(node.countExecutors()); nodeOffline.incrementAndGet(); } }); totalExecutors.record(totalExecutorsBusy.get(), Attributes.of(STATUS, "busy")); totalExecutors.record(totalExecutorsIdle.get(), Attributes.of(STATUS, "idle")); + cicdWorkers.record(totalExecutorsBusy.get(), + Attributes.of(CicdIncubatingAttributes.CICD_WORKER_STATE, CicdIncubatingAttributes.CicdWorkerStateIncubatingValues.BUSY)); + cicdWorkers.record(totalExecutorsIdle.get(), + Attributes.of(CicdIncubatingAttributes.CICD_WORKER_STATE, CicdIncubatingAttributes.CicdWorkerStateIncubatingValues.AVAILABLE)); + cicdWorkers.record(totalExecutorsOffline.get(), + Attributes.of(CicdIncubatingAttributes.CICD_WORKER_STATE, CicdIncubatingAttributes.CicdWorkerStateIncubatingValues.OFFLINE)); nodes.record(nodeOnline.get(), Attributes.of(STATUS, "online")); nodes.record(nodeOffline.get(), Attributes.of(STATUS, "offline")); @@ -133,6 +144,6 @@ public void postConstruct() { definedExecutors.record(loadStatisticsSnapshot.getDefinedExecutors(), attributes); connectingExecutors.record(loadStatisticsSnapshot.getConnectingExecutors(), attributes); }); - }, availableExecutors, busyExecutors, idleExecutors, onlineExecutors, connectingExecutors, definedExecutors, totalExecutors, executors, nodes, queueLength); + }, availableExecutors, busyExecutors, idleExecutors, onlineExecutors, connectingExecutors, definedExecutors, totalExecutors, executors, nodes, queueLength, cicdWorkers); } } diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index bba4e282e..8eb97cd0b 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -32,7 +32,9 @@ import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; import io.jenkins.plugins.opentelemetry.semconv.JenkinsMetrics; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogramBuilder; import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.Meter; @@ -49,6 +51,7 @@ import io.opentelemetry.context.Scope; import io.opentelemetry.context.propagation.TextMapGetter; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.semconv.ErrorAttributes; import io.opentelemetry.semconv.ExceptionAttributes; import io.opentelemetry.semconv.incubating.CicdIncubatingAttributes; import jenkins.YesNoMaybe; @@ -118,6 +121,8 @@ public class MonitoringRunListener extends OtelContextAwareAbstractRunListener i private DoubleHistogram cicdPipelineRunDurationHistogram; private LongUpDownCounter cicdPipelineRunActiveCounter; + private LongCounter cicdPipelineRunErrorsCounter; + private LongCounter cicdSystemErrorsCounter; @PostConstruct public void postConstruct() { @@ -140,15 +145,26 @@ public void postConstruct() { // METRICS activeRunGauge = new AtomicInteger(); - runDurationHistogram = meter.histogramBuilder(JenkinsMetrics.CI_PIPELINE_RUN_DURATION) + DoubleHistogramBuilder runDurationHistogramBuilder = meter.histogramBuilder(JenkinsMetrics.CI_PIPELINE_RUN_DURATION) .setUnit("s") - .setExplicitBucketBoundariesAdvice(DURATION_SECONDS_BUCKETS) - .build(); + .setExplicitBucketBoundariesAdvice(DURATION_SECONDS_BUCKETS); + if (runDurationHistogramBuilder instanceof ExtendedDoubleHistogramBuilder extendedBuilder) { + extendedBuilder.setAttributesAdvice(List.of( + ExtendedJenkinsAttributes.CI_PIPELINE_ID, + ExtendedJenkinsAttributes.CI_PIPELINE_RUN_RESULT + )); + } + runDurationHistogram = runDurationHistogramBuilder.build(); runDurationHistogramAllowList = MATCH_ANYTHING; // allow all runDurationHistogramDenyList = MATCH_NOTHING; // deny nothing cicdPipelineRunDurationHistogram = CicdMetrics.newCiCdPipelineRunDurationHistogram(meter); cicdPipelineRunActiveCounter = CicdMetrics.newCiCdPipelineRunActiveCounter(meter); + cicdPipelineRunErrorsCounter = CicdMetrics.newCiCdPipelineRunErrorsCounter(meter); + /** + * FIXME when to qualify a build failure as a cicd system error? + */ + cicdSystemErrorsCounter = CicdMetrics.newCiCdSystemErrorsCounter(meter); meter.gaugeBuilder(JenkinsMetrics.CI_PIPELINE_RUN_ACTIVE) .ofLongs() @@ -451,8 +467,16 @@ public void _onFinalized(@NonNull Run run) { parentSpan.setAttribute(ExceptionAttributes.EXCEPTION_TYPE, "PIPELINE_" + runResult); parentSpan.setAttribute(ExceptionAttributes.EXCEPTION_MESSAGE, "PIPELINE_" + runResult); parentSpan.setStatus(StatusCode.ERROR, runResult.toString()); + cicdPipelineRunErrorsCounter.add(1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + ErrorAttributes.ERROR_TYPE, runResult.toString() + )); } else if (Result.ABORTED.equals(runResult) || Result.NOT_BUILT.equals(runResult)) { parentSpan.setStatus(StatusCode.UNSET, runResult.toString()); + cicdPipelineRunErrorsCounter.add(1, Attributes.of( + CicdIncubatingAttributes.CICD_PIPELINE_NAME, PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + ErrorAttributes.ERROR_TYPE, runResult.toString() + )); } } // NODE diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java index 09a1795cf..3088b9260 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/CicdMetrics.java @@ -11,6 +11,7 @@ import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; import io.opentelemetry.semconv.ErrorAttributes; import io.opentelemetry.semconv.incubating.CicdIncubatingAttributes; @@ -60,7 +61,10 @@ public static LongUpDownCounter newCiCdPipelineRunActiveCounter(Meter meter) { return cicdPipelineRunActiveCounterBuilder.build(); } - public static LongUpDownCounter newCiCdWorkerCount(Meter meter) { + /** + * FIXME shouldn't it be a gauge rather than an upDownCounter? + */ + public static ObservableLongMeasurement newCiCdWorkerCounter(Meter meter) { LongUpDownCounterBuilder cicdWorkerCountBuilder = meter .upDownCounterBuilder("cicd.worker.count") @@ -73,10 +77,10 @@ public static LongUpDownCounter newCiCdWorkerCount(Meter meter) { CicdIncubatingAttributes.CICD_WORKER_STATE)); } - return cicdWorkerCountBuilder.build(); + return cicdWorkerCountBuilder.buildObserver(); } - public static LongCounter newCiCdPipelineRunErrors(Meter meter) { + public static LongCounter newCiCdPipelineRunErrorsCounter(Meter meter) { LongCounterBuilder cicdPipelineRunErrorsBuilder = meter .counterBuilder("cicd.pipeline.run.errors") @@ -93,7 +97,7 @@ public static LongCounter newCiCdPipelineRunErrors(Meter meter) { return cicdPipelineRunErrorsBuilder.build(); } - public static LongCounter newCiCdSystemErrors(Meter meter) { + public static LongCounter newCiCdSystemErrorsCounter(Meter meter) { LongCounterBuilder cicdSystemErrorsBuilder = meter .counterBuilder("cicd.system.errors") From c7f24bbcc05c9af427e83d61f22001367b5e394c Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Wed, 23 Apr 2025 23:26:42 +0200 Subject: [PATCH 04/11] Add OTel cicd.* metrics and attributes --- .../jenkins/plugins/opentelemetry/job/MonitoringRunListener.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index 8eb97cd0b..067d2cb68 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -527,6 +527,7 @@ public void _onFinalized(@NonNull Run run) { TimeUnit.SECONDS.convert(run.getDuration(), TimeUnit.MILLISECONDS), Attributes.of( CicdIncubatingAttributes.CICD_PIPELINE_NAME, pipelineId, + CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.FINALIZING, CicdIncubatingAttributes.CICD_PIPELINE_RESULT, CicdMetrics.fromJenkinsResultToOtelCicdPipelineResult(result))); cicdPipelineRunActiveCounter.add(-1, Attributes.of( From 5d037ff5c498b5637cccb603949a85f37ef08ed9 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Thu, 24 Apr 2025 18:50:44 +0200 Subject: [PATCH 05/11] Add OTel cicd.* metrics and attributes --- .../job/MonitoringRunListener.java | 33 +++++++----- .../job/runhandler/DefaultRunHandler.java | 16 +++--- .../job/runhandler/JobDslRunHandler.java | 51 +++++++++---------- .../job/runhandler/MatrixRunHandler.java | 30 ++++++----- .../job/runhandler/RunHandler.java | 13 +++-- .../semconv/ExtendedJenkinsAttributes.java | 6 +-- 6 files changed, 82 insertions(+), 67 deletions(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index 067d2cb68..80d94655b 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -80,7 +80,7 @@ import static com.google.common.base.Verify.verifyNotNull; /** - * TODO support reconfiguration + * TODO support reconfiguration. allow & deny lists are NOT reconfigurable at runtime for the moment. */ @Extension(dynamicLoadable = YesNoMaybe.YES, optional = true) public class MonitoringRunListener extends OtelContextAwareAbstractRunListener implements OpenTelemetryLifecycleListener { @@ -249,23 +249,30 @@ public void _onInitialize(@NonNull Run run) { CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.PENDING )); - RunHandler runHandler = getRunHandlers().stream().filter(rh -> rh.canCreateSpanBuilder(run)).findFirst() + RunHandler runHandler = getRunHandlers().stream().filter(rh -> rh.matches(run)).findFirst() .orElseThrow((Supplier) () -> new IllegalStateException("No RunHandler found for run " + run.getClass() + " - " + run)); - SpanBuilder rootSpanBuilder = runHandler.createSpanBuilder(run, getTracer()); - + String pipelineShortName = runHandler.getPipelineShortName(run); + SpanBuilder rootSpanBuilder = getTracer().spanBuilder(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_ROOT_SPAN_NAME_PREFIX + pipelineShortName); + runHandler.enrichPipelineRunSpan(run, rootSpanBuilder); rootSpanBuilder.setSpanKind(SpanKind.SERVER); - String runUrl = Objects.toString(Jenkins.get().getRootUrl(), "") + run.getUrl(); + String runUrl = Optional.ofNullable(Jenkins.get().getRootUrl()).orElse("") + run.getUrl(); // TODO move this to a pluggable span enrichment API with implementations for different observability backends rootSpanBuilder .setAttribute(ExtendedJenkinsAttributes.ELASTIC_TRANSACTION_TYPE, "job"); rootSpanBuilder + .setAttribute(CicdIncubatingAttributes.CICD_PIPELINE_NAME, pipelineShortName) + .setAttribute(CicdIncubatingAttributes.CICD_PIPELINE_RUN_URL_FULL, runUrl) + .setAttribute(CicdIncubatingAttributes.CICD_PIPELINE_RUN_ID, String.valueOf(run.getNumber())) + .setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_ID, run.getParent().getFullName()) .setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_NAME, run.getParent().getFullDisplayName()) + .setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_TYPE, OtelUtils.getProjectType(run)) + .setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_URL, runUrl) .setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_NUMBER, (long) run.getNumber()) - .setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_TYPE, OtelUtils.getProjectType(run)); + ; // CULPRITS Set culpritIds; @@ -338,8 +345,7 @@ public void _onInitialize(@NonNull Run run) { // unclear why this could happen. Maybe during the installation of the plugin if the plugin is // installed while a parent job triggers a downstream job w3cTraceContext = Collections.emptyMap(); - } else if (upstreamCause instanceof BuildUpstreamCause) { - BuildUpstreamCause buildUpstreamCause = (BuildUpstreamCause) cause; + } else if (upstreamCause instanceof BuildUpstreamCause buildUpstreamCause) { String upstreamNodeId = buildUpstreamCause.getNodeId(); w3cTraceContext = monitoringAction.getW3cTraceContext(upstreamNodeId); } else { @@ -509,12 +515,15 @@ public void _onFinalized(@NonNull Run run) { this.runAbortedCounter.add(1); } - String jobFullName = run.getParent().getFullName(); + // TODO perf optimization, reuse resolution done in `#_onInitialize(run)` + String pipelineShortName = getRunHandlers().stream().filter(rh -> rh.matches(run)).findFirst() + .orElseThrow((Supplier) () -> new IllegalStateException("No RunHandler found for run " + run.getClass() + " - " + run)) + .getPipelineShortName(run); String pipelineId = - runDurationHistogramAllowList.matcher(jobFullName).matches() + runDurationHistogramAllowList.matcher(pipelineShortName).matches() && - !runDurationHistogramDenyList.matcher(jobFullName).matches() ? - jobFullName : PIPELINE_NAME_OTHER; + !runDurationHistogramDenyList.matcher(pipelineShortName).matches() ? + pipelineShortName : PIPELINE_NAME_OTHER; runDurationHistogram.record( TimeUnit.SECONDS.convert(run.getDuration(), TimeUnit.MILLISECONDS), Attributes.of( diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/DefaultRunHandler.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/DefaultRunHandler.java index e4a640bbd..23d6ac877 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/DefaultRunHandler.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/DefaultRunHandler.java @@ -22,8 +22,8 @@ import java.util.Locale; /** - * Use same root span name for all pull change request pipelines (pull request, merge request) - * Use different span names for different branches. + * Use same short pipeline name for all change request pipelines (pull request, merge request) + * Use different short pipeline name for different branches. */ @Extension public class DefaultRunHandler implements RunHandler { @@ -33,21 +33,21 @@ public class DefaultRunHandler implements RunHandler { "-" + ChangeRequestCheckoutStrategy.MERGE.name().toLowerCase(Locale.ENGLISH))); @Override - public boolean canCreateSpanBuilder(@NonNull Run run) { + public boolean matches(@NonNull Run run) { return true; } @NonNull @Override - public SpanBuilder createSpanBuilder(@NonNull Run run, @NonNull Tracer tracer) { + public String getPipelineShortName(@NonNull Run run) { SCMHead head = SCMHead.HeadByItem.findHead(run.getParent()); - String spanName; + String pipelineShortName; if (head instanceof ChangeRequestSCMHead) { - spanName = getChangeRequestRootSpanName(run.getParent().getFullName()); + pipelineShortName = getChangeRequestRootSpanName(run.getParent().getFullName()); } else { - spanName = run.getParent().getFullName(); + pipelineShortName = run.getParent().getFullName(); } - return tracer.spanBuilder(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_ROOT_SPAN_NAME_PREFIX + spanName); + return pipelineShortName; } @VisibleForTesting diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/JobDslRunHandler.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/JobDslRunHandler.java index 1c372b8e3..bcb9b25a6 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/JobDslRunHandler.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/JobDslRunHandler.java @@ -6,6 +6,7 @@ package io.jenkins.plugins.opentelemetry.job.runhandler; +import edu.umd.cs.findbugs.annotations.NonNull; import hudson.Extension; import hudson.model.Action; import hudson.model.Item; @@ -13,15 +14,14 @@ import hudson.model.Run; import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import javaposse.jobdsl.plugin.actions.SeedJobAction; import javaposse.jobdsl.plugin.actions.SeedJobTransientActionFactory; import jenkins.YesNoMaybe; -import edu.umd.cs.findbugs.annotations.NonNull; import javax.inject.Inject; import java.util.Collection; +import java.util.Optional; @Extension(optional = true, dynamicLoadable = YesNoMaybe.YES) @@ -37,16 +37,31 @@ public JobDslRunHandler() throws ClassNotFoundException { } @Override - public boolean canCreateSpanBuilder(@NonNull Run run) { + public boolean matches(@NonNull Run run) { Job job = run.getParent(); // perf optimization: directly lookup up in the SeedJobTransientActionFactory over `job.getAction(SeedJobAction.class)` Collection actions = seedJobTransientActionFactory.createFor(job); return !actions.isEmpty(); } + @NonNull @Override - public SpanBuilder createSpanBuilder(@NonNull Run run, @NonNull Tracer tracer) { + public String getPipelineShortName(@NonNull Run run) { + Job job = run.getParent(); + // perf optimization: directly lookup up in the SeedJobTransientActionFactory over `job.getAction(SeedJobAction.class)` + Collection actions = seedJobTransientActionFactory.createFor(job); + + SeedJobAction seedJobAction = (SeedJobAction) actions.stream().filter(action -> action instanceof SeedJobAction).findFirst().orElseThrow(IllegalStateException::new); + + // TODO understand the difference between seedJobAction.getTemplateJob() and seedJobAction.getSeedJob() + Optional seedJob = Optional.ofNullable(seedJobAction.getSeedJob()); + + return collapseJobName? job.getFullName() : seedJob.map(Item::getFullName).map(fn -> "Job from seed '" + fn + "'").orElse(job.getFullName()); + } + + @Override + public void enrichPipelineRunSpan(@NonNull Run run, @NonNull SpanBuilder spanBuilder) { Job job = run.getParent(); // perf optimization: directly lookup up in the SeedJobTransientActionFactory over `job.getAction(SeedJobAction.class)` Collection actions = seedJobTransientActionFactory.createFor(job); @@ -54,29 +69,11 @@ public SpanBuilder createSpanBuilder(@NonNull Run run, @NonNull Tracer tra SeedJobAction seedJobAction = (SeedJobAction) actions.stream().filter(action -> action instanceof SeedJobAction).findFirst().orElseThrow(IllegalStateException::new); // TODO understand the difference between seedJobAction.getTemplateJob() and seedJobAction.getSeedJob() - Item seedJob = seedJobAction.getSeedJob(); - - String templateFullName; - String templateUrl; - String spanName; - if (seedJob == null) { - templateFullName = null; - templateUrl = null; - spanName = job.getFullName(); - } else { - templateFullName = seedJob.getFullName(); - templateUrl = seedJob.getUrl(); - spanName = collapseJobName ? "Job from seed '" + templateFullName + "'" : job.getFullName(); - } - - SpanBuilder spanBuilder = tracer.spanBuilder(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_ROOT_SPAN_NAME_PREFIX + spanName); - if (templateFullName != null) { - spanBuilder.setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_TEMPLATE_ID, templateFullName); - } - if (templateUrl != null) { - spanBuilder.setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_TEMPLATE_URL, templateUrl); - } - return spanBuilder; + Optional seedJob = Optional.ofNullable(seedJobAction.getSeedJob()); + + seedJob.map(Item::getFullName).ifPresent(templateFullName -> spanBuilder.setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_TEMPLATE_ID, templateFullName)); + seedJob.map(Item::getUrl).ifPresent(templateUrl -> spanBuilder.setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_TEMPLATE_URL, templateUrl)); + } @Override diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/MatrixRunHandler.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/MatrixRunHandler.java index 504a19b08..3e52818d8 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/MatrixRunHandler.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/MatrixRunHandler.java @@ -5,6 +5,7 @@ package io.jenkins.plugins.opentelemetry.job.runhandler; +import edu.umd.cs.findbugs.annotations.NonNull; import hudson.Extension; import hudson.matrix.Combination; import hudson.matrix.MatrixBuild; @@ -14,11 +15,9 @@ import hudson.model.Run; import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import jenkins.YesNoMaybe; -import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; import java.util.List; @@ -33,20 +32,29 @@ public MatrixRunHandler() throws ClassNotFoundException { } @Override - public boolean canCreateSpanBuilder(@NonNull Run run) { + public boolean matches(@NonNull Run run) { return run instanceof MatrixRun || run instanceof MatrixBuild; } @NonNull @Override - public SpanBuilder createSpanBuilder(@NonNull Run run, @NonNull Tracer tracer) { + public String getPipelineShortName(@NonNull Run run) { if (run instanceof MatrixRun matrixRun) { MatrixConfiguration matrixConfiguration = matrixRun.getParent(); - MatrixProject matrixProject = matrixConfiguration.getParent(); - String spanName = expandJobName ? run.getParent().getFullName() : matrixProject.getFullName() + "/execution"; - SpanBuilder spanBuilder = tracer.spanBuilder(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_ROOT_SPAN_NAME_PREFIX + spanName); - Combination combination = matrixConfiguration.getCombination(); + return expandJobName ? run.getParent().getFullName() : matrixProject.getFullName() + "/execution"; + } else if (run instanceof MatrixBuild matrixBuild) { + return matrixBuild.getParent().getFullName(); + } else { + throw new IllegalStateException("Unsupported run type " + run); + } + } + + @Override + public void enrichPipelineRunSpan(@NonNull Run run, @NonNull SpanBuilder spanBuilder) { + if (run instanceof MatrixRun matrixRun) { + + Combination combination = matrixRun.getParent().getCombination(); List axisNames = new ArrayList<>(); List axisValues = new ArrayList<>(); @@ -56,12 +64,6 @@ public SpanBuilder createSpanBuilder(@NonNull Run run, @NonNull Tracer tra }); spanBuilder.setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_AXIS_NAMES, axisNames); spanBuilder.setAttribute(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_AXIS_VALUES, axisValues); - - return spanBuilder; - } else if (run instanceof MatrixBuild matrixBuild) { - return tracer.spanBuilder(ExtendedJenkinsAttributes.CI_PIPELINE_RUN_ROOT_SPAN_NAME_PREFIX + matrixBuild.getParent().getFullName()); - } else { - throw new IllegalStateException("Unsupported run type " + run); } } diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/RunHandler.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/RunHandler.java index 44355b9f5..343124964 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/RunHandler.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/runhandler/RunHandler.java @@ -7,7 +7,6 @@ import hudson.model.Run; import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import edu.umd.cs.findbugs.annotations.NonNull; @@ -17,10 +16,18 @@ public interface RunHandler extends Comparable { default void configure(ConfigProperties config) { } - boolean canCreateSpanBuilder(@NonNull Run run); + boolean matches(@NonNull Run run); + /** + * Low cardinality pipeline name that fits with + * {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_NAME}. + * High cardinality elements like the SCM pull request names of a Jenkins multibranch pipeline + * should be excluded from the pipeline short name. + */ @NonNull - SpanBuilder createSpanBuilder(@NonNull Run run, @NonNull Tracer tracer); + String getPipelineShortName(@NonNull Run run); + + default void enrichPipelineRunSpan(@NonNull Run run, @NonNull SpanBuilder spanBuilder){} /** * @return the ordinal of this handler to execute run handlers in predictable order. The smallest ordinal is executed first. diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java index 226f0075c..ae43deec0 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/semconv/ExtendedJenkinsAttributes.java @@ -24,11 +24,11 @@ public final class ExtendedJenkinsAttributes extends JenkinsAttributes { public static final AttributeKey CI_PIPELINE_TYPE = AttributeKey.stringKey("ci.pipeline.type"); public static final AttributeKey CI_PIPELINE_MULTIBRANCH_TYPE = AttributeKey.stringKey("ci.pipeline.multibranch.type"); /** - * FIXME should we use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_NAME} for the jenkins job name or display name? + * Full Jenkins job name, different from the low cardinality + * {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_NAME} that relies on + * {@link io.jenkins.plugins.opentelemetry.job.runhandler.RunHandler#getPipelineShortName(Run)} * @see Job#getFullName() - * @deprecated use {@link io.opentelemetry.semconv.incubating.CicdIncubatingAttributes#CICD_PIPELINE_NAME} */ - @Deprecated public static final AttributeKey CI_PIPELINE_ID = AttributeKey.stringKey("ci.pipeline.id"); public static final AttributeKey CI_PIPELINE_NAME = AttributeKey.stringKey("ci.pipeline.name"); public static final AttributeKey CI_PIPELINE_TEMPLATE_ID = AttributeKey.stringKey("ci.pipeline.template.id"); From 5499fbf67eb25a4732740930d41d92929ee0a913 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Sat, 26 Apr 2025 19:27:23 +0200 Subject: [PATCH 06/11] Add OTel cicd.* metrics and attributes --- .../plugins/opentelemetry/job/MonitoringRunListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index 80d94655b..d23b1c26b 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -80,7 +80,7 @@ import static com.google.common.base.Verify.verifyNotNull; /** - * TODO support reconfiguration. allow & deny lists are NOT reconfigurable at runtime for the moment. + * TODO support reconfiguration. allow and deny lists are NOT reconfigurable at runtime for the moment. */ @Extension(dynamicLoadable = YesNoMaybe.YES, optional = true) public class MonitoringRunListener extends OtelContextAwareAbstractRunListener implements OpenTelemetryLifecycleListener { From 714292090b05e02292182a6a82705c0ef0a8f356 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Mon, 19 May 2025 10:36:32 +0200 Subject: [PATCH 07/11] Add config flag `otel.semconv-stability.opt-in=cicd` --- .../JenkinsExecutorMonitoringInitializer.java | 162 ++++++++++++++---- .../opentelemetry/SemconvStability.java | 67 ++++++++ 2 files changed, 191 insertions(+), 38 deletions(-) create mode 100644 src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java b/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java index cf51efe73..0eb1a7c24 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/init/JenkinsExecutorMonitoringInitializer.java @@ -5,31 +5,33 @@ package io.jenkins.plugins.opentelemetry.init; -import static io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes.STATUS; - import hudson.Extension; import hudson.model.Computer; import hudson.model.LoadStatistics; import hudson.model.Node; import io.jenkins.plugins.opentelemetry.JenkinsControllerOpenTelemetry; import io.jenkins.plugins.opentelemetry.api.OpenTelemetryLifecycleListener; +import io.jenkins.plugins.opentelemetry.opentelemetry.SemconvStability; import io.jenkins.plugins.opentelemetry.semconv.CicdMetrics; import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.semconv.incubating.CicdIncubatingAttributes; import jenkins.YesNoMaybe; import jenkins.model.Jenkins; -import javax.annotation.PostConstruct; import javax.inject.Inject; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; +import static io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes.STATUS; import static io.jenkins.plugins.opentelemetry.semconv.JenkinsMetrics.*; @Extension(dynamicLoadable = YesNoMaybe.MAYBE, optional = true) @@ -40,35 +42,34 @@ public class JenkinsExecutorMonitoringInitializer implements OpenTelemetryLifecy @Inject JenkinsControllerOpenTelemetry jenkinsControllerOpenTelemetry; + @Inject + SemconvStability semconvStability; + + final AtomicBoolean initialized = new AtomicBoolean(false); + public JenkinsExecutorMonitoringInitializer() { logger.log(Level.FINE, () -> "JenkinsExecutorMonitoringInitializer constructor"); } - @PostConstruct - public void postConstruct() { + @Override + public synchronized void afterConfiguration(ConfigProperties configProperties) { + if (initialized.get()) { + logger.log(Level.FINE, () -> "JenkinsExecutorMonitoringInitializer already initialized. " + + "This component doesn't support reconfiguration beyond changing the OTel SDK. " + + "It doesn't support reconfiguration of 'otel.semconv-stability.opt-in' 'cicd' vs 'cicd/dup' "); + return; + } + initialized.set(true); - logger.log(Level.FINE, () -> "Start monitoring Jenkins controller executor pool..."); + logger.log(Level.INFO, () -> "Start monitoring Jenkins controller executor pool..."); Meter meter = Objects.requireNonNull(jenkinsControllerOpenTelemetry).getDefaultMeter(); + Meter oldSemConventionsMeter = semconvStability.emitOldCicdSemconv() ? meter : OpenTelemetry.noop().getMeter("jenkins.opentelemetry"); + Meter newSemConventionsMeter = semconvStability.emitStableCicdSemconv() ? meter : OpenTelemetry.noop().getMeter("jenkins.opentelemetry"); final ObservableLongMeasurement queueLength = meter.gaugeBuilder(JENKINS_EXECUTOR_QUEUE).setUnit("${items}").setDescription("Executors queue items").ofLongs().buildObserver(); - final ObservableLongMeasurement totalExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_TOTAL).setUnit("${executors}").setDescription("Total executors").ofLongs().buildObserver(); - final ObservableLongMeasurement nodes = meter.gaugeBuilder(JENKINS_NODE).setUnit("${nodes}").setDescription("Nodes").ofLongs().buildObserver(); final ObservableLongMeasurement executors = meter.gaugeBuilder(JENKINS_EXECUTOR_COUNT).setUnit("${executors}").setDescription("Count of executors per label").ofLongs().buildObserver(); - ObservableLongMeasurement cicdWorkers = CicdMetrics.newCiCdWorkerCounter(meter); - - // TODO the metrics below should be deprecated in favor of - // * `jenkins.executor` metric with the `status` and `label`attributes - // * `jenkins.node` metric with the `status` attribute - // * `jenkins.executor.total` metric with the `status` attribute - final ObservableLongMeasurement availableExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_AVAILABLE).setUnit("${executors}").setDescription("Available executors").ofLongs().buildObserver(); - final ObservableLongMeasurement busyExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_BUSY).setUnit("${executors}").setDescription("Busy executors").ofLongs().buildObserver(); - final ObservableLongMeasurement idleExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_IDLE).setUnit("${executors}").setDescription("Idle executors").ofLongs().buildObserver(); - final ObservableLongMeasurement onlineExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_ONLINE).setUnit("${executors}").setDescription("Online executors").ofLongs().buildObserver(); - final ObservableLongMeasurement connectingExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_CONNECTING).setUnit("${executors}").setDescription("Connecting executors").ofLongs().buildObserver(); - final ObservableLongMeasurement definedExecutors = meter.gaugeBuilder(JENKINS_EXECUTOR_DEFINED).setUnit("${executors}").setDescription("Defined executors").ofLongs().buildObserver(); - - logger.log(Level.FINER, () -> "Metrics: " + availableExecutors + ", " + busyExecutors + ", " + idleExecutors + ", " + onlineExecutors + ", " + connectingExecutors + ", " + definedExecutors + ", " + queueLength); + final ObservableLongMeasurement nodes = meter.gaugeBuilder(JENKINS_NODE).setUnit("${nodes}").setDescription("Nodes").ofLongs().buildObserver(); meter.batchCallback(() -> { logger.log(Level.FINE, () -> "Recording Jenkins controller executor pool metrics..."); @@ -79,14 +80,49 @@ public void postConstruct() { } // TOTAL EXECUTORS + AtomicInteger nodeOnline = new AtomicInteger(); + AtomicInteger nodeOffline = new AtomicInteger(); + + jenkins.getNodes().stream().map(Node::toComputer).filter(Objects::nonNull).forEach(node -> { + if (node.isOnline()) { + nodeOnline.incrementAndGet(); + } else { + nodeOffline.incrementAndGet(); + } + }); + + nodes.record(nodeOnline.get(), Attributes.of(STATUS, "online")); + nodes.record(nodeOffline.get(), Attributes.of(STATUS, "offline")); + + // PER LABEL + jenkins.getLabels().forEach(label -> { + LoadStatistics.LoadStatisticsSnapshot loadStatisticsSnapshot = label.loadStatistics.computeSnapshot(); + Attributes attributes = Attributes.of(ExtendedJenkinsAttributes.LABEL, label.getDisplayName()); + + executors.record(loadStatisticsSnapshot.getBusyExecutors(), attributes.toBuilder().put(STATUS, "busy").build()); + executors.record(loadStatisticsSnapshot.getIdleExecutors(), attributes.toBuilder().put(STATUS, "idle").build()); + executors.record(loadStatisticsSnapshot.getConnectingExecutors(), attributes.toBuilder().put(STATUS, "connecting").build()); + queueLength.record(loadStatisticsSnapshot.getQueueLength(), attributes); + + }); + }, executors, nodes, queueLength); + + ObservableLongMeasurement cicdWorkers = CicdMetrics.newCiCdWorkerCounter(newSemConventionsMeter); + + newSemConventionsMeter.batchCallback(() -> { + logger.log(Level.FINE, () -> "Recording Jenkins controller executor pool metrics..."); + Jenkins jenkins = Jenkins.getInstanceOrNull(); + if (jenkins == null) { + logger.log(Level.FINE, "Jenkins instance is null, skipping executor pool metrics recording"); + return; + } + + // EXECUTORS - AGGREGATED INDICATORS AtomicInteger totalExecutorsIdle = new AtomicInteger(); AtomicInteger totalExecutorsBusy = new AtomicInteger(); AtomicInteger totalExecutorsOffline = new AtomicInteger(); - AtomicInteger nodeOnline = new AtomicInteger(); - AtomicInteger nodeOffline = new AtomicInteger(); if (jenkins.getNumExecutors() > 0) { - nodeOnline.incrementAndGet(); Optional.ofNullable(jenkins.toComputer()) .map(Computer::getExecutors) .ifPresent(e -> e.forEach(executor -> { @@ -99,7 +135,6 @@ public void postConstruct() { } jenkins.getNodes().stream().map(Node::toComputer).filter(Objects::nonNull).forEach(node -> { if (node.isOnline()) { - nodeOnline.incrementAndGet(); node.getExecutors() .forEach(executor -> { if (executor.isIdle()) { @@ -110,31 +145,81 @@ public void postConstruct() { }); } else { totalExecutorsOffline.addAndGet(node.countExecutors()); - nodeOffline.incrementAndGet(); } }); - totalExecutors.record(totalExecutorsBusy.get(), Attributes.of(STATUS, "busy")); - totalExecutors.record(totalExecutorsIdle.get(), Attributes.of(STATUS, "idle")); cicdWorkers.record(totalExecutorsBusy.get(), Attributes.of(CicdIncubatingAttributes.CICD_WORKER_STATE, CicdIncubatingAttributes.CicdWorkerStateIncubatingValues.BUSY)); cicdWorkers.record(totalExecutorsIdle.get(), Attributes.of(CicdIncubatingAttributes.CICD_WORKER_STATE, CicdIncubatingAttributes.CicdWorkerStateIncubatingValues.AVAILABLE)); cicdWorkers.record(totalExecutorsOffline.get(), Attributes.of(CicdIncubatingAttributes.CICD_WORKER_STATE, CicdIncubatingAttributes.CicdWorkerStateIncubatingValues.OFFLINE)); - nodes.record(nodeOnline.get(), Attributes.of(STATUS, "online")); - nodes.record(nodeOffline.get(), Attributes.of(STATUS, "offline")); - // PER LABEL + }, cicdWorkers); + + createOldSemanticConventionsMeasurements(oldSemConventionsMeter); + } + + private static void createOldSemanticConventionsMeasurements(Meter oldSemConventionsMeter) { + final ObservableLongMeasurement totalExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_TOTAL).setUnit("${executors}").setDescription("Total executors").ofLongs().buildObserver(); + + // TODO the metrics below should be deprecated in favor of + // * `jenkins.executor` metric with the `status` and `label`attributes + // * `jenkins.node` metric with the `status` attribute + // * `jenkins.executor.total` metric with the `status` attribute + final ObservableLongMeasurement availableExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_AVAILABLE).setUnit("${executors}").setDescription("Available executors").ofLongs().buildObserver(); + final ObservableLongMeasurement busyExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_BUSY).setUnit("${executors}").setDescription("Busy executors").ofLongs().buildObserver(); + final ObservableLongMeasurement idleExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_IDLE).setUnit("${executors}").setDescription("Idle executors").ofLongs().buildObserver(); + final ObservableLongMeasurement onlineExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_ONLINE).setUnit("${executors}").setDescription("Online executors").ofLongs().buildObserver(); + final ObservableLongMeasurement connectingExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_CONNECTING).setUnit("${executors}").setDescription("Connecting executors").ofLongs().buildObserver(); + final ObservableLongMeasurement definedExecutors = oldSemConventionsMeter.gaugeBuilder(JENKINS_EXECUTOR_DEFINED).setUnit("${executors}").setDescription("Defined executors").ofLongs().buildObserver(); + + oldSemConventionsMeter.batchCallback(() -> { + logger.log(Level.FINE, () -> "Recording Jenkins controller executor pool metrics..."); + Jenkins jenkins = Jenkins.getInstanceOrNull(); + if (jenkins == null) { + logger.log(Level.FINE, "Jenkins instance is null, skipping executor pool metrics recording"); + return; + } + + // EXECUTORS - AGGREGATED INDICATORS + AtomicInteger totalExecutorsIdle = new AtomicInteger(); + AtomicInteger totalExecutorsBusy = new AtomicInteger(); + + if (jenkins.getNumExecutors() > 0) { + Optional.ofNullable(jenkins.toComputer()) + .map(Computer::getExecutors) + .ifPresent(e -> e.forEach(executor -> { + if (executor.isIdle()) { + totalExecutorsIdle.incrementAndGet(); + } else { + totalExecutorsBusy.incrementAndGet(); + } + })); + } + jenkins.getNodes().stream().map(Node::toComputer).filter(Objects::nonNull).forEach(node -> { + if (node.isOnline()) { + node.getExecutors() + .forEach(executor -> { + if (executor.isIdle()) { + totalExecutorsIdle.incrementAndGet(); + } else { + totalExecutorsBusy.incrementAndGet(); + } + }); + } else { + //offline + } + }); + + totalExecutors.record(totalExecutorsBusy.get(), Attributes.of(STATUS, "busy")); + totalExecutors.record(totalExecutorsIdle.get(), Attributes.of(STATUS, "idle")); + + // EXECUTORS - PER LABEL INDICATORS jenkins.getLabels().forEach(label -> { LoadStatistics.LoadStatisticsSnapshot loadStatisticsSnapshot = label.loadStatistics.computeSnapshot(); Attributes attributes = Attributes.of(ExtendedJenkinsAttributes.LABEL, label.getDisplayName()); - executors.record(loadStatisticsSnapshot.getBusyExecutors(), attributes.toBuilder().put(STATUS, "busy").build()); - executors.record(loadStatisticsSnapshot.getIdleExecutors(), attributes.toBuilder().put(STATUS, "idle").build()); - executors.record(loadStatisticsSnapshot.getConnectingExecutors(), attributes.toBuilder().put(STATUS, "connecting").build()); - queueLength.record(loadStatisticsSnapshot.getQueueLength(), attributes); - // TODO the metrics below should be deprecated in favor of `jenkins.executor` metric with the `status` // and `label`attributes availableExecutors.record(loadStatisticsSnapshot.getAvailableExecutors(), attributes); @@ -144,6 +229,7 @@ public void postConstruct() { definedExecutors.record(loadStatisticsSnapshot.getDefinedExecutors(), attributes); connectingExecutors.record(loadStatisticsSnapshot.getConnectingExecutors(), attributes); }); - }, availableExecutors, busyExecutors, idleExecutors, onlineExecutors, connectingExecutors, definedExecutors, totalExecutors, executors, nodes, queueLength, cicdWorkers); + }, availableExecutors, busyExecutors, idleExecutors, onlineExecutors, connectingExecutors, definedExecutors, totalExecutors); } + } diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java new file mode 100644 index 000000000..d3d968542 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java @@ -0,0 +1,67 @@ +package io.jenkins.plugins.opentelemetry.opentelemetry; + +import hudson.Extension; +import io.jenkins.plugins.opentelemetry.api.OpenTelemetryLifecycleListener; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import jenkins.YesNoMaybe; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Extension(dynamicLoadable = YesNoMaybe.MAYBE, optional = true) +public class SemconvStability implements OpenTelemetryLifecycleListener { + + private static final Logger logger = Logger.getLogger(SemconvStability.class.getName()); + + private final AtomicInteger configurationCounter = new AtomicInteger(0); + + private boolean emitOldCicdSemconv = true; + private boolean emitStableCicdSemconv = false; + + public boolean emitOldCicdSemconv() { + return emitOldCicdSemconv; + } + + public boolean emitStableCicdSemconv() { + return emitStableCicdSemconv; + } + + @Override + public void afterConfiguration(ConfigProperties configProperties) { + boolean oldCicd = true; + boolean stableCicd = false; + + String value = configProperties.getString("otel.semconv-stability.opt-in"); + if (value != null) { + Set values = new HashSet<>(Arrays.asList(value.split(","))); + if (values.contains("cicd")) { + oldCicd = false; + stableCicd = true; + } + // no else -- technically it's possible to set "cicd,cicd/dup", in which case we + // should emit both sets of attributes + if (values.contains("cicd/dup")) { + oldCicd = true; + stableCicd = true; + } + } + if (configurationCounter.get() > 0 && (emitOldCicdSemconv != oldCicd || emitStableCicdSemconv != stableCicd)) { + logger.log(Level.INFO, "SemconvStability: configuration changes from " + + "emitOldCicdSemconv=" + emitOldCicdSemconv + " to " + oldCicd + + ", emitStableCicdSemconv=" + emitStableCicdSemconv + " to " + stableCicd + " may not support hot reload and may require restart"); + } + emitOldCicdSemconv = oldCicd; + emitStableCicdSemconv = stableCicd; + configurationCounter.incrementAndGet(); + logger.log(Level.INFO, () -> "SemconvStability: emitOldCicdSemconv=" + emitOldCicdSemconv + ", emitStableCicdSemconv=" + emitStableCicdSemconv); + } + + @Override + public int ordinal() { + return Integer.MIN_VALUE; + } +} From e1a769322e3946ecec3b1231b25347fa7fea4f0a Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Mon, 19 May 2025 14:59:34 +0200 Subject: [PATCH 08/11] Add config flag `otel.semconv-stability.opt-in=cicd` --- .../job/MonitoringRunListener.java | 74 ++++++++++++++----- .../opentelemetry/SemconvStability.java | 2 +- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index d23b1c26b..ea022934a 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -26,11 +26,13 @@ import io.jenkins.plugins.opentelemetry.job.cause.CauseHandler; import io.jenkins.plugins.opentelemetry.job.opentelemetry.OtelContextAwareAbstractRunListener; import io.jenkins.plugins.opentelemetry.job.runhandler.RunHandler; +import io.jenkins.plugins.opentelemetry.opentelemetry.SemconvStability; import io.jenkins.plugins.opentelemetry.queue.RemoteSpanAction; import io.jenkins.plugins.opentelemetry.semconv.CicdMetrics; import io.jenkins.plugins.opentelemetry.semconv.ConfigurationKey; import io.jenkins.plugins.opentelemetry.semconv.ExtendedJenkinsAttributes; import io.jenkins.plugins.opentelemetry.semconv.JenkinsMetrics; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogramBuilder; import io.opentelemetry.api.metrics.DoubleHistogram; @@ -101,18 +103,43 @@ public class MonitoringRunListener extends OtelContextAwareAbstractRunListener i */ @Deprecated private AtomicInteger activeRunGauge; - private List causeHandlers; /** * @deprecated use {@link #cicdPipelineRunDurationHistogram} */ @Deprecated private DoubleHistogram runDurationHistogram; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private LongCounter runLaunchedCounter; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private LongCounter runStartedCounter; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private LongCounter runCompletedCounter; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private LongCounter runAbortedCounter; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private LongCounter runSuccessCounter; + /** + * @deprecated use {@link #cicdPipelineRunDurationHistogram} + */ + @Deprecated private LongCounter runFailedCounter; + + private List causeHandlers; private List runHandlers; @VisibleForTesting Pattern runDurationHistogramAllowList; @@ -129,6 +156,9 @@ public void postConstruct() { LOGGER.log(Level.FINE, () -> "Start monitoring Jenkins build executions..."); Meter meter = getMeter(); ConfigProperties configProperties = getConfigProperties(); + SemconvStability semconvStability = new SemconvStability(); + semconvStability.afterConfiguration(configProperties); + Meter newSemConventionsMeter = semconvStability.emitStableCicdSemconv() ? meter : OpenTelemetry.noop().getMeter("jenkins.opentelemetry"); // CAUSE HANDLERS List causeHandlers = new ArrayList<>(ExtensionList.lookup(CauseHandler.class)); @@ -143,9 +173,24 @@ public void postConstruct() { this.runHandlers = runHandlers; // METRICS + runDurationHistogramAllowList = MATCH_ANYTHING; // allow all + runDurationHistogramDenyList = MATCH_NOTHING; // deny nothing + + cicdPipelineRunDurationHistogram = CicdMetrics.newCiCdPipelineRunDurationHistogram(newSemConventionsMeter); + cicdPipelineRunActiveCounter = CicdMetrics.newCiCdPipelineRunActiveCounter(newSemConventionsMeter); + cicdPipelineRunErrorsCounter = CicdMetrics.newCiCdPipelineRunErrorsCounter(newSemConventionsMeter); + // TODO when to qualify a build failure as a cicd system error? + cicdSystemErrorsCounter = CicdMetrics.newCiCdSystemErrorsCounter(newSemConventionsMeter); + + createOldSemanticConventionsMeasurements(semconvStability, meter); + } + + private void createOldSemanticConventionsMeasurements(SemconvStability semconvStability, Meter meter) { + Meter oldSemConventionsMeter = semconvStability.emitOldCicdSemconv() ? meter : OpenTelemetry.noop().getMeter("jenkins.opentelemetry"); + activeRunGauge = new AtomicInteger(); - DoubleHistogramBuilder runDurationHistogramBuilder = meter.histogramBuilder(JenkinsMetrics.CI_PIPELINE_RUN_DURATION) + DoubleHistogramBuilder runDurationHistogramBuilder = oldSemConventionsMeter.histogramBuilder(JenkinsMetrics.CI_PIPELINE_RUN_DURATION) .setUnit("s") .setExplicitBucketBoundariesAdvice(DURATION_SECONDS_BUCKETS); if (runDurationHistogramBuilder instanceof ExtendedDoubleHistogramBuilder extendedBuilder) { @@ -155,49 +200,38 @@ public void postConstruct() { )); } runDurationHistogram = runDurationHistogramBuilder.build(); - runDurationHistogramAllowList = MATCH_ANYTHING; // allow all - runDurationHistogramDenyList = MATCH_NOTHING; // deny nothing - - cicdPipelineRunDurationHistogram = CicdMetrics.newCiCdPipelineRunDurationHistogram(meter); - cicdPipelineRunActiveCounter = CicdMetrics.newCiCdPipelineRunActiveCounter(meter); - cicdPipelineRunErrorsCounter = CicdMetrics.newCiCdPipelineRunErrorsCounter(meter); - /** - * FIXME when to qualify a build failure as a cicd system error? - */ - cicdSystemErrorsCounter = CicdMetrics.newCiCdSystemErrorsCounter(meter); - - meter.gaugeBuilder(JenkinsMetrics.CI_PIPELINE_RUN_ACTIVE) + oldSemConventionsMeter.gaugeBuilder(JenkinsMetrics.CI_PIPELINE_RUN_ACTIVE) .ofLongs() .setDescription("Gauge of active jobs") .setUnit("{jobs}") .buildWithCallback(valueObserver -> valueObserver.record(this.activeRunGauge.get())); runLaunchedCounter = - meter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_LAUNCHED) + oldSemConventionsMeter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_LAUNCHED) .setDescription("Job launched") .setUnit("{jobs}") .build(); runStartedCounter = - meter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_STARTED) + oldSemConventionsMeter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_STARTED) .setDescription("Job started") .setUnit("{jobs}") .build(); runSuccessCounter = - meter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_SUCCESS) + oldSemConventionsMeter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_SUCCESS) .setDescription("Job succeed") .setUnit("{jobs}") .build(); runFailedCounter = - meter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_FAILED) + oldSemConventionsMeter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_FAILED) .setDescription("Job failed") .setUnit("{jobs}") .build(); runAbortedCounter = - meter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_ABORTED) + oldSemConventionsMeter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_ABORTED) .setDescription("Job aborted") .setUnit("{jobs}") .build(); runCompletedCounter = - meter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_COMPLETED) + oldSemConventionsMeter.counterBuilder(JenkinsMetrics.CI_PIPELINE_RUN_COMPLETED) .setDescription("Job completed") .setUnit("{jobs}") .build(); diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java index d3d968542..8af2656c3 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java @@ -57,7 +57,7 @@ public void afterConfiguration(ConfigProperties configProperties) { emitOldCicdSemconv = oldCicd; emitStableCicdSemconv = stableCicd; configurationCounter.incrementAndGet(); - logger.log(Level.INFO, () -> "SemconvStability: emitOldCicdSemconv=" + emitOldCicdSemconv + ", emitStableCicdSemconv=" + emitStableCicdSemconv); + logger.log(Level.FINE, () -> "SemconvStability: emitOldCicdSemconv=" + emitOldCicdSemconv + ", emitStableCicdSemconv=" + emitStableCicdSemconv); } @Override From 327a59e10df055e57f91dc91cf8409ff628136f3 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Mon, 19 May 2025 15:11:38 +0200 Subject: [PATCH 09/11] Add config flag `otel.semconv-stability.opt-in=cicd` --- .../plugins/opentelemetry/opentelemetry/SemconvStability.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java index 8af2656c3..b87e62c52 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java @@ -33,7 +33,7 @@ public boolean emitStableCicdSemconv() { @Override public void afterConfiguration(ConfigProperties configProperties) { boolean oldCicd = true; - boolean stableCicd = false; + boolean stableCicd = true; String value = configProperties.getString("otel.semconv-stability.opt-in"); if (value != null) { From 01a9221393e44bc8453379245f4e1ecb14eb4593 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Mon, 26 May 2025 16:34:29 +0200 Subject: [PATCH 10/11] Add config flag `otel.semconv-stability.opt-in=[cicd,cicd/dup,cicd/old]` --- .../opentelemetry/SemconvStability.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java index b87e62c52..b6416952f 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java @@ -20,7 +20,7 @@ public class SemconvStability implements OpenTelemetryLifecycleListener { private final AtomicInteger configurationCounter = new AtomicInteger(0); private boolean emitOldCicdSemconv = true; - private boolean emitStableCicdSemconv = false; + private boolean emitStableCicdSemconv = true; public boolean emitOldCicdSemconv() { return emitOldCicdSemconv; @@ -38,15 +38,17 @@ public void afterConfiguration(ConfigProperties configProperties) { String value = configProperties.getString("otel.semconv-stability.opt-in"); if (value != null) { Set values = new HashSet<>(Arrays.asList(value.split(","))); - if (values.contains("cicd")) { - oldCicd = false; - stableCicd = true; - } - // no else -- technically it's possible to set "cicd,cicd/dup", in which case we - // should emit both sets of attributes + + // technically it's possible to set "cicd,cicd/dup" or "cicd,cicd/old" if (values.contains("cicd/dup")) { oldCicd = true; stableCicd = true; + } else if (values.contains("cicd/old")) { + oldCicd = true; + stableCicd = false; + } else if (values.contains("cicd")) { + oldCicd = false; + stableCicd = true; } } if (configurationCounter.get() > 0 && (emitOldCicdSemconv != oldCicd || emitStableCicdSemconv != stableCicd)) { From 3262dcf8609bda2e25df49d63857d9144d0f6903 Mon Sep 17 00:00:00 2001 From: Cyrille Le Clerc Date: Tue, 13 Jan 2026 14:36:56 +0100 Subject: [PATCH 11/11] WIP --- .../job/MonitoringRunListener.java | 30 ++++++++++++------- .../opentelemetry/SemconvStability.java | 8 ++++- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java index 341e2cfa5..fcf375133 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/job/MonitoringRunListener.java @@ -151,18 +151,13 @@ public class MonitoringRunListener extends OtelContextAwareAbstractRunListener private DoubleHistogram cicdPipelineRunDurationHistogram; private LongUpDownCounter cicdPipelineRunActiveCounter; private LongCounter cicdPipelineRunErrorsCounter; - private LongCounter cicdSystemErrorsCounter; + private LongCounter cicdSystemErrorsCounter; // TODO implement @PostConstruct public void postConstruct() { LOGGER.log(Level.FINE, () -> "Start monitoring Jenkins build executions..."); Meter meter = getMeter(); ConfigProperties configProperties = getConfigProperties(); - SemconvStability semconvStability = new SemconvStability(); - semconvStability.afterConfiguration(configProperties); - Meter newSemConventionsMeter = semconvStability.emitStableCicdSemconv() - ? meter - : OpenTelemetry.noop().getMeter("jenkins.opentelemetry"); // CAUSE HANDLERS List causeHandlers = new ArrayList<>(ExtensionList.lookup(CauseHandler.class)); @@ -180,6 +175,15 @@ public void postConstruct() { runDurationHistogramAllowList = MATCH_ANYTHING; // allow all runDurationHistogramDenyList = MATCH_NOTHING; // deny nothing + // keep the init of the new semantic conventions metrics in the `postConstruct()` to be consistent with + // the initialization of other metrics + // TODO only create new CICD semconv metrics if semconvStability.emitStableCicdSemconv() is true + // else shall we use use no-op metrics + SemconvStability semconvStability = new SemconvStability(); + semconvStability.afterConfiguration(configProperties); + Meter newSemConventionsMeter = semconvStability.emitStableCicdSemconv() + ? meter + : OpenTelemetry.noop().getMeter("jenkins.opentelemetry"); cicdPipelineRunDurationHistogram = CicdMetrics.newCiCdPipelineRunDurationHistogram(newSemConventionsMeter); cicdPipelineRunActiveCounter = CicdMetrics.newCiCdPipelineRunActiveCounter(newSemConventionsMeter); cicdPipelineRunErrorsCounter = CicdMetrics.newCiCdPipelineRunErrorsCounter(newSemConventionsMeter); @@ -189,6 +193,10 @@ public void postConstruct() { createOldSemanticConventionsMeasurements(semconvStability, meter); } + /** + * Locate the initialization of old semantic conventions measurements in a dedicated method to isolate the code, + * prevent misuse of the no-op meter, and ase removal of the code + */ private void createOldSemanticConventionsMeasurements(SemconvStability semconvStability, Meter meter) { Meter oldSemConventionsMeter = semconvStability.emitOldCicdSemconv() ? meter @@ -557,7 +565,7 @@ public void _onFinalized(@NonNull Run run) { 1, Attributes.of( CicdIncubatingAttributes.CICD_PIPELINE_NAME, - PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + PIPELINE_NAME_OTHER, // FIXME IMPLEMENT CARDINALITY PROTECTION ErrorAttributes.ERROR_TYPE, runResult.toString())); } else if (Result.ABORTED.equals(runResult) || Result.NOT_BUILT.equals(runResult)) { @@ -566,7 +574,7 @@ public void _onFinalized(@NonNull Run run) { 1, Attributes.of( CicdIncubatingAttributes.CICD_PIPELINE_NAME, - PIPELINE_NAME_OTHER, // FIXME CARDINALITY PROTECTION + PIPELINE_NAME_OTHER, // FIXME IMPLEMENT CARDINALITY PROTECTION ErrorAttributes.ERROR_TYPE, runResult.toString())); } @@ -608,7 +616,7 @@ public void _onFinalized(@NonNull Run run) { .orElseThrow((Supplier) () -> new IllegalStateException("No RunHandler found for run " + run.getClass() + " - " + run)) .getPipelineShortName(run); - String pipelineId = + String pipelineName = runDurationHistogramAllowList.matcher(pipelineShortName).matches() && !runDurationHistogramDenyList .matcher(pipelineShortName) @@ -619,7 +627,7 @@ public void _onFinalized(@NonNull Run run) { TimeUnit.SECONDS.convert(run.getDuration(), TimeUnit.MILLISECONDS), Attributes.of( ExtendedJenkinsAttributes.CI_PIPELINE_ID, - pipelineId, + pipelineName, ExtendedJenkinsAttributes.CI_PIPELINE_RUN_RESULT, result.toString())); @@ -628,7 +636,7 @@ public void _onFinalized(@NonNull Run run) { TimeUnit.SECONDS.convert(run.getDuration(), TimeUnit.MILLISECONDS), Attributes.of( CicdIncubatingAttributes.CICD_PIPELINE_NAME, - pipelineId, + pipelineName, CicdIncubatingAttributes.CICD_PIPELINE_RUN_STATE, CicdIncubatingAttributes.CicdPipelineRunStateIncubatingValues.FINALIZING, CicdIncubatingAttributes.CICD_PIPELINE_RESULT, diff --git a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java index ecb46b465..d96b3b962 100644 --- a/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java +++ b/src/main/java/io/jenkins/plugins/opentelemetry/opentelemetry/SemconvStability.java @@ -11,6 +11,11 @@ import java.util.logging.Logger; import jenkins.YesNoMaybe; +/** + * Manages the configuration for semantic convention stability opt-in. + * Users can configure which version of the CI/CD semantic conventions to emit. + * TODO support hot config changes or document the need to restart Jenkins on config change. + */ @Extension(dynamicLoadable = YesNoMaybe.MAYBE, optional = true) public class SemconvStability implements OpenTelemetryLifecycleListener { @@ -34,7 +39,8 @@ public void afterConfiguration(ConfigProperties configProperties) { boolean oldCicd = true; boolean stableCicd = true; - String value = configProperties.getString("otel.semconv-stability.opt-in"); + // default to just emitting the old style metrics + String value = configProperties.getString("otel.semconv-stability.opt-in", "cicd/old"); if (value != null) { Set values = new HashSet<>(Arrays.asList(value.split(",")));